From c1f9ed991b9679d789389741eaa3248075774572 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 30 Jul 2017 21:43:59 +0200 Subject: [PATCH 001/798] vagrant: add exe location to PATH when we build an exe --- Vagrantfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 87897c023b..0f94c257fa 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -227,7 +227,7 @@ end def install_pyenv(boxname) script = <<-EOF curl -s -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash - echo 'export PATH="$HOME/.pyenv/bin:/vagrant/borg:$PATH"' >> ~/.bash_profile + echo 'export PATH="$HOME/.pyenv/bin:$PATH"' >> ~/.bash_profile echo 'eval "$(pyenv init -)"' >> ~/.bash_profile echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bash_profile echo 'export PYTHON_CONFIGURE_OPTS="--enable-shared"' >> ~/.bash_profile @@ -320,6 +320,7 @@ def build_binary_with_pyinstaller(boxname) . borg-env/bin/activate cd borg pyinstaller --clean --distpath=/vagrant/borg scripts/borg.exe.spec + echo 'export PATH="/vagrant/borg:$PATH"' >> ~/.bash_profile EOF end From 1f5eba0cb8679cf4520f519be36efd18a18e29e9 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 30 Jul 2017 21:55:15 +0200 Subject: [PATCH 002/798] vagrant: fix netbsd version in PKG_PATH the VM is using netbsd 7.0.1, likely updating the version in the PKG_PATH was forgotten when the machine itself was upgraded to a newer version. it became visible now as the 6.1.5 stuff vanished from the ftp server. also: remove trailing / --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 0f94c257fa..473d68ab0e 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -139,7 +139,7 @@ end def packages_netbsd return <<-EOF hostname netbsd # the box we use has an invalid hostname - PKG_PATH="ftp://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/amd64/6.1.5/All/" + PKG_PATH="ftp://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/amd64/7.0.1/All" export PKG_PATH pkg_add mozilla-rootcerts lz4 git bash chsh -s bash vagrant From 4e40f8507dbcb26e027166b8dc7209599633fecc Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 30 Jul 2017 22:02:43 +0200 Subject: [PATCH 003/798] vagrant: netbsd: bash is already installed --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 473d68ab0e..2df83ff658 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -141,7 +141,7 @@ def packages_netbsd hostname netbsd # the box we use has an invalid hostname PKG_PATH="ftp://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/amd64/7.0.1/All" export PKG_PATH - pkg_add mozilla-rootcerts lz4 git bash + pkg_add mozilla-rootcerts lz4 git chsh -s bash vagrant mkdir -p /usr/local/opt/lz4/include mkdir -p /usr/local/opt/lz4/lib From a1bffc193b808fc9ef742b1e39f779f4306f50c4 Mon Sep 17 00:00:00 2001 From: Simon Frei Date: Sun, 30 Jul 2017 19:35:38 +0200 Subject: [PATCH 004/798] Only compare contents when chunker params match (fixes #2899) --- src/borg/archiver.py | 25 +++++++++++++------------ src/borg/testsuite/archiver.py | 14 ++++++++------ 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 4dc5dacce5..31e0759bb2 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -1013,18 +1013,19 @@ def contents_changed(item1, item2): def compare_content(path, item1, item2): if contents_changed(item1, item2): if item1.get('deleted'): - return ('added {:>13}'.format(format_file_size(sum_chunk_size(item2)))) - elif item2.get('deleted'): - return ('removed {:>11}'.format(format_file_size(sum_chunk_size(item1)))) - else: - chunk_ids1 = {c.id for c in item1.chunks} - chunk_ids2 = {c.id for c in item2.chunks} - added_ids = chunk_ids2 - chunk_ids1 - removed_ids = chunk_ids1 - chunk_ids2 - added = sum_chunk_size(item2, added_ids) - removed = sum_chunk_size(item1, removed_ids) - return ('{:>9} {:>9}'.format(format_file_size(added, precision=1, sign=True), - format_file_size(-removed, precision=1, sign=True))) + return 'added {:>13}'.format(format_file_size(sum_chunk_size(item2))) + if item2.get('deleted'): + return 'removed {:>11}'.format(format_file_size(sum_chunk_size(item1))) + if not can_compare_chunk_ids: + return 'modified' + chunk_ids1 = {c.id for c in item1.chunks} + chunk_ids2 = {c.id for c in item2.chunks} + added_ids = chunk_ids2 - chunk_ids1 + removed_ids = chunk_ids1 - chunk_ids2 + added = sum_chunk_size(item2, added_ids) + removed = sum_chunk_size(item1, removed_ids) + return '{:>9} {:>9}'.format(format_file_size(added, precision=1, sign=True), + format_file_size(-removed, precision=1, sign=True)) def compare_directory(item1, item2): if item2.get('deleted') and not item1.get('deleted'): diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 58b52866d0..c342ee8cbd 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -3276,9 +3276,10 @@ def test_basic_functionality(self): self.cmd('create', self.repository_location + '::test1a', 'input') self.cmd('create', '--chunker-params', '16,18,17,4095', self.repository_location + '::test1b', 'input') - def do_asserts(output, archive): + def do_asserts(output, can_compare_ids): # File contents changed (deleted and replaced with a new file) - assert 'B input/file_replaced' in output + change = 'B' if can_compare_ids else '{:<19}'.format('modified') + assert '{} input/file_replaced'.format(change) in output # File unchanged assert 'input/file_unchanged' not in output @@ -3307,9 +3308,10 @@ def do_asserts(output, archive): # The inode has two links and the file contents changed. Borg # should notice the changes in both links. However, the symlink # pointing to the file is not changed. - assert '0 B input/empty' in output + change = '0 B' if can_compare_ids else '{:<19}'.format('modified') + assert '{} input/empty'.format(change) in output if are_hardlinks_supported(): - assert '0 B input/hardlink_contents_changed' in output + assert '{} input/hardlink_contents_changed'.format(change) in output if are_symlinks_supported(): assert 'input/link_target_contents_changed' not in output @@ -3336,9 +3338,9 @@ def do_asserts(output, archive): if are_hardlinks_supported(): assert 'input/hardlink_target_replaced' not in output - do_asserts(self.cmd('diff', self.repository_location + '::test0', 'test1a'), '1a') + do_asserts(self.cmd('diff', self.repository_location + '::test0', 'test1a'), True) # We expect exit_code=1 due to the chunker params warning - do_asserts(self.cmd('diff', self.repository_location + '::test0', 'test1b', exit_code=1), '1b') + do_asserts(self.cmd('diff', self.repository_location + '::test0', 'test1b', exit_code=1), False) def test_sort_option(self): self.cmd('init', '--encryption=repokey', self.repository_location) From 68d505c974a27741f7c56244abb08986619eea87 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 6 Aug 2017 04:18:49 +0200 Subject: [PATCH 005/798] fix exitcode of borg serve, fixes #2910 (cherry picked from commit 583de3eeb15ba9f1c5574691eb42a9d10270a5a9) --- src/borg/archiver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 4dc5dacce5..7df210e5b0 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -231,12 +231,13 @@ def build_matcher(inclexcl_patterns, include_paths): def do_serve(self, args): """Start in server mode. This command is usually not used manually.""" - return RepositoryServer( + RepositoryServer( restrict_to_paths=args.restrict_to_paths, restrict_to_repositories=args.restrict_to_repositories, append_only=args.append_only, storage_quota=args.storage_quota, ).serve() + return EXIT_SUCCESS @with_repository(create=True, exclusive=True, manifest=False) def do_init(self, args, repository): From d17553add64d0875a81ec87021c95c2140be946a Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 6 Aug 2017 01:59:55 +0200 Subject: [PATCH 006/798] archive listing: use iso8601 timestamp format with --json-lines like yyyy-mm-ddThh:mm:ss - no tz yet, this likely needs more refactoring to tz aware and utc datetime objects everywhere, currently there are naive datetime objects and also localtime at quite some places. (cherry picked from commit 043d794b91a6c3b3ce18fef18cdbd256df398d04) --- src/borg/helpers.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 1b6d38d834..b28ac7c5b6 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -717,11 +717,17 @@ def safe_timestamp(item_timestamp_ns): def format_time(t): - """use ISO-8601 date and time format + """use ISO-8601-like date and time format (human readable, with wkday and blank date/time separator) """ return t.strftime('%a, %Y-%m-%d %H:%M:%S') +def isoformat_time(t): + """use ISO-8601 date and time format (machine readable, no wkday, no microseconds either) + """ + return t.strftime('%Y-%m-%dT%H:%M:%S') # note: first make all datetime objects tz aware before adding %z here. + + def format_timedelta(td): """Format timedelta in a human friendly format """ @@ -1762,6 +1768,12 @@ def __init__(self, archive, format, *, json_lines=False): 'archiveid': archive.fpr, } static_keys.update(self.FIXED_KEYS) + if self.json_lines: + self.item_data = {} + self.format_item = self.format_item_json + self.format_time = self.format_time_json + else: + self.item_data = static_keys self.format = partial_format(format, static_keys) self.format_keys = {f[1] for f in Formatter().parse(format)} self.call_keys = { @@ -1781,11 +1793,6 @@ def __init__(self, archive, format, *, json_lines=False): for hash_function in hashlib.algorithms_guaranteed: self.add_key(hash_function, partial(self.hash_item, hash_function)) self.used_call_keys = set(self.call_keys) & self.format_keys - if self.json_lines: - self.item_data = {} - self.format_item = self.format_item_json - else: - self.item_data = static_keys def format_item_json(self, item): return json.dumps(self.get_item_data(item)) + '\n' @@ -1863,7 +1870,12 @@ def hash_item(self, hash_function, item): return hash.hexdigest() def format_time(self, key, item): - return format_time(safe_timestamp(item.get(key) or item.mtime)) + t = self.time(key, item) + return format_time(t) + + def format_time_json(self, key, item): + t = self.time(key, item) + return isoformat_time(t) def time(self, key, item): return safe_timestamp(item.get(key) or item.mtime) From 65940be21a3212f3c676fa7e7667c3b69ea5d312 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 6 Aug 2017 02:13:17 +0200 Subject: [PATCH 007/798] archives list: use iso8601 timestamp format with --json like yyyy-mm-ddThh:mm:ss - no tz yet, this likely needs more refactoring to tz aware and utc datetime objects everywhere, currently there are naive datetime objects and also localtime at quite some places. (cherry picked from commit b64561fe6f067ec28ccd2b89d00f8e38e3397814) --- src/borg/helpers.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/borg/helpers.py b/src/borg/helpers.py index b28ac7c5b6..d6720854c6 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -1660,6 +1660,7 @@ def __init__(self, format, repository, manifest, key, *, json=False): if self.json: self.item_data = {} self.format_item = self.format_item_json + self.format_time = self.format_time_json else: self.item_data = static_keys @@ -1676,8 +1677,8 @@ def get_item_data(self, archive_info): 'archive': remove_surrogates(archive_info.name), 'barchive': archive_info.name, 'id': bin_to_hex(archive_info.id), - 'time': format_time(to_localtime(archive_info.ts)), - 'start': format_time(to_localtime(archive_info.ts)), + 'time': self.format_time(archive_info.ts), + 'start': self.format_time(archive_info.ts), }) for key in self.used_call_keys: item_data[key] = self.call_keys[key]() @@ -1695,7 +1696,15 @@ def get_comment(self, rs): return remove_surrogates(self.archive.comment) if rs else self.archive.comment def get_ts_end(self): - return format_time(to_localtime(self.archive.ts_end)) + return self.format_time(self.archive.ts_end) + + def format_time(self, ts): + t = to_localtime(ts) + return format_time(t) + + def format_time_json(self, ts): + t = to_localtime(ts) + return isoformat_time(t) class ItemFormatter(BaseFormatter): From 74c9277d7649d9168b4b40660cbef6ec97b478f2 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 6 Aug 2017 02:22:38 +0200 Subject: [PATCH 008/798] repo last_modified: use iso8601 timestamp format with --json like yyyy-mm-ddThh:mm:ss - no tz yet, this likely needs more refactoring to tz aware and utc datetime objects everywhere, currently there are naive datetime objects and also localtime at quite some places. (cherry picked from commit 32174dd9c885e2267a939cb81fe2d4130308d487) --- src/borg/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/borg/helpers.py b/src/borg/helpers.py index d6720854c6..fed3b769b9 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -2212,7 +2212,7 @@ def basic_json_data(manifest, *, cache=None, extra=None): 'mode': key.ARG_NAME, }, }) - data['repository']['last_modified'] = format_time(to_localtime(manifest.last_timestamp.replace(tzinfo=timezone.utc))) + data['repository']['last_modified'] = isoformat_time(to_localtime(manifest.last_timestamp.replace(tzinfo=timezone.utc))) if key.NAME.startswith('key file'): data['encryption']['keyfile'] = key.find_key() if cache: From eb7d473dc79f601b07f49b4a87b3b1ba30c91046 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 6 Aug 2017 23:49:18 +0200 Subject: [PATCH 009/798] test json timestamps for iso format (cherry picked from commit c3b0214e89e99b4cb2ae1fd6e0070603dabdffef) --- src/borg/testsuite/archiver.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index c342ee8cbd..a26bc47cfd 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -60,6 +60,8 @@ src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +ISO_FORMAT = '%Y-%m-%dT%H:%M:%S' + def exec_cmd(*args, archiver=None, fork=False, exe=None, input=b'', binary_output=False, **kw): if fork: @@ -1304,6 +1306,7 @@ def test_info_json(self): repository = info_repo['repository'] assert len(repository['id']) == 64 assert 'last_modified' in repository + assert datetime.strptime(repository['last_modified'], ISO_FORMAT) # must not raise assert info_repo['encryption']['mode'] == 'repokey' assert 'keyfile' not in info_repo['encryption'] cache = info_repo['cache'] @@ -1846,9 +1849,11 @@ def test_list_json(self): list_repo = json.loads(self.cmd('list', '--json', self.repository_location)) repository = list_repo['repository'] assert len(repository['id']) == 64 - assert 'last_modified' in repository + assert datetime.strptime(repository['last_modified'], ISO_FORMAT) # must not raise assert list_repo['encryption']['mode'] == 'repokey' assert 'keyfile' not in list_repo['encryption'] + archive0 = list_repo['archives'][0] + assert datetime.strptime(archive0['time'], ISO_FORMAT) # must not raise list_archive = self.cmd('list', '--json-lines', self.repository_location + '::test') items = [json.loads(s) for s in list_archive.splitlines()] @@ -1856,6 +1861,7 @@ def test_list_json(self): file1 = items[1] assert file1['path'] == 'input/file1' assert file1['size'] == 81920 + assert datetime.strptime(file1['isomtime'], ISO_FORMAT) # must not raise list_archive = self.cmd('list', '--json-lines', '--format={sha256}', self.repository_location + '::test') items = [json.loads(s) for s in list_archive.splitlines()] From 09e3a02fbc5edd64125ac437cb3ec6fb20ce730a Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 7 Aug 2017 04:37:20 +0200 Subject: [PATCH 010/798] migrate locks to child PID when daemonize is used also: increase platform api version due to change in get_process_id behaviour. (cherry picked from commit 6f94949a36607f977bdb8ac835e98647896e4b7a) (cherry picked from commit 5bad764637ece004cccce6273d5028b97148c9dc) --- src/borg/fuse.py | 6 ++++- src/borg/helpers.py | 8 ++++++- src/borg/locking.py | 40 ++++++++++++++++++++++++++++++++++ src/borg/platform/base.py | 2 +- src/borg/platform/darwin.pyx | 2 +- src/borg/platform/freebsd.pyx | 2 +- src/borg/platform/linux.pyx | 2 +- src/borg/platform/posix.pyx | 12 +++++----- src/borg/repository.py | 5 +++++ src/borg/testsuite/locking.py | 41 +++++++++++++++++++++++++++++++++++ 10 files changed, 107 insertions(+), 13 deletions(-) diff --git a/src/borg/fuse.py b/src/borg/fuse.py index 7868eb3782..129e3b20b7 100644 --- a/src/borg/fuse.py +++ b/src/borg/fuse.py @@ -22,6 +22,7 @@ from .helpers import daemonize, hardlinkable, signal_handler, format_file_size from .item import Item from .lrucache import LRUCache +from .remote import RemoteRepository # Does this version of llfuse support ns precision? have_fuse_xtime_ns = hasattr(llfuse.EntryAttributes, 'st_mtime_ns') @@ -285,7 +286,10 @@ def mount(self, mountpoint, mount_options, foreground=False): self._create_filesystem() llfuse.init(self, mountpoint, options) if not foreground: - daemonize() + old_id, new_id = daemonize() + if not isinstance(self.repository_uncached, RemoteRepository): + # local repo and the locking process' PID just changed, migrate it: + self.repository_uncached.migrate_lock(old_id, new_id) # If the file system crashes, we do not want to umount because in that # case the mountpoint suddenly appears to become empty. This can have diff --git a/src/borg/helpers.py b/src/borg/helpers.py index fed3b769b9..242b939651 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -139,7 +139,7 @@ def check_extension_modules(): raise ExtensionModuleError if borg.crypto.low_level.API_VERSION != '1.1_02': raise ExtensionModuleError - if platform.API_VERSION != platform.OS_API_VERSION != '1.1_01': + if platform.API_VERSION != platform.OS_API_VERSION != '1.1_02': raise ExtensionModuleError if item.API_VERSION != '1.1_02': raise ExtensionModuleError @@ -1183,7 +1183,11 @@ def make_path_safe(path): def daemonize(): """Detach process from controlling terminal and run in background + + Returns: old and new get_process_id tuples """ + from .platform import get_process_id + old_id = get_process_id() pid = os.fork() if pid: os._exit(0) @@ -1199,6 +1203,8 @@ def daemonize(): os.dup2(fd, 0) os.dup2(fd, 1) os.dup2(fd, 2) + new_id = get_process_id() + return old_id, new_id class StableDict(dict): diff --git a/src/borg/locking.py b/src/borg/locking.py index c3d69674aa..0fc0927504 100644 --- a/src/borg/locking.py +++ b/src/borg/locking.py @@ -202,6 +202,16 @@ def break_lock(self): os.unlink(os.path.join(self.path, name)) os.rmdir(self.path) + def migrate_lock(self, old_id, new_id): + """migrate the lock ownership from old_id to new_id""" + assert self.id == old_id + new_unique_name = os.path.join(self.path, "%s.%d-%x" % new_id) + if self.is_locked() and self.by_me(): + with open(new_unique_name, "wb"): + pass + os.unlink(self.unique_name) + self.id, self.unique_name = new_id, new_unique_name + class LockRoster: """ @@ -271,6 +281,25 @@ def modify(self, key, op): roster[key] = list(list(e) for e in elements) self.save(roster) + def migrate_lock(self, key, old_id, new_id): + """migrate the lock ownership from old_id to new_id""" + assert self.id == old_id + # need to temporarily switch off stale lock killing as we want to + # rather migrate than kill them (at least the one made by old_id). + killing, self.kill_stale_locks = self.kill_stale_locks, False + try: + try: + self.modify(key, REMOVE) + except KeyError: + # entry was not there, so no need to add a new one, but still update our id + self.id = new_id + else: + # old entry removed, update our id and add a updated entry + self.id = new_id + self.modify(key, ADD) + finally: + self.kill_stale_locks = killing + class Lock: """ @@ -373,3 +402,14 @@ def got_exclusive_lock(self): def break_lock(self): self._roster.remove() self._lock.break_lock() + + def migrate_lock(self, old_id, new_id): + assert self.id == old_id + self.id = new_id + if self.is_exclusive: + self._lock.migrate_lock(old_id, new_id) + self._roster.migrate_lock(EXCLUSIVE, old_id, new_id) + else: + with self._lock: + self._lock.migrate_lock(old_id, new_id) + self._roster.migrate_lock(SHARED, old_id, new_id) diff --git a/src/borg/platform/base.py b/src/borg/platform/base.py index ba938e66a1..f56c03be48 100644 --- a/src/borg/platform/base.py +++ b/src/borg/platform/base.py @@ -15,7 +15,7 @@ are correctly composed into the base functionality. """ -API_VERSION = '1.1_01' +API_VERSION = '1.1_02' fdatasync = getattr(os, 'fdatasync', os.fsync) diff --git a/src/borg/platform/darwin.pyx b/src/borg/platform/darwin.pyx index f13b366207..b7e439ab67 100644 --- a/src/borg/platform/darwin.pyx +++ b/src/borg/platform/darwin.pyx @@ -4,7 +4,7 @@ from ..helpers import user2uid, group2gid from ..helpers import safe_decode, safe_encode from .posix import swidth -API_VERSION = '1.1_01' +API_VERSION = '1.1_02' cdef extern from "sys/acl.h": ctypedef struct _acl_t: diff --git a/src/borg/platform/freebsd.pyx b/src/borg/platform/freebsd.pyx index 4b38982125..3344de1658 100644 --- a/src/borg/platform/freebsd.pyx +++ b/src/borg/platform/freebsd.pyx @@ -4,7 +4,7 @@ from ..helpers import posix_acl_use_stored_uid_gid from ..helpers import safe_encode, safe_decode from .posix import swidth -API_VERSION = '1.1_01' +API_VERSION = '1.1_02' cdef extern from "errno.h": int errno diff --git a/src/borg/platform/linux.pyx b/src/borg/platform/linux.pyx index 4352d864a2..7c3cf1b0b5 100644 --- a/src/borg/platform/linux.pyx +++ b/src/borg/platform/linux.pyx @@ -13,7 +13,7 @@ from .posix import swidth from libc cimport errno from libc.stdint cimport int64_t -API_VERSION = '1.1_01' +API_VERSION = '1.1_02' cdef extern from "sys/types.h": int ACL_TYPE_ACCESS diff --git a/src/borg/platform/posix.pyx b/src/borg/platform/posix.pyx index b8689e2c29..0144e1aa3b 100644 --- a/src/borg/platform/posix.pyx +++ b/src/borg/platform/posix.pyx @@ -18,23 +18,21 @@ def swidth(s): return str_len -# only determine the PID and hostname once. -# for FUSE mounts, we fork a child process that needs to release -# the lock made by the parent, so it needs to use the same PID for that. -_pid = os.getpid() +# for performance reasons, only determine the hostname once. # XXX this sometimes requires live internet access for issuing a DNS query in the background. _hostname = '%s@%s' % (socket.getfqdn(), uuid.getnode()) def get_process_id(): """ - Return identification tuple (hostname, pid, thread_id) for 'us'. If this is a FUSE process, then the PID will be - that of the parent, not the forked FUSE child. + Return identification tuple (hostname, pid, thread_id) for 'us'. + This always returns the current pid, which might be different from before, e.g. if daemonize() was used. Note: Currently thread_id is *always* zero. """ thread_id = 0 - return _hostname, _pid, thread_id + pid = os.getpid() + return _hostname, pid, thread_id def process_alive(host, pid, thread): diff --git a/src/borg/repository.py b/src/borg/repository.py index b443bdb4ef..a61e4e9491 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -346,6 +346,11 @@ def get_transaction_id(self): def break_lock(self): Lock(os.path.join(self.path, 'lock')).break_lock() + def migrate_lock(self, old_id, new_id): + # note: only needed for local repos + if self.lock is not None: + self.lock.migrate_lock(old_id, new_id) + def open(self, path, exclusive, lock_wait=None, lock=True): self.path = path if not os.path.isdir(path): diff --git a/src/borg/testsuite/locking.py b/src/borg/testsuite/locking.py index fe8676d4a4..64a7928116 100644 --- a/src/borg/testsuite/locking.py +++ b/src/borg/testsuite/locking.py @@ -3,6 +3,7 @@ import pytest +from ..helpers import daemonize from ..platform import get_process_id, process_alive from ..locking import TimeoutTimer, ExclusiveLock, Lock, LockRoster, \ ADD, REMOVE, SHARED, EXCLUSIVE, LockTimeout, NotLocked, NotMyLock @@ -76,6 +77,19 @@ def test_kill_stale(self, lockpath, free_pid): with pytest.raises(LockTimeout): ExclusiveLock(lockpath, id=our_id, kill_stale_locks=True, timeout=0.1).acquire() + def test_migrate_lock(self, lockpath): + old_id, new_id = ID1, ID2 + assert old_id[1] != new_id[1] # different PIDs (like when doing daemonize()) + lock = ExclusiveLock(lockpath, id=old_id).acquire() + assert lock.id == old_id # lock is for old id / PID + old_unique_name = lock.unique_name + assert lock.by_me() # we have the lock + lock.migrate_lock(old_id, new_id) # fix the lock + assert lock.id == new_id # lock corresponds to the new id / PID + new_unique_name = lock.unique_name + assert lock.by_me() # we still have the lock + assert old_unique_name != new_unique_name # locking filename is different now + class TestLock: def test_shared(self, lockpath): @@ -155,6 +169,22 @@ def test_kill_stale(self, lockpath, free_pid): with pytest.raises(LockTimeout): Lock(lockpath, id=our_id, kill_stale_locks=True, timeout=0.1).acquire() + def test_migrate_lock(self, lockpath): + old_id, new_id = ID1, ID2 + assert old_id[1] != new_id[1] # different PIDs (like when doing daemonize()) + + lock = Lock(lockpath, id=old_id, exclusive=True).acquire() + assert lock.id == old_id + lock.migrate_lock(old_id, new_id) # fix the lock + assert lock.id == new_id + lock.release() + + lock = Lock(lockpath, id=old_id, exclusive=False).acquire() + assert lock.id == old_id + lock.migrate_lock(old_id, new_id) # fix the lock + assert lock.id == new_id + lock.release() + @pytest.fixture() def rosterpath(tmpdir): @@ -207,3 +237,14 @@ def test_kill_stale(self, rosterpath, free_pid): other_killer_roster = LockRoster(rosterpath, kill_stale_locks=True) # Did not kill us, since we're alive assert other_killer_roster.get(SHARED) == {our_id, cant_know_if_dead_id} + + def test_migrate_lock(self, rosterpath): + old_id, new_id = ID1, ID2 + assert old_id[1] != new_id[1] # different PIDs (like when doing daemonize()) + roster = LockRoster(rosterpath, id=old_id) + assert roster.id == old_id + roster.modify(SHARED, ADD) + assert roster.get(SHARED) == {old_id} + roster.migrate_lock(SHARED, old_id, new_id) # fix the lock + assert roster.id == new_id + assert roster.get(SHARED) == {new_id} From 0b3f8ad2611b004c81525025e8c93f845006def8 Mon Sep 17 00:00:00 2001 From: rugk Date: Sun, 16 Jul 2017 20:04:56 +0200 Subject: [PATCH 011/798] Renew asciinema/screencasts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Created with borg v1.1.0, so more up-to-date and split into different parts (install, basic, advanced)… Fixes https://github.com/borgbackup/borg/issues/669 --- docs/misc/asciinema/advanced.json | 7733 +++++++++++++++++++ docs/misc/asciinema/advanced.sh | 65 + docs/misc/asciinema/basic.json | 4862 ++++++++++++ docs/misc/asciinema/basic.sh | 53 + docs/misc/asciinema/install.json | 1354 ++++ docs/misc/asciinema/install.sh | 21 + docs/misc/asciinema/install_and_basics.json | 5550 ------------- docs/misc/asciinema/install_and_basics.txt | 51 - 8 files changed, 14088 insertions(+), 5601 deletions(-) create mode 100644 docs/misc/asciinema/advanced.json create mode 100644 docs/misc/asciinema/advanced.sh create mode 100644 docs/misc/asciinema/basic.json create mode 100644 docs/misc/asciinema/basic.sh create mode 100644 docs/misc/asciinema/install.json create mode 100644 docs/misc/asciinema/install.sh delete mode 100644 docs/misc/asciinema/install_and_basics.json delete mode 100644 docs/misc/asciinema/install_and_basics.txt diff --git a/docs/misc/asciinema/advanced.json b/docs/misc/asciinema/advanced.json new file mode 100644 index 0000000000..d9f951e6b5 --- /dev/null +++ b/docs/misc/asciinema/advanced.json @@ -0,0 +1,7733 @@ +{ + "version": 1, + "width": 78, + "height": 24, + "duration": 446.783754, + "command": null, + "title": null, + "env": { + "TERM": "xterm-256color", + "SHELL": "/bin/zsh" + }, + "stdout": [ + [ + 0.29658, + "\b\u001b[1m$ # \u001b[1mFor the pro users, here are some advanced features of borg, so you can imp\u001b[1mr\u001b[1mess your friends. ;)\u001b[0m\u001b[39m\u001b[K" + ], + [ + 1.025674, + "\u001b[?1l\u001b>" + ], + [ + 0.000375, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000796, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.000953, + "\u001b]7;\u0007" + ], + [ + 0.000799, + "\u001b]7;\u0007" + ], + [ + 7.7e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 4.6e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000368, + "\u001b[?2004h" + ], + [ + 0.857202, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.269836, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.277016, + "\b\b\u001b[1m#\u001b[1m \u001b[1mN\u001b[0m\u001b[39m" + ], + [ + 0.185115, + "\b\u001b[1mN\u001b[1mo\u001b[0m\u001b[39m" + ], + [ + 0.222294, + "\b\u001b[1mo\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.098908, + "\b\u001b[1mt\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.471037, + "\b\u001b[1me\u001b[1m:\u001b[0m\u001b[39m" + ], + [ + 0.276132, + "\b\u001b[1m:\u001b[1m This screencast was made with borg version 1.1.0 – older or newer bo\u001b[1mr\u001b[1mg versions may behave differently.\u001b[0m\u001b[39m\u001b[K" + ], + [ + 1.063392, + "\u001b[?1l\u001b>" + ], + [ + 0.001402, + "\u001b[?2004l\r\r\n" + ], + [ + 0.001228, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.002846, + "\u001b]7;\u0007" + ], + [ + 0.002554, + "\u001b]7;\u0007" + ], + [ + 6.6e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000229, + "\u001b[?1h\u001b=" + ], + [ + 0.000858, + "\u001b[?2004h" + ], + [ + 0.944947, + "\u001b[?1l\u001b>" + ], + [ + 0.000319, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000652, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001131, + "\u001b]7;\u0007" + ], + [ + 0.000871, + "\u001b]7;\u0007" + ], + [ + 9.6e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000117, + "\u001b[?1h\u001b=" + ], + [ + 0.00014, + "\u001b[?2004h" + ], + [ + 0.91046, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.350642, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.226284, + "\b\b\u001b[1m#\u001b[1m \u001b[1mF\u001b[0m\u001b[39m" + ], + [ + 0.190635, + "\b\u001b[1mF\u001b[1mi\u001b[0m\u001b[39m" + ], + [ + 0.226298, + "\b\u001b[1mi\u001b[1mr\u001b[0m\u001b[39m" + ], + [ + 0.094075, + "\b\u001b[1mr\u001b[1ms\u001b[0m\u001b[39m" + ], + [ + 0.125931, + "\b\u001b[1ms\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.210409, + "\b\u001b[1mt\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.333349, + "\b\u001b[1m \u001b[1mof all, we can use several environment variables for borg.\u001b[0m\u001b[39m" + ], + [ + 1.115007, + "\u001b[?1l\u001b>" + ], + [ + 0.000418, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000665, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001185, + "\u001b]7;\u0007" + ], + [ + 0.00091, + "\u001b]7;\u0007" + ], + [ + 2.5e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 8.9e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000298, + "\u001b[?2004h" + ], + [ + 1.193161, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.249128, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.253119, + "\b\b\u001b[1m#\u001b[1m \u001b[1mE\u001b[0m\u001b[39m" + ], + [ + 0.328187, + "\b\u001b[1mE\u001b[1m.\u001b[0m\u001b[39m" + ], + [ + 0.873845, + "\b\u001b[1m.\u001b[1mg\u001b[0m\u001b[39m" + ], + [ + 0.164238, + "\b\u001b[1mg\u001b[1m.\u001b[0m\u001b[39m" + ], + [ + 0.211331, + "\b\u001b[1m.\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.15971, + "\b\u001b[1m \u001b[1mw\u001b[0m\u001b[39m" + ], + [ + 0.133833, + "\b\u001b[1mw\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 2.95423, + "\b\u001b[1me\u001b[1m do not want to type in our repo path and password again and again…" + ], + [ + 1.769654, + "\u001b[K" + ], + [ + 2.7e-05, + "\u001b[?1l\u001b>" + ], + [ + 0.000616, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000594, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.00144, + "\u001b]7;\u0007" + ], + [ + 0.001172, + "\u001b]7;\u0007" + ], + [ + 3.7e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 7.2e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000419, + "\u001b[?2004h" + ], + [ + 0.975676, + "\u001b[1m\u001b[31me\u001b[0m\u001b[39m" + ], + [ + 0.156719, + "\b\u001b[0m\u001b[32me\u001b[32mx\u001b[39m" + ], + [ + 0.121911, + "\b\b\u001b[1m\u001b[31me\u001b[1m\u001b[31mx\u001b[1m\u001b[31mp\u001b[0m\u001b[39m" + ], + [ + 0.15502, + "\b\u001b[1m\u001b[31mp\u001b[1m\u001b[31mo\u001b[0m\u001b[39m" + ], + [ + 0.26241, + "\b\u001b[1m\u001b[31mo\u001b[1m\u001b[31mr\u001b[0m\u001b[39m" + ], + [ + 0.126933, + "\b\b\b\b\b\u001b[0m\u001b[33me\u001b[0m\u001b[33mx\u001b[0m\u001b[33mp\u001b[0m\u001b[33mo\u001b[0m\u001b[33mr\u001b[33mt\u001b[39m" + ], + [ + 0.192182, + " " + ], + [ + 0.304561, + "B" + ], + [ + 0.192073, + "O" + ], + [ + 0.136183, + "R" + ], + [ + 0.114362, + "G" + ], + [ + 0.576349, + "_" + ], + [ + 0.103719, + "R" + ], + [ + 0.113626, + "E" + ], + [ + 0.159395, + "P" + ], + [ + 0.141942, + "O" + ], + [ + 0.554082, + "=" + ], + [ + 0.74644, + "'" + ], + [ + 0.69222, + "/" + ], + [ + 0.20093, + "m" + ], + [ + 0.108068, + "e" + ], + [ + 0.125576, + "d" + ], + [ + 0.161298, + "i" + ], + [ + 0.107949, + "a" + ], + [ + 0.423969, + "/" + ], + [ + 0.623591, + "b" + ], + [ + 0.102775, + "a" + ], + [ + 0.146442, + "c" + ], + [ + 0.116202, + "k" + ], + [ + 0.133034, + "u" + ], + [ + 0.282831, + "p" + ], + [ + 0.436512, + "/" + ], + [ + 0.551147, + "b" + ], + [ + 0.208373, + "o" + ], + [ + 0.108883, + "r" + ], + [ + 0.137272, + "g" + ], + [ + 0.218057, + "d" + ], + [ + 0.122586, + "e" + ], + [ + 0.133605, + "m" + ], + [ + 0.170095, + "o" + ], + [ + 0.795644, + "'" + ], + [ + 0.928899, + "\u001b[?1l\u001b>" + ], + [ + 0.001469, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000802, + "\u001b]2;export BORG_REPO='/media/backup/borgdemo' \u0007\u001b]1;export\u0007" + ], + [ + 0.000109, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001347, + "\u001b]7;\u0007" + ], + [ + 0.001006, + "\u001b]7;\u0007" + ], + [ + 5.1e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 7.2e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000186, + "\u001b[?2004h" + ], + [ + 0.718289, + "\u001b[1m\u001b[31me\u001b[0m\u001b[39m" + ], + [ + 0.19628, + "\b\u001b[0m\u001b[32me\u001b[32mx\u001b[39m" + ], + [ + 0.269637, + "\b\b\u001b[1m\u001b[31me\u001b[1m\u001b[31mx\u001b[1m\u001b[31mp\u001b[0m\u001b[39m" + ], + [ + 0.164388, + "\b\u001b[1m\u001b[31mp\u001b[1m\u001b[31mo\u001b[0m\u001b[39m" + ], + [ + 0.332999, + "\b\u001b[1m\u001b[31mo\u001b[1m\u001b[31mr\u001b[0m\u001b[39m" + ], + [ + 0.121063, + "\b\b\b\b\b\u001b[0m\u001b[33me\u001b[0m\u001b[33mx\u001b[0m\u001b[33mp\u001b[0m\u001b[33mo\u001b[0m\u001b[33mr\u001b[33mt\u001b[39m" + ], + [ + 0.265335, + " " + ], + [ + 0.311313, + "B" + ], + [ + 0.205307, + "O" + ], + [ + 0.159682, + "R" + ], + [ + 0.141683, + "G" + ], + [ + 0.553563, + "_" + ], + [ + 0.225583, + "P" + ], + [ + 0.10739, + "A" + ], + [ + 0.204722, + "S" + ], + [ + 0.145905, + "S" + ], + [ + 0.312666, + "P" + ], + [ + 0.311469, + "H" + ], + [ + 0.209393, + "R" + ], + [ + 0.069618, + "A" + ], + [ + 0.208505, + "S" + ], + [ + 0.202229, + "E" + ], + [ + 0.719142, + "=" + ], + [ + 0.61979, + "'" + ], + [ + 0.414834, + "1" + ], + [ + 0.208777, + "2" + ], + [ + 0.193519, + "3" + ], + [ + 0.171001, + "4" + ], + [ + 0.542373, + "'" + ], + [ + 0.876006, + "\u001b[?1l\u001b>" + ], + [ + 0.002877, + "\u001b[?2004l\r\r\n" + ], + [ + 0.001161, + "\u001b]2;export BORG_PASSPHRASE='1234' \u0007\u001b]1;export\u0007" + ], + [ + 8.5e-05, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.003438, + "\u001b]7;\u0007" + ], + [ + 0.002065, + "\u001b]7;\u0007" + ], + [ + 0.000146, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 8.9e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000508, + "\u001b[?2004h" + ], + [ + 1.238676, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.273221, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.248131, + "\b\b\u001b[1m#\u001b[1m \u001b[1mP\u001b[0m\u001b[39m" + ], + [ + 0.142137, + "\b\u001b[1mP\u001b[1mr\u001b[0m\u001b[39m" + ], + [ + 0.089312, + "\b\u001b[1mr\u001b[1mo\u001b[0m\u001b[39m" + ], + [ + 0.19919, + "\b\u001b[1mo\u001b[1mb\u001b[0m\u001b[39m" + ], + [ + 0.207691, + "\b\u001b[1mb\u001b[1ml\u001b[0m\u001b[39m" + ], + [ + 0.105529, + "\b\u001b[1ml\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.075159, + "\b\u001b[1me\u001b[1mm\u001b[0m\u001b[39m" + ], + [ + 0.625428, + "\b\u001b[1mm\u001b[1m solved, borg will use this automatically… :)\u001b[0m\u001b[39m" + ], + [ + 0.442303, + "\u001b[?1l\u001b>" + ], + [ + 0.0004, + "\u001b[?2004l\r\r\n" + ], + [ + 0.00077, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001065, + "\u001b]7;\u0007" + ], + [ + 0.001105, + "\u001b]7;\u0007" + ], + [ + 2.2e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 7.3e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000266, + "\u001b[?2004h" + ], + [ + 1.570802, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.218966, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.191279, + "\b\b\u001b[1m#\u001b[1m \u001b[1mW\u001b[0m\u001b[39m" + ], + [ + 0.144698, + "\b\u001b[1mW\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.313061, + "\b\u001b[1me\u001b[1m'\u001b[0m\u001b[39m" + ], + [ + 0.245196, + "\b\u001b[1m'\u001b[1mll use this right away…\u001b[0m\u001b[39m" + ], + [ + 0.532339, + "\u001b[?1l\u001b>" + ], + [ + 0.000412, + "\u001b[?2004l\r\r\n" + ], + [ + 0.00062, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001169, + "\u001b]7;\u0007" + ], + [ + 0.00087, + "\u001b]7;\u0007" + ], + [ + 2.3e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000101, + "\u001b[?1h\u001b=" + ], + [ + 0.000279, + "\u001b[?2004h" + ], + [ + 0.63892, + "\u001b[?1l\u001b>" + ], + [ + 0.000369, + "\u001b[?2004l\r\r\n" + ], + [ + 0.00044, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.002911, + "\u001b]7;\u0007" + ], + [ + 0.002442, + "\u001b]7;\u0007" + ], + [ + 0.000162, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 8.8e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.00059, + "\u001b[?2004h" + ], + [ + 0.548725, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.113549, + "\b\u001b[1m#\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.290577, + "\b\b\u001b[1m#\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.262532, + "\b\u001b[1m \u001b[1mA\u001b[0m\u001b[39m" + ], + [ + 0.41846, + "\b\u001b[1mA\u001b[1mDVANCED CREATION ##\u001b[0m\u001b[39m" + ], + [ + 0.535376, + "\u001b[?1l\u001b>" + ], + [ + 0.001234, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000938, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.002912, + "\u001b]7;\u0007" + ], + [ + 0.001987, + "\u001b]7;\u0007" + ], + [ + 7e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000134, + "\u001b[?1h\u001b=" + ], + [ + 0.000671, + "\u001b[?2004h" + ], + [ + 0.759129, + "\u001b[?1l\u001b>" + ], + [ + 0.000397, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000757, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001297, + "\u001b]7;\u0007" + ], + [ + 0.00131, + "\u001b]7;\u0007" + ], + [ + 3.1e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.0001, + "\u001b[?1h\u001b=" + ], + [ + 0.000135, + "\u001b[?2004h" + ], + [ + 0.425509, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.233111, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.185443, + "\b\b\u001b[1m#\u001b[1m \u001b[1mW\u001b[0m\u001b[39m" + ], + [ + 0.151433, + "\b\u001b[1mW\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.157168, + "\b\u001b[1me\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.148414, + "\b\u001b[1m \u001b[1mc\u001b[0m\u001b[39m" + ], + [ + 0.200586, + "\b\u001b[1mc\u001b[1ma\u001b[0m\u001b[39m" + ], + [ + 0.145343, + "\b\u001b[1ma\u001b[1mn\u001b[0m\u001b[39m" + ], + [ + 0.414343, + "\b\u001b[1mn\u001b[1m also use some placeholders in our archive name…\u001b[0m\u001b[39m" + ], + [ + 1.198174, + "\u001b[?1l\u001b>" + ], + [ + 0.000433, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000647, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001494, + "\u001b]7;\u0007" + ], + [ + 0.001069, + "\u001b]7;\u0007" + ], + [ + 8.2e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 4.6e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000395, + "\u001b[?2004h" + ], + [ + 0.832499, + "\u001b[1m\u001b[31mb\u001b[0m\u001b[39m" + ], + [ + 0.186742, + "\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[0m\u001b[39m" + ], + [ + 0.076839, + "\b\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[1m\u001b[31mr\u001b[0m\u001b[39m" + ], + [ + 0.15706, + "\b\b\b\u001b[0m\u001b[32mb\u001b[0m\u001b[32mo\u001b[0m\u001b[32mr\u001b[32mg\u001b[39m" + ], + [ + 0.175773, + " " + ], + [ + 0.265231, + "c" + ], + [ + 0.198791, + "r" + ], + [ + 0.162497, + "e" + ], + [ + 0.08856, + "a" + ], + [ + 0.135865, + "t" + ], + [ + 0.112707, + "e" + ], + [ + 0.634063, + " " + ], + [ + 0.621186, + "-" + ], + [ + 0.112118, + "-" + ], + [ + 0.270276, + "s" + ], + [ + 0.135637, + "t" + ], + [ + 0.130994, + "a" + ], + [ + 0.086801, + "t" + ], + [ + 0.119778, + "s" + ], + [ + 0.24882, + " " + ], + [ + 0.47677, + "-" + ], + [ + 0.112232, + "-" + ], + [ + 0.26855, + "p" + ], + [ + 0.218974, + "r" + ], + [ + 0.14527, + "o" + ], + [ + 0.21975, + "g" + ], + [ + 0.104406, + "r" + ], + [ + 0.168975, + "e" + ], + [ + 0.224875, + "s" + ], + [ + 0.161557, + "s" + ], + [ + 0.556139, + " " + ], + [ + 0.90841, + "-" + ], + [ + 0.117065, + "-" + ], + [ + 0.268496, + "c" + ], + [ + 0.118758, + "o" + ], + [ + 0.13892, + "m" + ], + [ + 0.17322, + "p" + ], + [ + 0.146756, + "r" + ], + [ + 0.196139, + "e" + ], + [ + 0.249655, + "s" + ], + [ + 0.157202, + "s" + ], + [ + 0.236521, + "i" + ], + [ + 0.120624, + "o" + ], + [ + 0.175143, + "n" + ], + [ + 0.321073, + " " + ], + [ + 0.249849, + "l" + ], + [ + 0.281988, + "z" + ], + [ + 0.281179, + "4" + ], + [ + 1.223567, + " " + ], + [ + 0.604439, + ":" + ], + [ + 0.099497, + ":" + ], + [ + 0.760652, + "{" + ], + [ + 0.504646, + "u" + ], + [ + 0.249702, + "s" + ], + [ + 0.310204, + "e" + ], + [ + 0.156776, + "r" + ], + [ + 0.927624, + "}" + ], + [ + 0.972074, + "-" + ], + [ + 0.979824, + "{" + ], + [ + 0.397346, + "n" + ], + [ + 0.195251, + "o" + ], + [ + 0.203266, + "w" + ], + [ + 0.716944, + "}" + ], + [ + 0.992466, + " " + ], + [ + 0.404348, + "\u001b[4mW\u001b[24m" + ], + [ + 0.098053, + "\b\u001b[4mW\u001b[4ma\u001b[24m \b" + ], + [ + 0.440872, + "\b\u001b[4ma\u001b[4ml\u001b[24m" + ], + [ + 0.130433, + "\b\u001b[4ml\u001b[4ml\u001b[24m" + ], + [ + 0.079918, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.009903, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mborg\u001b[39m create --stats --progress --compression lz4 ::{user}-{now} \u001b[4mWallpaper\u001b[24m\u001b[K" + ], + [ + 1.432747, + "\u001b[?1l\u001b>" + ], + [ + 0.006238, + "\u001b[?2004l\r\r\n" + ], + [ + 0.001309, + "\u001b]2;borg create --stats --progress --compression lz4 ::{user}-{now} Wallpaper\u0007\u001b]1;borg\u0007" + ], + [ + 0.703285, + "0 B O 0 B C 0 B D 0 N Wallpaper \r" + ], + [ + 0.059704, + "Initializing cache transaction: Reading config \r" + ], + [ + 0.000259, + "Initializing cache transaction: Reading chunks \r" + ], + [ + 0.000283, + "Initializing cache transaction: Reading files \r" + ], + [ + 0.00035, + " \r" + ], + [ + 0.302813, + "Compacting segments 0% \r" + ], + [ + 0.000422, + "Compacting segments 50% \r" + ], + [ + 2.6e-05, + " \r" + ], + [ + 0.053481, + "Saving files cache \r" + ], + [ + 0.010102, + "Saving chunks cache \r" + ], + [ + 0.000354, + "Saving cache config \r" + ], + [ + 0.08865, + " \r" + ], + [ + 2.6e-05, + " \r" + ], + [ + 0.000371, + "------------------------------------------------------------------------------\r\n" + ], + [ + 3.4e-05, + "Archive name: rugk-2017-07-16T18:51:34\r\n" + ], + [ + 8e-06, + "Archive fingerprint: d054cc411324d4bd848b39d1c9cad909073f9ff1a1a503a676d3e050be140396\r\n" + ], + [ + 0.000101, + "Time (start): Sun, 2017-07-16 18:51:34\r\nTime (end): Sun, 2017-07-16 18:51:35\r\n" + ], + [ + 7.5e-05, + "Duration: 0.18 seconds\r\nNumber of files: 1\r\n" + ], + [ + 8.8e-05, + "Utilization of maximum supported archive size: 0%\r\n" + ], + [ + 7e-05, + "------------------------------------------------------------------------------\r\n Original size Compressed size Deduplicated size\r\n" + ], + [ + 1.6e-05, + "This archive: 3.78 MB 3.80 MB 916 B\r\n" + ], + [ + 5.2e-05, + "All archives: 1.86 GB 1.86 GB 561.88 MB\r\n" + ], + [ + 1.3e-05, + "\r\n" + ], + [ + 2.4e-05, + " Unique chunks Total chunks\r\n" + ], + [ + 2.4e-05, + "Chunk index: 1008 3288\r\n" + ], + [ + 2.4e-05, + "------------------------------------------------------------------------------\r\n" + ], + [ + 0.049018, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.00124, + "\u001b]7;\u0007" + ], + [ + 0.000936, + "\u001b]7;\u0007" + ], + [ + 0.000124, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 8.9e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.00019, + "\u001b[?2004h" + ], + [ + 0.814358, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.326066, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.279288, + "\b\b\u001b[1m#\u001b[1m \u001b[1mN\u001b[0m\u001b[39m" + ], + [ + 0.200695, + "\b\u001b[1mN\u001b[1mo\u001b[0m\u001b[39m" + ], + [ + 0.2241, + "\b\u001b[1mo\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.221056, + "\b\u001b[1mt\u001b[1mi\u001b[0m\u001b[39m" + ], + [ + 0.341582, + "\b\u001b[1mi\u001b[1mce the backup name.\u001b[0m\u001b[39m" + ], + [ + 1.40396, + "\u001b[?1l\u001b>" + ], + [ + 0.000442, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000701, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.00108, + "\u001b]7;\u0007" + ], + [ + 0.000942, + "\u001b]7;\u0007" + ], + [ + 5e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 4.3e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.00028, + "\u001b[?2004h" + ], + [ + 1.540998, + "\u001b[?1l\u001b>" + ], + [ + 0.000288, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000571, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.0013, + "\u001b]7;\u0007" + ], + [ + 0.000852, + "\u001b]7;\u0007" + ], + [ + 0.000106, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 8.6e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000164, + "\u001b[?2004h" + ], + [ + 0.402376, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.27499, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.220032, + "\b\b\u001b[1m#\u001b[1m \u001b[1mA\u001b[0m\u001b[39m" + ], + [ + 0.127907, + "\b\u001b[1mA\u001b[1mn\u001b[0m\u001b[39m" + ], + [ + 0.092357, + "\b\u001b[1mn\u001b[1md\u001b[0m\u001b[39m" + ], + [ + 0.145572, + "\b\u001b[1md\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.222962, + "\b\u001b[1m \u001b[1mw\u001b[0m\u001b[39m" + ], + [ + 0.178534, + "\b\u001b[1mw\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.196668, + "\b\u001b[1me\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.225933, + "\b\u001b[1m \u001b[1mc\u001b[0m\u001b[39m" + ], + [ + 0.175493, + "\b\u001b[1mc\u001b[1ma\u001b[0m\u001b[39m" + ], + [ + 0.119503, + "\b\u001b[1ma\u001b[1mn\u001b[0m\u001b[39m" + ], + [ + 0.425112, + "\b\u001b[1mn\u001b[1m put completely different data, with different backup settings, i\u001b[1mn\u001b[1m our backup. It will be deduplicated, anyway:\u001b[0m\u001b[39m\u001b[K" + ], + [ + 1.421849, + "\u001b[?1l\u001b>" + ], + [ + 0.000749, + "\u001b[?2004l\r\r\n" + ], + [ + 0.00066, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.00197, + "\u001b]7;\u0007" + ], + [ + 0.001476, + "\u001b]7;\u0007" + ], + [ + 5.7e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000124, + "\u001b[?1h\u001b=" + ], + [ + 0.000525, + "\u001b[?2004h" + ], + [ + 1.444268, + "\u001b[1m\u001b[31mb\u001b[0m\u001b[39m" + ], + [ + 0.209812, + "\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[0m\u001b[39m" + ], + [ + 0.118788, + "\b\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[1m\u001b[31mr\u001b[0m\u001b[39m" + ], + [ + 0.145792, + "\b\b\b\u001b[0m\u001b[32mb\u001b[0m\u001b[32mo\u001b[0m\u001b[32mr\u001b[32mg\u001b[39m" + ], + [ + 0.20446, + " " + ], + [ + 0.309592, + "c" + ], + [ + 0.201447, + "r" + ], + [ + 0.151315, + "e" + ], + [ + 0.084953, + "a" + ], + [ + 0.156918, + "t" + ], + [ + 0.091724, + "e" + ], + [ + 0.324287, + " " + ], + [ + 0.861486, + "-" + ], + [ + 0.134231, + "-" + ], + [ + 0.491182, + "s" + ], + [ + 0.195253, + "t" + ], + [ + 0.097572, + "a" + ], + [ + 0.09545, + "t" + ], + [ + 0.111782, + "s" + ], + [ + 0.301387, + " " + ], + [ + 0.524478, + "-" + ], + [ + 0.112538, + "-" + ], + [ + 0.397406, + "p" + ], + [ + 0.175509, + "r" + ], + [ + 0.203203, + "o" + ], + [ + 0.257392, + "g" + ], + [ + 0.1453, + "r" + ], + [ + 0.174285, + "e" + ], + [ + 0.353531, + "s" + ], + [ + 0.176989, + "s" + ], + [ + 0.386157, + " " + ], + [ + 0.510691, + "-" + ], + [ + 0.115919, + "-" + ], + [ + 0.225102, + "c" + ], + [ + 0.145577, + "o" + ], + [ + 0.133821, + "m" + ], + [ + 0.171364, + "p" + ], + [ + 0.157255, + "r" + ], + [ + 0.162989, + "e" + ], + [ + 0.256274, + "s" + ], + [ + 0.167254, + "s" + ], + [ + 0.253369, + "i" + ], + [ + 0.1197, + "o" + ], + [ + 0.178105, + "n" + ], + [ + 0.824434, + " " + ], + [ + 0.734608, + "z" + ], + [ + 0.237239, + "l" + ], + [ + 0.158877, + "i" + ], + [ + 0.148988, + "b" + ], + [ + 0.289236, + "," + ], + [ + 0.349273, + "6" + ], + [ + 0.618231, + " " + ], + [ + 0.449031, + "-" + ], + [ + 0.119307, + "-" + ], + [ + 0.451923, + "e" + ], + [ + 0.330743, + "x" + ], + [ + 0.232655, + "c" + ], + [ + 0.197384, + "l" + ], + [ + 0.176276, + "u" + ], + [ + 0.104427, + "d" + ], + [ + 0.141163, + "e" + ], + [ + 0.359309, + " " + ], + [ + 1.198529, + "\u001b[4m~\u001b[24m" + ], + [ + 0.338729, + "\b\u001b[4m~\u001b[4m/\u001b[24m" + ], + [ + 0.352573, + "\b\u001b[4m/\u001b[4mD\u001b[24m" + ], + [ + 0.190254, + "\b\u001b[4mD\u001b[4mo\u001b[24m" + ], + [ + 0.113631, + "\b\u001b[4mo\u001b[4mw\u001b[24m" + ], + [ + 0.743216, + "\b\u001b[4mw\u001b[4mn\u001b[24m" + ], + [ + 0.613852, + "\b\u001b[4mn\u001b[4ml\u001b[24m" + ], + [ + 0.121501, + "\b\u001b[4ml\u001b[4mo\u001b[24m" + ], + [ + 0.068625, + "\b\u001b[4mo\u001b[4ma\u001b[24m" + ], + [ + 0.183855, + "\b\u001b[4ma\u001b[4md\u001b[24m" + ], + [ + 0.152099, + "\b\u001b[4md\u001b[4ms\u001b[24m" + ], + [ + 0.793349, + "\b\u001b[4ms\u001b[4m/\u001b[24m" + ], + [ + 0.477575, + "\b\u001b[4m/\u001b[4mb\u001b[24m" + ], + [ + 0.198072, + "\b\u001b[4mb\u001b[4mi\u001b[24m \r\u001b[K" + ], + [ + 0.175276, + "\u001b[A\u001b[77C\u001b[4mi\u001b[4mg\u001b[24m" + ], + [ + 0.647369, + "\r\u001b[4mg\u001b[24m " + ], + [ + 0.439418, + ":" + ], + [ + 0.108932, + ":" + ], + [ + 0.556615, + "{" + ], + [ + 0.244626, + "u" + ], + [ + 0.097534, + "s" + ], + [ + 0.187502, + "e" + ], + [ + 0.16023, + "r" + ], + [ + 0.675542, + "}" + ], + [ + 0.988946, + "-" + ], + [ + 0.545789, + "{" + ], + [ + 0.33121, + "n" + ], + [ + 0.204667, + "o" + ], + [ + 0.141818, + "w" + ], + [ + 0.397217, + "}" + ], + [ + 0.979478, + " " + ], + [ + 0.768118, + "\u001b[4m~\u001b[24m" + ], + [ + 0.589532, + "\b\u001b[4m~\u001b[4m/\u001b[24m" + ], + [ + 0.515186, + "\b\u001b[4m/\u001b[4mD\u001b[24m" + ], + [ + 0.17703, + "\b\u001b[4mD\u001b[4mo\u001b[24m" + ], + [ + 0.121294, + "\b\u001b[4mo\u001b[4mw\u001b[24m" + ], + [ + 0.153543, + "\b\u001b[4mw\u001b[4mn\u001b[24m" + ], + [ + 0.282343, + "\b\u001b[4mn\u001b[4ml\u001b[24m" + ], + [ + 0.129573, + "\b\u001b[4ml\u001b[4mo\u001b[24m" + ], + [ + 0.095125, + "\b\u001b[4mo\u001b[4ma\u001b[24m" + ], + [ + 0.19963, + "\b\u001b[4ma\u001b[4md\u001b[24m" + ], + [ + 0.142667, + "\b\u001b[4md\u001b[4ms\u001b[24m" + ], + [ + 1.499285, + "\u001b[?1l\u001b>" + ], + [ + 0.003081, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000637, + "\u001b]2;borg create --stats --progress --compression zlib,6 --exclude ~/Downloads/big\u0007\u001b]1;borg\u0007" + ], + [ + 0.687457, + "0 B O 0 B C 0 B D 0 N home/rugk/Downloads \r" + ], + [ + 0.025551, + "Initializing cache transaction: Reading config \r" + ], + [ + 0.000326, + "Initializing cache transaction: Reading chunks \r" + ], + [ + 0.000273, + "Initializing cache transaction: Reading files \r" + ], + [ + 0.000394, + " \r" + ], + [ + 0.220691, + "1.31 MB O 1.29 MB C 1.29 MB D 1 N home/rugk/Downloads...chiveWithStuffHere.zip\r" + ], + [ + 0.26224, + "7.70 MB O 6.91 MB C 6.91 MB D 2 N home/rugk/Downloads...droid.gms-11059462.apk\r" + ], + [ + 0.32599, + "Compacting segments 0% \r" + ], + [ + 0.026073, + "Compacting segments 50% \r" + ], + [ + 0.001982, + " \r" + ], + [ + 0.058565, + "Saving files cache \r" + ], + [ + 0.011363, + "Saving chunks cache \r" + ], + [ + 0.000378, + "Saving cache config \r" + ], + [ + 0.12955, + " \r" + ], + [ + 3.4e-05, + " \r" + ], + [ + 0.00039, + "------------------------------------------------------------------------------\r\n" + ], + [ + 1.6e-05, + "Archive name: rugk-2017-07-16T18:52:19\r\n" + ], + [ + 3.1e-05, + "Archive fingerprint: 0de98f590b004ad7545f2013c4c9f2d4e3eed1415d177c89d6c2b7ff05918d2e\r\n" + ], + [ + 2.2e-05, + "Time (start): Sun, 2017-07-16 18:52:19\r\n" + ], + [ + 7.2e-05, + "Time (end): Sun, 2017-07-16 18:52:20\r\nDuration: 0.63 seconds\r\n" + ], + [ + 3e-05, + "Number of files: 6\r\n" + ], + [ + 2.5e-05, + "Utilization of maximum supported archive size: 0%\r\n" + ], + [ + 2.4e-05, + "------------------------------------------------------------------------------\r\n" + ], + [ + 1.8e-05, + " Original size Compressed size Deduplicated size\r\n" + ], + [ + 2.5e-05, + "This archive: 9.55 MB 8.04 MB 8.04 MB\r\n" + ], + [ + 2.4e-05, + "All archives: 1.87 GB 1.86 GB 569.92 MB\r\n" + ], + [ + 2.5e-05, + "\r\n" + ], + [ + 2.4e-05, + " Unique chunks Total chunks\r\n" + ], + [ + 2.4e-05, + "Chunk index: 1023 3303\r\n" + ], + [ + 2.4e-05, + "------------------------------------------------------------------------------\r\n" + ], + [ + 0.063104, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001326, + "\u001b]7;\u0007" + ], + [ + 0.001145, + "\u001b]7;\u0007" + ], + [ + 8.9e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 8.9e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.0002, + "\u001b[?2004h" + ], + [ + 3.131399, + "\u001b[?1l\u001b>" + ], + [ + 0.000281, + "\u001b[?2004l\r\r\n" + ], + [ + 0.00048, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001354, + "\u001b]7;\u0007" + ], + [ + 0.000923, + "\u001b]7;\u0007" + ], + [ + 6.6e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 5.4e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000161, + "\u001b[?2004h" + ], + [ + 0.285262, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.419379, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.277555, + "\b\b\u001b[1m#\u001b[1m \u001b[1mO\u001b[0m\u001b[39m" + ], + [ + 0.015676, + "\b\u001b[1mO\u001b[0m\u001b[39m" + ], + [ + 0.119839, + "\u001b[1mr\u001b[0m\u001b[39m" + ], + [ + 0.315418, + "\b\u001b[1mr\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.224426, + "\b\u001b[1m \u001b[1ml\u001b[0m\u001b[39m" + ], + [ + 0.10624, + "\b\u001b[1ml\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.170324, + "\b\u001b[1me\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.995665, + "\b\u001b[1mt\u001b[1m'\u001b[0m\u001b[39m" + ], + [ + 0.139331, + "\b\u001b[1m'\u001b[1ms\u001b[0m\u001b[39m" + ], + [ + 0.174188, + "\b\u001b[1ms\u001b[1m backup a device via STDIN.\u001b[0m\u001b[39m" + ], + [ + 1.117059, + "\u001b[?1l\u001b>" + ], + [ + 0.000376, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000566, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001272, + "\u001b]7;\u0007" + ], + [ + 0.000893, + "\u001b]7;\u0007" + ], + [ + 8.1e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 3.9e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000291, + "\u001b[?2004h" + ], + [ + 2.390246, + "\u001b[1m\u001b[31ms\u001b[0m\u001b[39m" + ], + [ + 0.179283, + "\b\u001b[0m\u001b[32ms\u001b[32mu\u001b[39m" + ], + [ + 0.08919, + "\b\b\u001b[1m\u001b[31ms\u001b[1m\u001b[31mu\u001b[1m\u001b[31md\u001b[0m\u001b[39m" + ], + [ + 0.156134, + "\b\b\b\u001b[0m\u001b[4m\u001b[32ms\u001b[0m\u001b[4m\u001b[32mu\u001b[0m\u001b[4m\u001b[32md\u001b[4m\u001b[32mo\u001b[24m\u001b[39m" + ], + [ + 0.939511, + " " + ], + [ + 0.219491, + "\u001b[32md\u001b[39m" + ], + [ + 0.128817, + "\b\u001b[32md\u001b[32md\u001b[39m" + ], + [ + 0.317081, + " " + ], + [ + 0.206442, + "i" + ], + [ + 0.127682, + "f" + ], + [ + 0.497718, + "=" + ], + [ + 0.79125, + "/" + ], + [ + 0.162326, + "d" + ], + [ + 0.141147, + "e" + ], + [ + 0.17081, + "v" + ], + [ + 0.229501, + "/" + ], + [ + 0.309668, + "s" + ], + [ + 0.201626, + "d" + ], + [ + 0.121565, + "x" + ], + [ + 1.112764, + " " + ], + [ + 0.458342, + "b" + ], + [ + 0.13412, + "s" + ], + [ + 0.426796, + "=" + ], + [ + 0.325514, + "1" + ], + [ + 0.182735, + "0" + ], + [ + 0.635284, + "M" + ], + [ + 0.571527, + " " + ], + [ + 0.644682, + "|" + ], + [ + 0.668689, + " " + ], + [ + 0.368219, + "\u001b[1m\u001b[31mb\u001b[0m\u001b[39m" + ], + [ + 0.197192, + "\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[0m\u001b[39m" + ], + [ + 0.069454, + "\b\u001b[1m\u001b[31mo\u001b[1m\u001b[31mr\u001b[0m\u001b[39m" + ], + [ + 0.15983, + "\b\b\b\u001b[0m\u001b[32mb\u001b[0m\u001b[32mo\u001b[0m\u001b[32mr\u001b[32mg\u001b[39m" + ], + [ + 0.193693, + " " + ], + [ + 0.342177, + "c" + ], + [ + 0.213502, + "r" + ], + [ + 0.165989, + "e" + ], + [ + 0.101269, + "a" + ], + [ + 0.20561, + "t" + ], + [ + 0.172574, + "e" + ], + [ + 0.302751, + " " + ], + [ + 0.524261, + "-" + ], + [ + 0.112867, + "-" + ], + [ + 0.358854, + "p" + ], + [ + 0.158933, + "r" + ], + [ + 0.146881, + "o" + ], + [ + 0.235592, + "g" + ], + [ + 0.153909, + "r" + ], + [ + 0.187519, + "e" + ], + [ + 0.278997, + "s" + ], + [ + 0.161351, + "s" + ], + [ + 0.536239, + " " + ], + [ + 0.472536, + "-" + ], + [ + 0.103445, + "-" + ], + [ + 0.315142, + "s" + ], + [ + 0.188015, + "t" + ], + [ + 0.092463, + "a" + ], + [ + 0.121697, + "t" + ], + [ + 0.108331, + "s" + ], + [ + 0.863705, + " " + ], + [ + 0.547363, + ":" + ], + [ + 0.101957, + ":" + ], + [ + 0.713103, + "s" + ], + [ + 0.172527, + "p" + ], + [ + 0.143374, + "e" + ], + [ + 0.495475, + "c" + ], + [ + 0.184747, + "i" + ], + [ + 0.118626, + "a" + ], + [ + 0.21782, + "l" + ], + [ + 0.61779, + "b" + ], + [ + 0.056813, + "a" + ], + [ + 0.18761, + "c" + ], + [ + 0.116227, + "k" + ], + [ + 0.143399, + "u \r\u001b[K" + ], + [ + 0.31621, + "p" + ], + [ + 0.174943, + "\rp " + ], + [ + 0.964699, + "-" + ], + [ + 1.23368, + "\u001b[?1l\u001b>" + ], + [ + 0.003628, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000824, + "\u001b]2;sudo dd if=/dev/sdx bs=10M | borg create --progress --stats ::specialbackup\u0007\u001b]1;dd\u0007" + ], + [ + 0.023411, + "[sudo] password for rugk: " + ], + [ + 3.286582, + "\r\n" + ], + [ + 0.077852, + "Initializing cache transaction: Reading config \r" + ], + [ + 0.000267, + "Initializing cache transaction: Reading chunks \r" + ], + [ + 0.000293, + "Initializing cache transaction: Reading files \r" + ], + [ + 0.00045, + " \r" + ], + [ + 0.083816, + "8.39 MB O 34.25 kB C 34.25 kB D 0 N stdin \r" + ], + [ + 0.228267, + "41.94 MB O 166.40 kB C 100.50 kB D 0 N stdin \r" + ], + [ + 0.216716, + "75.50 MB O 298.20 kB C 100.50 kB D 0 N stdin \r" + ], + [ + 0.218476, + "109.05 MB O 430.00 kB C 100.50 kB D 0 N stdin \r" + ], + [ + 0.219164, + "142.61 MB O 562.12 kB C 133.77 kB D 0 N stdin \r" + ], + [ + 0.216368, + "176.16 MB O 693.92 kB C 133.77 kB D 0 N stdin \r" + ], + [ + 0.222311, + "209.72 MB O 825.72 kB C 133.77 kB D 0 N stdin \r" + ], + [ + 0.217156, + "243.27 MB O 957.52 kB C 133.77 kB D 0 N stdin \r" + ], + [ + 0.22399, + "276.82 MB O 1.09 MB C 166.77 kB D 0 N stdin \r" + ], + [ + 0.223827, + "310.38 MB O 1.22 MB C 166.77 kB D 0 N stdin \r" + ], + [ + 0.220959, + "343.93 MB O 1.35 MB C 166.77 kB D 0 N stdin \r" + ], + [ + 0.223439, + "377.49 MB O 1.48 MB C 166.77 kB D 0 N stdin \r" + ], + [ + 0.226226, + "411.04 MB O 1.62 MB C 200.04 kB D 0 N stdin \r" + ], + [ + 0.239743, + "444.60 MB O 1.75 MB C 200.04 kB D 0 N stdin \r" + ], + [ + 0.229508, + "478.15 MB O 1.88 MB C 200.04 kB D 0 N stdin \r" + ], + [ + 0.220491, + "511.71 MB O 2.01 MB C 200.04 kB D 0 N stdin \r" + ], + [ + 0.2504, + "545.26 MB O 2.14 MB C 200.04 kB D 0 N stdin \r" + ], + [ + 0.241044, + "578.81 MB O 2.28 MB C 200.04 kB D 0 N stdin \r" + ], + [ + 0.215372, + "612.37 MB O 2.41 MB C 200.04 kB D 0 N stdin \r" + ], + [ + 0.113508, + "60+0 records in\r\n60+0 records out\r\n" + ], + [ + 3.9e-05, + "629145600 bytes (629 MB, 600 MiB) copied, 4.31277 s, 146 MB/s\r\n" + ], + [ + 0.231874, + "Compacting segments 0% \r" + ], + [ + 0.001188, + "Compacting segments 50% \r" + ], + [ + 3.7e-05, + " \r" + ], + [ + 0.078344, + "Saving chunks cache \r" + ], + [ + 0.000348, + "Saving cache config \r" + ], + [ + 0.087821, + " \r" + ], + [ + 2.8e-05, + " \r" + ], + [ + 0.000346, + "------------------------------------------------------------------------------\r\n" + ], + [ + 2.2e-05, + "Archive name: specialbackup\r\n" + ], + [ + 9.7e-05, + "Archive fingerprint: 68e942cc4a48402e48ba87f4887c24e5b9fe06e881b0ca241c791810a108bec0\r\nTime (start): Sun, 2017-07-16 18:52:58\r\n" + ], + [ + 0.000133, + "Time (end): Sun, 2017-07-16 18:53:05\r\nDuration: 6.99 seconds\r\n" + ], + [ + 1.3e-05, + "Number of files: 1\r\n" + ], + [ + 2.2e-05, + "Utilization of maximum supported archive size: 0%\r\n" + ], + [ + 7.3e-05, + "------------------------------------------------------------------------------\r\n" + ], + [ + 1.1e-05, + " Original size Compressed size Deduplicated size\r\n" + ], + [ + 2.7e-05, + "This archive: 629.15 MB 2.47 MB 234.02 kB\r\n" + ], + [ + 3.3e-05, + "All archives: 2.50 GB 1.87 GB 570.15 MB\r\n" + ], + [ + 3.3e-05, + "\r\n" + ], + [ + 3.3e-05, + " Unique chunks Total chunks\r\n" + ], + [ + 2.4e-05, + "Chunk index: 1032 3380\r\n" + ], + [ + 2.4e-05, + "------------------------------------------------------------------------------\r\n" + ], + [ + 0.047256, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001383, + "\u001b]7;\u0007" + ], + [ + 0.001024, + "\u001b]7;\u0007" + ], + [ + 8.3e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ " + ], + [ + 7e-06, + "\u001b[K" + ], + [ + 7.1e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.00021, + "\u001b[?2004h" + ], + [ + 3.669021, + "\u001b[?1l\u001b>" + ], + [ + 0.000291, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000719, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001178, + "\u001b]7;\u0007" + ], + [ + 0.0009, + "\u001b]7;\u0007" + ], + [ + 9.6e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 6.6e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.00022, + "\u001b[?2004h" + ], + [ + 0.311851, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.290767, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.23476, + "\b\b\u001b[1m#\u001b[1m \u001b[1mL\u001b[0m\u001b[39m" + ], + [ + 0.188456, + "\b\u001b[1mL\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.139916, + "\b\u001b[1me\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.522516, + "\b\u001b[1mt\u001b[1m'\u001b[0m\u001b[39m" + ], + [ + 0.157443, + "\b\u001b[1m'\u001b[1ms\u001b[0m\u001b[39m" + ], + [ + 0.460729, + "\b\u001b[1ms\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.1201, + "\b\u001b[1m \u001b[1mc\u001b[0m\u001b[39m" + ], + [ + 0.324466, + "\b\u001b[1mc\u001b[1montinue with some simple things:\u001b[0m\u001b[39m" + ], + [ + 0.634167, + "\u001b[?1l\u001b>" + ], + [ + 0.000434, + "\u001b[?2004l\r\r\n" + ], + [ + 0.0006, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.00124, + "\u001b]7;\u0007" + ], + [ + 0.001113, + "\u001b]7;\u0007" + ], + [ + 0.00012, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000136, + "\u001b[?1h\u001b=" + ], + [ + 0.000274, + "\u001b[?2004h" + ], + [ + 1.724466, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.116327, + "\b\u001b[1m#\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.26172, + "\b\b\u001b[1m#\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.250198, + "\b\u001b[1m \u001b[1mU\u001b[0m\u001b[39m" + ], + [ + 0.746624, + "\b\u001b[1mU\u001b[1mSEFUL COMMANDS ##\u001b[0m\u001b[39m" + ], + [ + 0.5602, + "\u001b[?1l\u001b>" + ], + [ + 0.001411, + "\u001b[?2004l\r\r\n" + ], + [ + 0.001009, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.003137, + "\u001b]7;\u0007" + ], + [ + 0.002454, + "\u001b]7;\u0007" + ], + [ + 0.000167, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000157, + "\u001b[?1h\u001b=" + ], + [ + 0.000746, + "\u001b[?2004h" + ], + [ + 1.207899, + "\u001b[?1l\u001b>" + ], + [ + 0.000322, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000472, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001289, + "\u001b]7;\u0007" + ], + [ + 0.000891, + "\u001b]7;\u0007" + ], + + [ + 9.5e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ " + ], + [ + 1.8e-05, + "\u001b[K" + ], + [ + 0.000115, + "\u001b[?1h\u001b=" + ], + [ + 0.000246, + "\u001b[?2004h" + ], + [ + 0.734707, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.247085, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.182467, + "\b\b\u001b[1m#\u001b[1m \u001b[1mY\u001b[0m\u001b[39m" + ], + [ + 0.123582, + "\b\u001b[1mY\u001b[1mo\u001b[0m\u001b[39m" + ], + [ + 0.16343, + "\b\u001b[1mo\u001b[1mu\u001b[0m\u001b[39m" + ], + [ + 0.183388, + "\b\u001b[1mu\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.083055, + "\b\u001b[1m \u001b[1mc\u001b[0m\u001b[39m" + ], + [ + 0.187526, + "\b\u001b[1mc\u001b[1ma\u001b[0m\u001b[39m" + ], + [ + 0.130988, + "\b\u001b[1ma\u001b[1mn\u001b[0m\u001b[39m" + ], + [ + 0.142246, + "\b\u001b[1mn\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.145489, + "\b\u001b[1m \u001b[1ms\u001b[0m\u001b[39m" + ], + [ + 0.132155, + "\b\u001b[1ms\u001b[1mh\u001b[0m\u001b[39m" + ], + [ + 0.192915, + "\b\u001b[1mh\u001b[1mo\u001b[0m\u001b[39m" + ], + [ + 0.142644, + "\b\u001b[1mo\u001b[1mw\u001b[0m\u001b[39m" + ], + [ + 0.149707, + "\b\u001b[1mw\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.134515, + "\b\u001b[1m \u001b[1ms\u001b[0m\u001b[39m" + ], + [ + 0.085942, + "\b\u001b[1ms\u001b[1mo\u001b[0m\u001b[39m" + ], + [ + 0.160772, + "\b\u001b[1mo\u001b[1mm\u001b[0m\u001b[39m" + ], + [ + 0.132016, + "\b\u001b[1mm\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.219601, + "\b\u001b[1me\u001b[1m information about an archive. You can even do it without \u001b[1mn\u001b[1meeding to specify the archive name:\u001b[0m\u001b[39m\u001b[K" + ], + [ + 0.644657, + "\u001b[?1l\u001b>" + ], + [ + 0.000392, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000705, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001347, + "\u001b]7;\u0007" + ], + [ + 0.001099, + "\u001b]7;\u0007" + ], + [ + 4.2e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.0001, + "\u001b[?1h\u001b=" + ], + [ + 0.000372, + "\u001b[?2004h" + ], + [ + 2.264862, + "\u001b[1m\u001b[31mb\u001b[0m\u001b[39m" + ], + [ + 0.182056, + "\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[0m\u001b[39m" + ], + [ + 0.083939, + "\b\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[1m\u001b[31mr\u001b[0m\u001b[39m" + ], + [ + 0.152072, + "\b\b\b\u001b[0m\u001b[32mb\u001b[0m\u001b[32mo\u001b[0m\u001b[32mr\u001b[32mg\u001b[39m" + ], + [ + 0.142791, + " " + ], + [ + 0.224315, + "i" + ], + [ + 0.130651, + "n" + ], + [ + 0.100647, + "f" + ], + [ + 0.155636, + "o" + ], + [ + 0.716063, + " " + ], + [ + 0.736635, + ":" + ], + [ + 0.107352, + ":" + ], + [ + 0.289804, + " " + ], + [ + 0.436564, + "-" + ], + [ + 0.131871, + "-" + ], + [ + 0.824072, + "l" + ], + [ + 0.061945, + "a" + ], + [ + 0.136723, + "s" + ], + [ + 0.143197, + "t" + ], + [ + 0.186833, + " " + ], + [ + 0.125784, + "1" + ], + [ + 0.924568, + "\u001b[?1l\u001b>" + ], + [ + 0.002555, + "\u001b[?2004l\r\r\n" + ], + [ + 0.00096, + "\u001b]2;borg info :: --last 1\u0007\u001b]1;borg\u0007" + ], + [ + 0.693043, + "Archive name: specialbackup\r\nArchive fingerprint: 68e942cc4a48402e48ba87f4887c24e5b9fe06e881b0ca241c791810a108bec0\r\nComment: \r\nHostname: tux\r\nUsername: rugk\r\nTime (start): Sun, 2017-07-16 18:52:58\r\nTime (end): Sun, 2017-07-16 18:53:05\r\nDuration: 6.99 seconds\r\nNumber of files: 1\r\nCommand line: borg create --progress --stats ::specialbackup -\r\nUtilization of maximum supported archive size: 0%\r\n------------------------------------------------------------------------------\r\n Original size Compressed size Deduplicated size\r\nThis archive: 629.15 MB 2.47 MB 234.02 kB\r\nAll archives: 2.50 GB 1.87 GB 570.15 MB\r\n\r\n Unique chunks Total chunks\r\nChunk index: 1032 3380\r\n" + ], + [ + 0.045207, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001204, + "\u001b]7;\u0007" + ], + [ + 0.000923, + "\u001b]7;\u0007" + ], + [ + 3.5e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000129, + "\u001b[?1h\u001b=" + ], + [ + 0.000196, + "\u001b[?2004h" + ], + [ + 1.70302, + "\u001b[?1l\u001b>" + ], + [ + 0.000314, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000475, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001262, + "\u001b]7;\u0007" + ], + [ + 0.00098, + "\u001b]7;\u0007" + ], + [ + 4.4e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 6.1e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000164, + "\u001b[?2004h" + ], + [ + 0.281651, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.234109, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.181326, + "\b\b\u001b[1m#\u001b[1m \u001b[1mS\u001b[0m\u001b[39m" + ], + [ + 0.12398, + "\b\u001b[1mS\u001b[1mo\u001b[0m\u001b[39m" + ], + [ + 0.166912, + "\b\u001b[1mo\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.490114, + "\b\u001b[1m \u001b[1ml\u001b[0m\u001b[39m" + ], + [ + 0.160581, + "\b\u001b[1ml\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.148283, + "\b\u001b[1me\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.453708, + "\b\u001b[1mt\u001b[1m'\u001b[0m\u001b[39m" + ], + [ + 0.118956, + "\b\u001b[1m'\u001b[1ms\u001b[0m\u001b[39m" + ], + [ + 0.125062, + "\b\u001b[1ms\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.130519, + "\b\u001b[1m \u001b[1mr\u001b[0m\u001b[39m" + ], + [ + 0.130132, + "\b\u001b[1mr\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.265033, + "\b\u001b[1me\u001b[1mname our last archive:\u001b[0m\u001b[39m" + ], + [ + 1.001935, + "\u001b[?1l\u001b>" + ], + [ + 0.000416, + "\u001b[?2004l\r\r\n" + ], + [ + 0.0006, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.00114, + "\u001b]7;\u0007" + ], + [ + 0.000898, + "\u001b]7;\u0007" + ], + [ + 2.7e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 9.5e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000286, + "\u001b[?2004h" + ], + [ + 1.253113, + "\u001b[1m\u001b[31mb\u001b[0m\u001b[39m" + ], + [ + 0.202007, + "\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[0m\u001b[39m" + ], + [ + 0.105752, + "\b\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[1m\u001b[31mr\u001b[0m\u001b[39m" + ], + [ + 0.134948, + "\b\b\b\u001b[0m\u001b[32mb\u001b[0m\u001b[32mo\u001b[0m\u001b[32mr\u001b[32mg\u001b[39m" + ], + [ + 0.14764, + " " + ], + [ + 0.157682, + "r" + ], + [ + 0.124491, + "e" + ], + [ + 0.118993, + "n" + ], + [ + 0.140445, + "a" + ], + [ + 0.101365, + "m" + ], + [ + 0.115953, + "e" + ], + [ + 1.107064, + " " + ], + [ + 0.561405, + ":" + ], + [ + 0.103305, + ":" + ], + [ + 0.263633, + "s" + ], + [ + 0.142089, + "p" + ], + [ + 0.134253, + "e" + ], + [ + 0.240688, + "c" + ], + [ + 0.136782, + "i" + ], + [ + 0.128372, + "a" + ], + [ + 0.170065, + "l" + ], + [ + 0.592209, + "b" + ], + [ + 0.348417, + "a" + ], + [ + 0.210896, + "c" + ], + [ + 0.259528, + "k" + ], + [ + 0.171523, + "u" + ], + [ + 0.245786, + "p" + ], + [ + 0.582735, + " " + ], + [ + 0.568884, + "b" + ], + [ + 0.101982, + "a" + ], + [ + 0.162673, + "c" + ], + [ + 0.104218, + "k" + ], + [ + 0.132828, + "u" + ], + [ + 0.245157, + "p" + ], + [ + 0.266242, + "-" + ], + [ + 0.316388, + "b" + ], + [ + 0.43535, + "l" + ], + [ + 0.133908, + "o" + ], + [ + 0.047013, + "c" + ], + [ + 0.622041, + "k" + ], + [ + 0.82215, + "-" + ], + [ + 0.183882, + "d" + ], + [ + 0.189034, + "e" + ], + [ + 0.181902, + "v" + ], + [ + 0.18728, + "i" + ], + [ + 0.052242, + "c" + ], + [ + 0.160462, + "e" + ], + [ + 0.645053, + "\u001b[?1l\u001b>" + ], + [ + 0.001146, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000741, + "\u001b]2;borg rename ::specialbackup backup-block-device\u0007\u001b]1;borg\u0007" + ], + [ + 1.136038, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001149, + "\u001b]7;\u0007" + ], + [ + 0.000968, + "\u001b]7;\u0007" + ], + [ + 7.4e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000107, + "\u001b[?1h\u001b=" + ], + [ + 0.000193, + "\u001b[?2004h" + ], + [ + 1.203902, + "\u001b[32mborg\u001b[39m rename ::specialbackup backup-block-device" + ], + [ + 0.192203, + "\u001b[47D\u001b[1m#\u001b[1m \u001b[1mS\u001b[1mo\u001b[1m \u001b[1ml\u001b[1me\u001b[1mt\u001b[1m'\u001b[1ms\u001b[1m \u001b[1mr\u001b[1me\u001b[1mn\u001b[1ma\u001b[1mm\u001b[1me\u001b[1m \u001b[1mo\u001b[1mu\u001b[1mr\u001b[1m \u001b[1ml\u001b[1ma\u001b[1ms\u001b[1mt\u001b[1m \u001b[1ma\u001b[1mr\u001b[1mc\u001b[1mh\u001b[1mi\u001b[1mv\u001b[1me\u001b[1m:\u001b[0m\u001b[39m \u001b[12D" + ], + [ + 0.528657, + "\u001b[35D\u001b[0m\u001b[32mb\u001b[0m\u001b[32mo\u001b[0m\u001b[32mr\u001b[0m\u001b[32mg\u001b[39m\u001b[0m\u001b[39m \u001b[0m\u001b[39mi\u001b[0m\u001b[39mn\u001b[0m\u001b[39mf\u001b[0m\u001b[39mo\u001b[0m\u001b[39m \u001b[0m\u001b[39m:\u001b[0m\u001b[39m:\u001b[0m\u001b[39m \u001b[0m\u001b[39m-\u001b[0m\u001b[39m-\u001b[0m\u001b[39ml\u001b[0m\u001b[39ma\u001b[0m\u001b[39ms\u001b[0m\u001b[39mt\u001b[0m\u001b[39m \u001b[0m\u001b[39m1\u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[14D" + ], + [ + 0.548884, + "\u001b[?1l\u001b>" + ], + [ + 0.003595, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000857, + "\u001b]2;borg info :: --last 1\u0007\u001b]1;borg\u0007" + ], + [ + 0.689879, + "Archive name: backup-block-device\r\nArchive fingerprint: 5fd9732b4809252742a7cb3fadf2a971dd6371afd11a07944c0b5803d57c240f\r\nComment: \r\nHostname: tux\r\nUsername: rugk\r\nTime (start): Sun, 2017-07-16 18:52:58\r\nTime (end): Sun, 2017-07-16 18:53:05\r\nDuration: 6.99 seconds\r\nNumber of files: 1\r\nCommand line: borg create --progress --stats ::specialbackup -\r\nUtilization of maximum supported archive size: 0%\r\n------------------------------------------------------------------------------\r\n Original size Compressed size Deduplicated size\r\nThis archive: 629.15 MB 2.47 MB 234.04 kB\r\nAll archives: 2.50 GB 1.87 GB 570.15 MB\r\n\r\n Unique chunks Total chunks\r\nChunk index: 1032 3380\r\n" + ], + [ + 0.044772, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001259, + "\u001b]7;\u0007" + ], + [ + 0.001013, + "\u001b]7;\u0007" + ], + [ + 8.6e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000109, + "\u001b[?1h\u001b=" + ], + [ + 0.000191, + "\u001b[?2004h" + ], + [ + 2.415375, + "\u001b[?1l\u001b>" + ], + [ + 0.000379, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000632, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001347, + "\u001b]7;\u0007" + ], + [ + 0.001044, + "\u001b]7;\u0007" + ], + [ + 8.9e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000101, + "\u001b[?1h\u001b=" + ], + [ + 0.000183, + "\u001b[?2004h" + ], + [ + 0.412865, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.250988, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.245192, + "\b\b\u001b[1m#\u001b[1m \u001b[1mA\u001b[0m\u001b[39m" + ], + [ + 0.706056, + "\b\u001b[1mA\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.273409, + "\b\u001b[1m \u001b[1mv\u001b[0m\u001b[39m" + ], + [ + 0.194462, + "\b\u001b[1mv\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.114445, + "\b\u001b[1me\u001b[1mr\u001b[0m\u001b[39m" + ], + [ + 0.097756, + "\b\u001b[1mr\u001b[1my\u001b[0m\u001b[39m" + ], + [ + 0.149155, + "\b\u001b[1my\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.258303, + "\b\u001b[1m \u001b[1mi\u001b[0m\u001b[39m" + ], + [ + 0.133528, + "\b\u001b[1mi\u001b[1mm\u001b[0m\u001b[39m" + ], + [ + 0.225062, + "\b\u001b[1mm\u001b[1mp\u001b[0m\u001b[39m" + ], + [ + 0.352638, + "\b\u001b[1mp\u001b[1mortant step if you choose keyfile mode (where the keyfile is onl\u001b[1my\u001b[1m saved locally) is to export your keyfile and possibly print it, etc.\u001b[0m\u001b[39m\u001b[K" + ], + [ + 1.170303, + "\u001b[?1l\u001b>" + ], + [ + 0.000524, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000714, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001487, + "\u001b]7;\u0007" + ], + [ + 0.001303, + "\u001b]7;\u0007" + ], + [ + 3.9e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 6.5e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000291, + "\u001b[?2004h" + ], + [ + 2.080689, + "\u001b[1m\u001b[31mb\u001b[0m\u001b[39m" + ], + [ + 0.197142, + "\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[0m\u001b[39m" + ], + [ + 0.172626, + "\b\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[1m\u001b[31mr\u001b[0m\u001b[39m" + ], + [ + 0.145083, + "\b\b\b\u001b[0m\u001b[32mb\u001b[0m\u001b[32mo\u001b[0m\u001b[32mr\u001b[32mg\u001b[39m" + ], + [ + 0.943024, + " " + ], + [ + 0.511742, + "k" + ], + [ + 0.274338, + "e" + ], + [ + 0.308416, + "y" + ], + [ + 0.568141, + " " + ], + [ + 0.62626, + "e" + ], + [ + 0.224255, + "x" + ], + [ + 2.028973, + "p" + ], + [ + 0.220629, + "o" + ], + [ + 0.395617, + "r" + ], + [ + 0.127004, + "t" + ], + [ + 0.635262, + " " + ], + [ + 0.728631, + ":" + ], + [ + 0.116567, + ":" + ], + [ + 0.347323, + " " + ], + [ + 1.713208, + "-" + ], + [ + 0.134471, + "-" + ], + [ + 0.298094, + "q" + ], + [ + 0.316108, + "r" + ], + [ + 0.373821, + "-" + ], + [ + 0.416623, + "c" + ], + [ + 0.400783, + "o" + ], + [ + 0.107762, + "d" + ], + [ + 0.134276, + "e" + ], + [ + 0.384438, + " " + ], + [ + 0.447909, + "f" + ], + [ + 0.162017, + "i" + ], + [ + 0.113187, + "l" + ], + [ + 0.069321, + "e" + ], + [ + 0.627894, + "." + ], + [ + 0.32877, + "h" + ], + [ + 0.137354, + "t" + ], + [ + 0.181468, + "m" + ], + [ + 0.156847, + "l" + ], + [ + 0.434616, + " " + ], + [ + 0.906636, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.546016, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 1.755972, + "\b\u001b[1m \u001b[1mthis creates a nice HTML, but when \u001b[1my\u001b[1mou want something simpler…\u001b[0m\u001b[39m\u001b[K" + ], + [ + 2.940038, + "\b\b\u001b[1mr\u001b[0m\u001b[39m\u001b[K" + ], + [ + 0.691374, + "\b\b\u001b[1me\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.501031, + "\b\b\u001b[1ml\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.0295, + "\b\b\u001b[1mp\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.029695, + "\b\b\u001b[1mm\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.029437, + "\b\b\u001b[1mi\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.03032, + "\b\b\u001b[1ms\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.029433, + "\b\b\u001b[1m \u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.030373, + "\b\b\u001b[1mg\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.029337, + "\b\b\u001b[1mn\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.031058, + "\b\b\u001b[1mi\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.029329, + "\b\b\u001b[1mh\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.031142, + "\b\b\u001b[1mt\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.029181, + "\b\b\u001b[1me\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.029786, + "\b\b\u001b[1mm\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.030603, + "\b\b\u001b[1mo\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.029332, + "\b\b\u001b[1ms\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.030813, + "\b\b\u001b[1m \u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.029428, + "\b\b\u001b[1mt\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.029368, + "\b\b\u001b[1mn\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.030166, + "\b\b\u001b[1ma\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.030524, + "\b\b\u001b[1mw\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.029333, + "\b\b\u001b[1m \u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.030607, + "\b\b\u001b[1mu\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.029346, + "\r\u001b[1my\u001b[1mo\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.031102, + "\r\u001b[1my\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.029544, + "\u001b[A\u001b[76C\u001b[1m \u001b[0m\u001b[39m \u001b[K\r" + ], + [ + 0.029675, + "\u001b[A\u001b[76C\u001b[1mn\u001b[0m\u001b[39m\u001b[K\u001b[1B\r\u001b[K\u001b[A\u001b[77C" + ], + [ + 0.030809, + "\b\b\u001b[1me\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.02987, + "\b\b\u001b[1mh\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.029707, + "\b\b\u001b[1mw\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.029901, + "\b\b\u001b[1m \u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.03057, + "\b\b\u001b[1mt\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.029469, + "\b\b\u001b[1mu\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.030219, + "\b\b\u001b[1mb\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.029227, + "\b\b\u001b[1m \u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.030465, + "\b\b\u001b[1m,\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.029423, + "\b\b\u001b[1mL\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.030292, + "\b\b\u001b[1mM\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.030715, + "\b\b\u001b[1mT\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.029641, + "\b\b\u001b[1mH\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.029367, + "\b\b\u001b[1m \u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.031235, + "\b\b\u001b[1me\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.030119, + "\b\b\u001b[1mc\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.030061, + "\b\b\u001b[1mi\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.030102, + "\b\b\u001b[1mn\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.029384, + "\b\b\u001b[1m \u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.029499, + "\b\b\u001b[1ma\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.03047, + "\b\b\u001b[1m \u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.03019, + "\b\b\u001b[1ms\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.029337, + "\b\b\u001b[1me\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.030138, + "\b\b\u001b[1mt\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.030049, + "\b\b\u001b[1ma\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.030132, + "\b\b\u001b[1me\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.029948, + "\b\b\u001b[1mr\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.029428, + "\b\b\u001b[1mc\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.030197, + "\b\b\u001b[1m \u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.030196, + "\b\b\u001b[1ms\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.03118, + "\b\b\u001b[1mi\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.028165, + "\b\b\u001b[1mh\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.03128, + "\b\b\u001b[1mt\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.029716, + "\b\b\u001b[1m \u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.03012, + "\b\b\u001b[1m#\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.346808, + "\b\u001b[0m\u001b[39m \b" + ], + [ + 0.19843, + "\b" + ], + [ + 0.307235, + "\b \b" + ], + [ + 0.499683, + "\b \b" + ], + [ + 0.028468, + "\b \b" + ], + [ + 0.029472, + "\b \b" + ], + [ + 0.030565, + "\b \b" + ], + [ + 0.029224, + "\b \b" + ], + [ + 0.030493, + "\b \b" + ], + [ + 0.030666, + "\b \b" + ], + [ + 0.029185, + "\b \b" + ], + [ + 0.02989, + "\b" + ], + [ + 0.029921, + "\b \b" + ], + [ + 0.029657, + "\b \b" + ], + [ + 0.154399, + "\b \b" + ], + [ + 0.165915, + "\b \b" + ], + [ + 0.154316, + "\b \b" + ], + [ + 0.154588, + "\b \b" + ], + [ + 0.147868, + "\b \b" + ], + [ + 1.555865, + "p" + ], + [ + 0.446126, + "a" + ], + [ + 0.188714, + "p" + ], + [ + 0.252833, + "e" + ], + [ + 0.142044, + "r" + ], + [ + 0.395895, + " " + ], + [ + 0.423453, + "\u001b[1m# this is a \"manual input\"-only backup (but it is\u001b[1m \u001b[1malso included in the --qr-code option)\u001b[0m\u001b[39m\u001b[K" + ], + [ + 3.71528, + "\u001b[?1l\u001b>" + ], + [ + 0.001413, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000757, + "\u001b]2;borg key export :: --paper\u0007\u001b]1;borg\u0007" + ], + [ + 0.550352, + "To restore key use borg key import --paper /path/to/repo\r\n\r\nBORG PAPER KEY v1\r\nid: 20 / 54f957 2d6d72 de8280 / 158a57 45bdc3 - f6\r\n 1: 86a961 6c676f 726974 686da6 736861 323536 - 14\r\n 2: a46461 7461da 00def1 7c9f3c 81ebc6 730a05 - 35\r\n 3: 12453e d02760 ffdeef 4d0daa 231d81 ae10d8 - e5\r\n 4: 7bb0a1 97c30f 312b61 7170ba d1ea91 da2c88 - 30\r\n 5: ca997e 177b74 38f906 709a66 fbf013 40ab3d - c4\r\n 6: 6af94b 8a36a9 e07b9d b0e08d 3935cd f1bbb9 - 5c\r\n 7: 2b10b6 ebb586 4c0967 f682b9 c64358 fbb63c - a4\r\n 8: b9fc94 240d08 072524 98b619 7bd1c5 21094e - ec\r\n 9: ac4f05 d65a6a 7f8a0d 8cc14e 405b36 c248e1 - 79\r\n10: d23b89 c61074 3e68c9 79c683 2384e8 cd9f82 - 50\r\n11: fc76a9 3f2a9e 05d5f1 313f95 ec4313 53e0c1 - 4a\r\n12: 654f1d ab2b51 2ccbe8 80be07 b6132f 86aeb5 - 11\r\n13: 7e6e48 5ff0d4 41e659 a421f0 5123df f88dff - c9\r\n14: 03db58 bbb410 87d7fc 075b14 5108a4 686173 - 9a\r\n15: 68da00 20524b 8769e9 e5bd18 a9b431 c05b49 - ba\r\n16: 505280 9b104a b081c0 f4efd1 1d3771 34c701 - 40\r\n17: aa6974 657261 74696f 6e73ce 000186 a0a473 - 15\r\n18: 616c7" + ], + [ + 7.2e-05, + "4 da0020 0be74e e1e9af 7b1364 3ee362 - 32\r\n19: 643069 b57a75 d30eb6 104c28 367e17 7dd4d9 - 79\r\n20: f556a7 766572 73696f 6e01 - 32\r\n\r\n" + ], + [ + 0.048873, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001193, + "\u001b]7;\u0007" + ], + [ + 0.000921, + "\u001b]7;\u0007" + ], + [ + 9.3e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 8e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000185, + "\u001b[?2004h" + ], + [ + 3.146565, + "\u001b[?1l\u001b>" + ], + [ + 0.000424, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000795, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001307, + "\u001b]7;\u0007" + ], + [ + 0.001444, + "\u001b]7;\u0007" + ], + [ + 8.9e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.00011, + "\u001b[?1h\u001b=" + ], + [ + 0.000263, + "\u001b[?2004h" + ], + [ + 0.441809, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.136081, + "\b\u001b[1m#\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.375389, + "\b\b\u001b[1m#\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.284554, + "\b\u001b[1m \u001b[1mM\u001b[0m\u001b[39m" + ], + [ + 0.395833, + "\b\u001b[1mM\u001b[1mA\u001b[0m\u001b[39m" + ], + [ + 0.434316, + "\b\u001b[1mA\u001b[1mINTENANCE ##\u001b[0m\u001b[39m" + ], + [ + 1.471226, + "\u001b[?1l\u001b>" + ], + [ + 0.00055, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000605, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001464, + "\u001b]7;\u0007" + ], + [ + 0.00092, + "\u001b]7;\u0007" + ], + [ + 9.4e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000104, + "\u001b[?1h\u001b=" + ], + [ + 0.000309, + "\u001b[?2004h" + ], + [ + 0.977805, + "\u001b[?1l\u001b>" + ], + [ + 0.000452, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000828, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001436, + "\u001b]7;\u0007" + ], + [ + 0.001464, + "\u001b]7;\u0007" + ], + [ + 3.8e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000171, + "\u001b[?1h\u001b=" + ], + [ + 0.000247, + "\u001b[?2004h" + ], + [ + 0.221358, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.374414, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.189751, + "\b\b\u001b[1m#\u001b[1m \u001b[1mS\u001b[0m\u001b[39m" + ], + [ + 0.087275, + "\b\u001b[1mS\u001b[1mo\u001b[0m\u001b[39m" + ], + [ + 0.140008, + "\b\u001b[1mo\u001b[1mm\u001b[0m\u001b[39m" + ], + [ + 0.150891, + "\b\u001b[1mm\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.387855, + "\b\u001b[1me\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.204067, + "\b\u001b[1mt\u001b[1mi\u001b[0m\u001b[39m" + ], + [ + 0.127209, + "\b\u001b[1mi\u001b[1mm\u001b[0m\u001b[39m" + ], + [ + 0.073999, + "\b\u001b[1mm\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.130356, + "\b\u001b[1me\u001b[1ms\u001b[0m\u001b[39m" + ], + [ + 0.224406, + "\b\u001b[1ms\u001b[1m backups get broken or we want a regular \"checkup\" that everythin\u001b[1mg\u001b[1m is okay…\u001b[0m\u001b[39m\u001b[K" + ], + [ + 2.361948, + "\u001b[?1l\u001b>" + ], + [ + 0.000402, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000743, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001212, + "\u001b]7;\u0007" + ], + [ + 0.000923, + "\u001b]7;\u0007" + ], + [ + 1.3e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 8.3e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000321, + "\u001b[?2004h" + ], + [ + 2.246766, + "\u001b[1m\u001b[31mb\u001b[0m\u001b[39m" + ], + [ + 0.18622, + "\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[0m\u001b[39m" + ], + [ + 0.121068, + "\b\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[1m\u001b[31mr\u001b[0m\u001b[39m" + ], + [ + 0.146401, + "\b\b\b\u001b[0m\u001b[32mb\u001b[0m\u001b[32mo\u001b[0m\u001b[32mr\u001b[32mg\u001b[39m" + ], + [ + 0.255479, + " " + ], + [ + 0.268833, + "c" + ], + [ + 0.154418, + "h" + ], + [ + 0.106649, + "e" + ], + [ + 0.142762, + "c" + ], + [ + 0.306359, + "k" + ], + [ + 0.697455, + " " + ], + [ + 1.113236, + "-" + ], + [ + 0.768765, + "v" + ], + [ + 0.477353, + " " + ], + [ + 0.387303, + ":" + ], + [ + 0.102251, + ":" + ], + [ + 0.749971, + "\u001b[?1l\u001b>" + ], + [ + 0.001961, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000798, + "\u001b]2;borg check -v ::\u0007\u001b]1;borg\u0007" + ], + [ + 0.54272, + "Starting repository check\r\n" + ], + [ + 1.152819, + "Starting repository index check\r\n" + ], + [ + 0.00038, + "Completed repository check, no problems found.\r\n" + ], + [ + 0.000129, + "Starting archive consistency check...\r\n" + ], + [ + 0.095799, + "Analyzing archive backup1 (1/6)\r\n" + ], + [ + 0.109358, + "Analyzing archive backup2 (2/6)\r\n" + ], + [ + 0.036555, + "Analyzing archive backup3 (3/6)\r\n" + ], + [ + 0.03649, + "Analyzing archive rugk-2017-07-16T18:51:34 (4/6)\r\n" + ], + [ + 0.000491, + "Analyzing archive rugk-2017-07-16T18:52:19 (5/6)\r\n" + ], + [ + 0.000729, + "Analyzing archive backup-block-device (6/6)\r\n" + ], + [ + 0.00119, + "Archive consistency check complete, no problems found.\r\n" + ], + [ + 0.081895, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001153, + "\u001b]7;\u0007" + ], + [ + 0.000924, + "\u001b]7;\u0007" + ], + [ + 0.000108, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 8.6e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.00022, + "\u001b[?2004h" + ], + [ + 2.243609, + "\u001b[?1l\u001b>" + ], + [ + 0.000511, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000535, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001789, + "\u001b]7;\u0007" + ], + [ + 0.00157, + "\u001b]7;\u0007" + ], + [ + 0.000139, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 7.7e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.00033, + "\u001b[?2004h" + ], + [ + 0.326751, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.24289, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.285802, + "\b\b\u001b[1m#\u001b[1m \u001b[1mN\u001b[0m\u001b[39m" + ], + [ + 0.191158, + "\b\u001b[1mN\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.184029, + "\b\u001b[1me\u001b[1mx\u001b[0m\u001b[39m" + ], + [ + 0.16373, + "\b\u001b[1mx\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.239936, + "\b\u001b[1mt\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.27885, + "\b\u001b[1m \u001b[1mp\u001b[0m\u001b[39m" + ], + [ + 0.12665, + "\b\u001b[1mp\u001b[1mr\u001b[0m\u001b[39m" + ], + [ + 0.154792, + "\b\u001b[1mr\u001b[1mo\u001b[0m\u001b[39m" + ], + [ + 0.372203, + "\b\u001b[1mo\u001b[1mblem: Usually you do not have infinite disk space. So you may need\u001b[1m \u001b[1mto prune your archive…\u001b[0m\u001b[39m\u001b[K" + ], + [ + 1.956234, + "\u001b[?1l\u001b>" + ], + [ + 0.000446, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000607, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001281, + "\u001b]7;\u0007" + ], + [ + 0.000983, + "\u001b]7;\u0007" + ], + [ + 2.2e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000142, + "\u001b[?1h\u001b=" + ], + [ + 0.00032, + "\u001b[?2004h" + ], + [ + 1.137641, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.26675, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.151609, + "\b\b\u001b[1m#\u001b[1m \u001b[1mY\u001b[0m\u001b[39m" + ], + [ + 0.11765, + "\b\u001b[1mY\u001b[1mo\u001b[0m\u001b[39m" + ], + [ + 0.158458, + "\b\u001b[1mo\u001b[1mu\u001b[0m\u001b[39m" + ], + [ + 0.149615, + "\b\u001b[1mu\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.080657, + "\b\u001b[1m \u001b[1mc\u001b[0m\u001b[39m" + ], + [ + 0.144379, + "\b\u001b[1mc\u001b[1ma\u001b[0m\u001b[39m" + ], + [ + 0.104266, + "\b\u001b[1ma\u001b[1mn\u001b[0m\u001b[39m" + ], + [ + 0.132218, + "\b\u001b[1mn\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.202965, + "\b\u001b[1m \u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.17807, + "\b\u001b[1mt\u001b[1mu\u001b[0m\u001b[39m" + ], + [ + 0.123814, + "\b\u001b[1mu\u001b[1mn\u001b[0m\u001b[39m" + ], + [ + 0.325016, + "\b\u001b[1mn\u001b[1me this in every detail. See the docs for details. Here only a s\u001b[1mi\u001b[1mmple example:\u001b[0m\u001b[39m\u001b[K" + ], + [ + 1.91505, + "\u001b[?1l\u001b>" + ], + [ + 0.000406, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000684, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001347, + "\u001b]7;\u0007" + ], + [ + 0.001084, + "\u001b]7;\u0007" + ], + [ + 0.000116, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000118, + "\u001b[?1h\u001b=" + ], + [ + 0.000246, + "\u001b[?2004h" + ], + [ + 2.556304, + "\u001b[1m\u001b[31mb\u001b[0m\u001b[39m" + ], + [ + 0.198214, + "\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[0m\u001b[39m" + ], + [ + 0.125589, + "\b\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[1m\u001b[31mr\u001b[0m\u001b[39m" + ], + [ + 0.147156, + "\b\b\b\u001b[0m\u001b[32mb\u001b[0m\u001b[32mo\u001b[0m\u001b[32mr\u001b[32mg\u001b[39m" + ], + [ + 0.202848, + " " + ], + [ + 0.369539, + "p" + ], + [ + 0.228714, + "r" + ], + [ + 0.184236, + "u" + ], + [ + 0.154014, + "n" + ], + [ + 0.136362, + "e" + ], + [ + 0.94169, + " " + ], + [ + 0.44829, + "-" + ], + [ + 0.112062, + "-" + ], + [ + 0.37454, + "l" + ], + [ + 0.157195, + "i" + ], + [ + 0.116633, + "s" + ], + [ + 0.193515, + "t" + ], + [ + 0.486369, + " " + ], + [ + 0.442107, + "-" + ], + [ + 0.12257, + "-" + ], + [ + 0.403774, + "k" + ], + [ + 0.214488, + "e" + ], + [ + 0.771743, + "e" + ], + [ + 0.349591, + "p" + ], + [ + 0.352253, + "-" + ], + [ + 0.201267, + "l" + ], + [ + 0.109728, + "a" + ], + [ + 0.146296, + "s" + ], + [ + 0.130476, + "t" + ], + [ + 0.234998, + " " + ], + [ + 0.264266, + "1" + ], + [ + 0.429572, + " " + ], + [ + 0.505667, + "-" + ], + [ + 0.105697, + "-" + ], + [ + 0.294354, + "d" + ], + [ + 0.178175, + "r" + ], + [ + 0.239011, + "y" + ], + [ + 0.561933, + "-" + ], + [ + 0.220564, + "r" + ], + [ + 0.172983, + "u" + ], + [ + 0.138969, + "n" + ], + [ + 0.891028, + "\u001b[?1l\u001b>" + ], + [ + 0.004152, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000975, + "\u001b]2;borg prune --list --keep-last 1 --dry-run\u0007\u001b]1;borg\u0007" + ], + [ + 0.658906, + "Keeping archive: backup-block-device Sun, 2017-07-16 18:52:58 [5fd9732b4809252742a7cb3fadf2a971dd6371afd11a07944c0b5803d57c240f]\r\n" + ], + [ + 0.000155, + "Would prune: rugk-2017-07-16T18:52:19 Sun, 2017-07-16 18:52:19 [0de98f590b004ad7545f2013c4c9f2d4e3eed1415d177c89d6c2b7ff05918d2e]\r\n" + ], + [ + 0.000118, + "Would prune: rugk-2017-07-16T18:51:34 Sun, 2017-07-16 18:51:34 [d054cc411324d4bd848b39d1c9cad909073f9ff1a1a503a676d3e050be140396]\r\n" + ], + [ + 6.5e-05, + "Would prune: backup3 Fri, 2017-07-14 21:55:37 [36cd8fdf9b8b2e3bbb3fc2bb600acd48609efaf3a0880f900e0701a47ff69d4d]\r\n" + ], + [ + 7.1e-05, + "Would prune: backup2 Fri, 2017-07-14 21:54:56 [5aaf03d1c710cf774f9c9ff1c6317b621c14e519c6bac459f6d64b31e3bbd200]\r\n" + ], + [ + 7.1e-05, + "Would prune: backup1 Fri, 2017-07-14 21:54:06 [9758c7db339a066360bffad17b2ffac4fb368c6722c0be3a47a7a9b631f06407]\r\n" + ], + [ + 0.047362, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001109, + "\u001b]7;\u0007" + ], + [ + 0.00093, + "\u001b]7;\u0007" + ], + [ + 7.6e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 8.8e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000163, + "\u001b[?2004h" + ], + [ + 2.173126, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.420696, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.658252, + "\b\b\u001b[1m#\u001b[1m \u001b[1mW\u001b[0m\u001b[39m" + ], + [ + 0.186236, + "\b\u001b[1mW\u001b[1mh\u001b[0m\u001b[39m" + ], + [ + 0.09843, + "\b\u001b[1mh\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.143515, + "\b\u001b[1me\u001b[1mn\u001b[0m\u001b[39m" + ], + [ + 0.153626, + "\b\u001b[1mn\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.136407, + "\b\u001b[1m \u001b[1ma\u001b[0m\u001b[39m" + ], + [ + 0.170555, + "\b\u001b[1ma\u001b[1mc\u001b[0m\u001b[39m" + ], + [ + 0.157309, + "\b\u001b[1mc\u001b[1mtually executing it in a script, you have to use it without the --dry\u001b[1m-\u001b[1mrun option, of course.\u001b[0m\u001b[39m\u001b[K" + ], + [ + 2.08243, + "\u001b[?1l\u001b>" + ], + [ + 0.000512, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000552, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001375, + "\u001b]7;\u0007" + ], + [ + 0.000922, + "\u001b]7;\u0007" + ], + [ + 3.6e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 8.1e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.00026, + "\u001b[?2004h" + ], + [ + 1.169356, + "\u001b[?1l\u001b>" + ], + [ + 0.000602, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000917, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001594, + "\u001b]7;\u0007" + ], + [ + 0.001826, + "\u001b]7;\u0007" + ], + [ + 7.1e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000204, + "\u001b[?1h\u001b=" + ], + [ + 0.000349, + "\u001b[?2004h" + ], + [ + 0.464206, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.135956, + "\b\u001b[1m#\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.484249, + "\b\b\u001b[1m#\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.299809, + "\b\u001b[1m \u001b[1mR\u001b[0m\u001b[39m" + ], + [ + 0.199072, + "\b\u001b[1mR\u001b[1mE\u001b[0m\u001b[39m" + ], + [ + 0.620669, + "\b\u001b[1mE\u001b[1mSTORE ##\u001b[0m\u001b[39m" + ], + [ + 0.924028, + "\u001b[?1l\u001b>" + ], + [ + 0.000399, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000744, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001142, + "\u001b]7;\u0007" + ], + [ + 0.000834, + "\u001b]7;\u0007" + ], + [ + 9.1e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000124, + "\u001b[?1h\u001b=" + ], + [ + 0.000294, + "\u001b[?2004h" + ], + [ + 0.797042, + "\u001b[?1l\u001b>" + ], + [ + 0.000325, + "\u001b[?2004l\r\r\n" + ], + [ + 0.001543, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.002662, + "\u001b]7;\u0007" + ], + [ + 0.001568, + "\u001b]7;\u0007" + ], + [ + 4.2e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 9.4e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000185, + "\u001b[?2004h" + ], + [ + 0.705049, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.50212, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 1.210452, + "\b\b\u001b[1m#\u001b[1m \u001b[1mW\u001b[0m\u001b[39m" + ], + [ + 0.1987, + "\b\u001b[1mW\u001b[1mh\u001b[0m\u001b[39m" + ], + [ + 0.12116, + "\b\u001b[1mh\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.152173, + "\b\u001b[1me\u001b[1mn\u001b[0m\u001b[39m" + ], + [ + 0.16582, + "\b\u001b[1mn\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.378037, + "\b\u001b[1m \u001b[1my\u001b[0m\u001b[39m" + ], + [ + 0.330829, + "\b\u001b[1my\u001b[1mo\u001b[0m\u001b[39m" + ], + [ + 0.180945, + "\b\u001b[1mo\u001b[1mu\u001b[0m\u001b[39m" + ], + [ + 0.152701, + "\b\u001b[1mu\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.121298, + "\b\u001b[1m \u001b[1mw\u001b[0m\u001b[39m" + ], + [ + 0.148067, + "\b\u001b[1mw\u001b[1ma\u001b[0m\u001b[39m" + ], + [ + 0.233865, + "\b\u001b[1ma\u001b[1mnt to see the diff between two archives use this command.\u001b[0m\u001b[39m" + ], + [ + 1.947763, + "\u001b[?1l\u001b>" + ], + [ + 0.000408, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000607, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001033, + "\u001b]7;\u0007" + ], + [ + 0.000979, + "\u001b]7;\u0007" + ], + [ + 0.000127, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 7.2e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000278, + "\u001b[?2004h" + ], + [ + 0.693036, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.275798, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.281158, + "\b\b\u001b[1m#\u001b[1m \u001b[1mE\u001b[0m\u001b[39m" + ], + [ + 0.386709, + "\b\u001b[1mE\u001b[1m.\u001b[0m\u001b[39m" + ], + [ + 0.136187, + "\b\u001b[1m.\u001b[1mg\u001b[0m\u001b[39m" + ], + [ + 0.262011, + "\b\u001b[1mg\u001b[1m.\u001b[0m\u001b[39m" + ], + [ + 0.234889, + "\b\u001b[1m.\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.361971, + "\b\u001b[1m \u001b[1mw\u001b[0m\u001b[39m" + ], + [ + 0.162798, + "\b\u001b[1mw\u001b[1mh\u001b[0m\u001b[39m" + ], + [ + 0.077265, + "\b\u001b[1mh\u001b[1ma\u001b[0m\u001b[39m" + ], + [ + 0.148774, + "\b\u001b[1ma\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.34541, + "\b\u001b[1mt\u001b[1m happened between the first two backups?\u001b[0m\u001b[39m" + ], + [ + 1.295996, + "\u001b[?1l\u001b>" + ], + [ + 0.000733, + "\u001b[?2004l\r\r\n" + ], + [ + 0.001102, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001634, + "\u001b]7;\u0007" + ], + [ + 0.000634, + "\u001b]7;\u0007" + ], + [ + 5.6e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 7.4e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000303, + "\u001b[?2004h" + ], + [ + 0.441685, + "\u001b[1m\u001b[31mb\u001b[0m\u001b[39m" + ], + [ + 0.182795, + "\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[0m\u001b[39m" + ], + [ + 0.072867, + "\b\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[1m\u001b[31mr\u001b[0m\u001b[39m" + ], + [ + 0.161104, + "\b\b\b\u001b[0m\u001b[32mb\u001b[0m\u001b[32mo\u001b[0m\u001b[32mr\u001b[32mg\u001b[39m" + ], + [ + 0.179655, + " " + ], + [ + 0.154676, + "d" + ], + [ + 0.132421, + "i" + ], + [ + 0.124239, + "f" + ], + [ + 0.13999, + "f" + ], + [ + 0.624444, + " " + ], + [ + 0.862302, + ":" + ], + [ + 0.1169, + ":" + ], + [ + 0.274626, + "b" + ], + [ + 0.100778, + "a" + ], + [ + 0.188526, + "c" + ], + [ + 0.097402, + "k" + ], + [ + 0.144999, + "u" + ], + [ + 0.22317, + "p" + ], + [ + 0.167969, + "1" + ], + [ + 0.44642, + " " + ], + [ + 0.240129, + "b" + ], + [ + 0.164579, + "a" + ], + [ + 0.190471, + "c" + ], + [ + 0.136211, + "k" + ], + [ + 0.12257, + "u" + ], + [ + 0.258587, + "p" + ], + [ + 0.215453, + "2" + ], + [ + 1.160869, + "\u001b[?1l\u001b>" + ], + [ + 0.001983, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000801, + "\u001b]2;borg diff ::backup1 backup2\u0007\u001b]1;borg\u0007" + ], + [ + 0.717522, + "added 20 B Wallpaper/newfile.txt\r\n" + ], + [ + 0.044186, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001157, + "\u001b]7;\u0007" + ], + [ + 0.000949, + "\u001b]7;\u0007" + ], + [ + 0.000108, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 9.3e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000147, + "\u001b[?2004h" + ], + [ + 1.545435, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.26435, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.178864, + "\b\b\u001b[1m#\u001b[1m \u001b[1mA\u001b[0m\u001b[39m" + ], + [ + 0.161899, + "\b\u001b[1mA\u001b[1mh\u001b[0m\u001b[39m" + ], + [ + 0.240289, + "\b\u001b[1mh\u001b[1m,\u001b[0m\u001b[39m" + ], + [ + 0.132971, + "\b\u001b[1m,\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.115812, + "\b\u001b[1m \u001b[1mw\u001b[0m\u001b[39m" + ], + [ + 0.111227, + "\b\u001b[1mw\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.159647, + "\b\u001b[1me\u001b[1m added a file, right…\u001b[0m\u001b[39m" + ], + [ + 0.97686, + "\u001b[?1l\u001b>" + ], + [ + 0.000441, + "\u001b[?2004l\r\r\n" + ], + [ + 0.00091, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001031, + "\u001b]7;\u0007" + ], + [ + 0.000995, + "\u001b]7;\u0007" + ], + [ + 2.5e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000141, + "\u001b[?1h\u001b=" + ], + [ + 0.000303, + "\u001b[?2004h" + ], + [ + 6.370198, + "\u001b[?1l\u001b>" + ], + [ + 0.000854, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000815, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.003101, + "\u001b]7;\u0007" + ], + [ + 0.002831, + "\u001b]7;\u0007" + ], + [ + 0.000107, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000314, + "\u001b[?1h\u001b=" + ], + [ + 0.000499, + "\u001b[?2004h" + ], + [ + 0.580198, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.240323, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.29592, + "\b\b\u001b[1m#\u001b[1m \u001b[1mT\u001b[0m\u001b[39m" + ], + [ + 0.135389, + "\b\u001b[1mT\u001b[1mh\u001b[0m\u001b[39m" + ], + [ + 0.115437, + "\b\u001b[1mh\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.157526, + "\b\u001b[1me\u001b[1mr\u001b[0m\u001b[39m" + ], + [ + 0.624235, + "\b\u001b[1mr\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.282742, + "\b\u001b[1me\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.133006, + "\b\u001b[1m \u001b[1ma\u001b[0m\u001b[39m" + ], + [ + 0.206434, + "\b\u001b[1ma\u001b[1mr\u001b[0m\u001b[39m" + ], + [ + 0.13301, + "\b\u001b[1mr\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.255991, + "\b\u001b[1me\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.196416, + "\b\u001b[1m \u001b[1ma\u001b[0m\u001b[39m" + ], + [ + 0.275594, + "\b\u001b[1ma\u001b[1mlso other ways to extract the data.\u001b[0m\u001b[39m" + ], + [ + 0.932018, + "\u001b[?1l\u001b>" + ], + [ + 0.001354, + "\u001b[?2004l\r\r\n" + ], + [ + 0.001071, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.00297, + "\u001b]7;\u0007" + ], + [ + 0.002675, + "\u001b]7;\u0007" + ], + [ + 0.000154, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000231, + "\u001b[?1h\u001b=" + ], + [ + 0.000895, + "\u001b[?2004h" + ], + [ + 1.021752, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.238058, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.245484, + "\b\b\u001b[1m#\u001b[1m \u001b[1mE\u001b[0m\u001b[39m" + ], + [ + 0.719467, + "\b\u001b[1mE\u001b[1m.\u001b[0m\u001b[39m" + ], + [ + 0.151468, + "\b\u001b[1m.\u001b[1mg\u001b[0m\u001b[39m" + ], + [ + 0.183213, + "\b\u001b[1mg\u001b[1m.\u001b[0m\u001b[39m" + ], + [ + 0.599958, + "\b\u001b[1m.\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.316279, + "\b\u001b[1m \u001b[1ma\u001b[0m\u001b[39m" + ], + [ + 0.166858, + "\b\u001b[1ma\u001b[1ms\u001b[0m\u001b[39m" + ], + [ + 0.551272, + "\b\u001b[1ms\u001b[1m a tar archive.\u001b[0m\u001b[39m" + ], + [ + 0.938861, + "\u001b[?1l\u001b>" + ], + [ + 0.000638, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000793, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001159, + "\u001b]7;\u0007" + ], + [ + 0.000867, + "\u001b]7;\u0007" + ], + [ + 9.1e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 8.5e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000282, + "\u001b[?2004h" + ], + [ + 0.860998, + "\u001b[1m\u001b[31mb\u001b[0m\u001b[39m" + ], + [ + 0.189263, + "\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[0m\u001b[39m" + ], + [ + 0.11245, + "\b\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[1m\u001b[31mr\u001b[0m\u001b[39m" + ], + [ + 0.133531, + "\b\b\b\u001b[0m\u001b[32mb\u001b[0m\u001b[32mo\u001b[0m\u001b[32mr\u001b[32mg\u001b[39m" + ], + [ + 0.62438, + " " + ], + [ + 0.295845, + "e" + ], + [ + 0.165874, + "x" + ], + [ + 0.180501, + "p" + ], + [ + 0.166254, + "o" + ], + [ + 0.27793, + "r" + ], + [ + 0.113477, + "t" + ], + [ + 0.46559, + "-" + ], + [ + 0.34577, + "t" + ], + [ + 0.148398, + "a" + ], + [ + 0.17144, + "r" + ], + [ + 0.920527, + " " + ], + [ + 0.40208, + "-" + ], + [ + 0.108683, + "-" + ], + [ + 0.326944, + "p" + ], + [ + 0.195982, + "r" + ], + [ + 0.175632, + "o" + ], + [ + 0.229442, + "g" + ], + [ + 0.133505, + "r" + ], + [ + 0.171995, + "e" + ], + [ + 0.244119, + "s" + ], + [ + 0.154514, + "s" + ], + [ + 0.579295, + " " + ], + [ + 0.575201, + ":" + ], + [ + 0.112098, + ":" + ], + [ + 0.355392, + "b" + ], + [ + 0.110008, + "a" + ], + [ + 0.172393, + "c" + ], + [ + 0.080739, + "k" + ], + [ + 0.134163, + "u" + ], + [ + 0.221434, + "p" + ], + [ + 0.276712, + "2" + ], + [ + 0.6747, + " " + ], + [ + 0.372614, + "b" + ], + [ + 0.09319, + "a" + ], + [ + 0.152876, + "c" + ], + [ + 0.089531, + "k" + ], + [ + 0.150747, + "u" + ], + [ + 0.233879, + "p" + ], + [ + 0.273301, + "." + ], + [ + 0.354416, + "t" + ], + [ + 0.107034, + "a" + ], + [ + 0.144993, + "r" + ], + [ + 0.463039, + "." + ], + [ + 0.352906, + "g" + ], + [ + 0.133262, + "z" + ], + [ + 1.083854, + "\u001b[?1l\u001b>" + ], + [ + 0.004197, + "\u001b[?2004l\r\r\n" + ], + [ + 0.001109, + "\u001b]2;borg export-tar --progress ::backup2 backup.tar.gz\u0007" + ], + [ + 4.7e-05, + "\u001b]1;borg\u0007" + ], + [ + 0.679042, + "Calculating size \r" + ], + [ + 0.036244, + " 0.0% Processing: Wallpaper/bigcollection/Macaws_USBingImage.jpg \r" + ], + [ + 0.020857, + " 0.1% Processing: Wallpaper/bigcollection/_A...r_the_town_of_Herrischried.jpg\r" + ], + [ + 0.030544, + " 0.2% Processing: Wallpaper/bigcollection/_A...her_Atlas__Marokko____Doug.jpg\r" + ], + [ + 0.030864, + " 0.3% Processing: Wallpaper/bigcollection/_A...ssar__S_d-Sulawesi__Indone.jpg\r" + ], + [ + 0.030144, + " 0.4% Processing: Wallpaper/bigcollection/_A..._N_he_von_Timimoun__Algeri.jpg\r" + ], + [ + 0.027643, + " 0.5% Processing: Wallpaper/bigcollection/_A...t_kleinen__aus_Servietten_.jpg\r" + ], + [ + 0.03121, + " 0.6% Processing: Wallpaper/bigcollection/_A...g_Norderoog__small_island_.jpg\r" + ], + [ + 0.031343, + " 0.7% Processing: Wallpaper/bigcollection/_A...im_Snowdonia-Nationalpark_.jpg\r" + ], + [ + 0.031862, + " 0.8% Processing: Wallpaper/bigcollection/_A...l_an_einem_Wasserloch_im_Q.jpg\r" + ], + [ + 0.034847, + " 0.9% Processing: Wallpaper/bigcollection/_A...nten____James_HagerOffset_.jpg\r" + ], + [ + 0.033989, + " 1.0% Processing: Wallpaper/bigcollection/_A...nen__Masai_Mara_National_R.jpg\r" + ], + [ + 0.027388, + " 1.1% Processing: Wallpaper/bigcollection/_A...ard_im_Londolozi-Wildreser.jpg\r" + ], + [ + 0.026632, + " 1.2% Processing: Wallpaper/bigcollection/_A...orning_fog__Aidling__Pfaff.jpg\r" + ], + [ + 0.030864, + " 1.3% Processing: Wallpaper/bigcollection/_A...hutzgebiet_Lewa_Wildlife_C.jpg\r" + ], + [ + 0.029943, + " 1.4% Processing: Wallpaper/bigcollection/_A...in_Delhi__Indien____AirPan.jpg\r" + ], + [ + 0.035404, + " 1.5% Processing: Wallpaper/bigcollection/_A...morial_Gardens_in_den_Dand.jpg\r" + ], + [ + 0.030931, + " 1.6% Processing: Wallpaper/bigcollection/_A...rthumberland__England____A.jpg\r" + ], + [ + 0.035605, + " 1.7% Processing: Wallpaper/bigcollection/_A...berg__Bayern__Deutschland_.jpg\r" + ], + [ + 0.026827, + " 1.8% Processing: Wallpaper/bigcollection/_A...ns_am_Little_Missouri_Rive.jpg\r" + ], + [ + 0.030196, + " 1.9% Processing: Wallpaper/bigcollection/_A...toberfest_in_Munich__Ger 1.jpg\r" + ], + [ + 0.025763, + " 2.0% Processing: Wallpaper/bigcollection/_A...toberfest_in_Munich__Ger 2.jpg\r" + ], + [ + 0.025306, + " 2.1% Processing: Wallpaper/bigcollection/_A...toberfest_in_Munich__Ger 3.jpg\r" + ], + [ + 0.027286, + " 2.2% Processing: Wallpaper/bigcollection/_A...Oktoberfest_in_Munich__Ger.jpg\r" + ], + [ + 0.02806, + " 2.3% Processing: Wallpaper/bigcollection/_A..._Florida-Scheibenanemone__.jpg\r" + ], + [ + 0.032994, + " 2.4% Processing: Wallpaper/bigcollection/_A...n__Nationalpark_Ankarafant.jpg\r" + ], + [ + 0.033538, + " 2.5% Processing: Wallpaper/bigcollection/_A..._der_N_he_von_Page__Arizon.jpg\r" + ], + [ + 0.030034, + " 2.6% Processing: Wallpaper/bigcollection/_A...r_Inselgruppe_L_archipel_d.jpg\r" + ], + [ + 0.030477, + " 2.7% Processing: Wallpaper/bigcollection/_A...land____Hercules_MilasAlam.jpg\r" + ], + [ + 0.033376, + " 2.8% Processing: Wallpaper/bigcollection/_A...maguchiFlickr_OpenGetty_Im.jpg\r" + ], + [ + 0.032919, + " 2.9% Processing: Wallpaper/bigcollection/_A...imetersubmillimeter_Array_.jpg\r" + ], + [ + 0.027034, + " 4.7% Processing: Wallpaper/bigcollection/_B...rairie_Creek_Redwoods_Stat.jpg\r" + ], + [ + 0.034892, + " 4.8% Processing: Wallpaper/bigcollection/_B...__Montana__USA____Jeff_Kro.jpg\r" + ], + [ + 0.031042, + " 4.9% Processing: Wallpaper/bigcollection/_B...gatta__Golf_von_Triest__It.jpg\r" + ], + [ + 0.030521, + " 5.0% Processing: Wallpaper/bigcollection/_B...and__Schleswig-Holstein__D.jpg\r" + ], + [ + 0.028755, + " 5.1% Processing: Wallpaper/bigcollection/_B..._Islands__Irland____Bart_B.jpg\r" + ], + [ + 0.031129, + " 5.2% Processing: Wallpaper/bigcollection/_B...e_im_Glacier-Nationalpark_.jpg\r" + ], + [ + 0.032588, + " 5.3% Processing: Wallpaper/bigcollection/_B...Nationalpark_Bayerischer_W.jpg\r" + ], + [ + 0.025077, + " 5.4% Processing: Wallpaper/bigcollection/_B...Arena_bei_Nacht__Stockholm.jpg\r" + ], + [ + 0.027803, + " 5.5% Processing: Wallpaper/bigcollection/_B...ner_Fernsehturm_w_hrend_de.jpg\r" + ], + [ + 0.031262, + " 5.6% Processing: Wallpaper/bigcollection/_B...nd__Bayern__Deutschland_mi.jpg\r" + ], + [ + 0.031721, + " 5.7% Processing: Wallpaper/bigcollection/_B...er__Schwarzwald__Baden-W_r.jpg\r" + ], + [ + 0.032768, + " 5.8% Processing: Wallpaper/bigcollection/_B...ebirge_oberhalb_der_Dad_s-.jpg\r" + ], + [ + 0.030763, + " 5.9% Processing: Wallpaper/bigcollection/_B...ngerburgbahn__Innsbruck___.jpg\r" + ], + [ + 0.028673, + " 7.6% Processing: Wallpaper/bigcollection/_B...rn_des__Wilson_Stump___ein.jpg\r" + ], + [ + 0.029182, + " 7.7% Processing: Wallpaper/bigcollection/_B...t_Jefferson-Wildschutzgebi.jpg\r" + ], + [ + 0.029225, + " 11.2% Processing: Wallpaper/bigcollection/_B...Saloum-Delta__Senegal____B.jpg\r" + ], + [ + 0.030837, + " 11.3% Processing: Wallpaper/bigcollection/_B..._Venedig__Italien____Digit.jpg\r" + ], + [ + 0.034033, + " 11.4% Processing: Wallpaper/bigcollection/_B..._Koblenz_und_Trier__Rheinl.jpg\r" + ], + [ + 0.028958, + " 11.5% Processing: Wallpaper/bigcollection/_B..._Baden-W_rttemberg__Deutsc.jpg\r" + ], + [ + 0.025933, + " 11.6% Processing: Wallpaper/bigcollection/_B..._Bisingen__Baden-W_rttembe.jpg\r" + ], + [ + 0.030318, + " 11.7% Processing: Wallpaper/bigcollection/_B...in_Koknese__Lettland____An.jpg\r" + ], + [ + 0.029535, + " 11.8% Processing: Wallpaper/bigcollection/_C...__Deutschland____R_diger_H.jpg\r" + ], + [ + 0.032432, + " 11.9% Processing: Wallpaper/bigcollection/_C...Toulouse__D_partement_Haut.jpg\r" + ], + [ + 0.032966, + " 12.0% Processing: Wallpaper/bigcollection/_C...pring__Germany____Boris_St.jpg\r" + ], + [ + 0.024881, + " 12.1% Processing: Wallpaper/bigcollection/_C...pring__Germany____Boris_St.jpg\r" + ], + [ + 0.02818, + " 12.2% Processing: Wallpaper/bigcollection/_C...Mallorca__Balearische_Inse.jpg\r" + ], + [ + 0.029353, + " 12.3% Processing: Wallpaper/bigcollection/_C...A__ESA__N._Smith__Universi.jpg\r" + ], + [ + 0.03626, + " 12.4% Processing: Wallpaper/bigcollection/_C...gebr_cke_bei_Ballintoy__Co.jpg\r" + ], + [ + 0.025838, + " 12.5% Processing: Wallpaper/bigcollection/_C...gebr_cke_bei_Ballintoy__Co.jpg\r" + ], + [ + 0.027176, + " 12.6% Processing: Wallpaper/bigcollection/_C...lona__Spanien____Nora_De_A.jpg\r" + ], + [ + 0.0298, + " 12.7% Processing: Wallpaper/bigcollection/_C...rcia__Nationalpark_Monti_S.jpg\r" + ], + [ + 0.027672, + " 12.8% Processing: Wallpaper/bigcollection/_C...vinz_Potenza__Italien____F.jpg\r" + ], + [ + 0.032259, + " 12.9% Processing: Wallpaper/bigcollection/_C...semite-Nationalpark__Kalif.jpg\r" + ], + [ + 0.031451, + " 13.0% Processing: Wallpaper/bigcollection/_C...um_Ludwig__Cologne__German.jpg\r" + ], + [ + 0.030096, + " 13.1% Processing: Wallpaper/bigcollection/_C..._Ludwig__Cologne__North_ 1.jpg\r" + ], + [ + 0.028235, + " 15.1% Processing: Wallpaper/bigcollection/_D...n_Hannover_bei_Nacht____Ma.jpg\r" + ], + [ + 0.028761, + " 15.2% Processing: Wallpaper/bigcollection/_D...rieb_befindliche_Opernhaus.jpg\r" + ], + [ + 0.027439, + " 15.3% Processing: Wallpaper/bigcollection/_D...esert_View_Watchtower__Gra.jpg\r" + ], + [ + 0.028598, + " 15.4% Processing: Wallpaper/bigcollection/_D..._Provinz_Shaanxi__Volksrep.jpg\r" + ], + [ + 0.031617, + " 15.5% Processing: Wallpaper/bigcollection/_D...gr__t_den_Hund_Sudo_in_Mel.jpg\r" + ], + [ + 0.032865, + " 17.5% Processing: Wallpaper/bigcollection/_D...s_du_Tarn__Nationalpark_Ce.jpg\r" + ], + [ + 0.031736, + " 17.6% Processing: Wallpaper/bigcollection/_D..._he_von_Sens__D_partement_.jpg\r" + ], + [ + 0.030474, + " 17.7% Processing: Wallpaper/bigcollection/_D...__Wales__Vereinigtes_K_nig.jpg\r" + ], + [ + 0.026112, + " 20.5% Processing: Wallpaper/bigcollection/_E...Junges_schnuppert_an_einer.jpg\r" + ], + [ + 0.027898, + " 20.6% Processing: Wallpaper/bigcollection/_E...chen_versteckt_sich_in_ein.jpg\r" + ], + [ + 0.027202, + " 20.7% Processing: Wallpaper/bigcollection/_E...r_Frosch_in_einem_Wassertr.jpg\r" + ], + [ + 0.027615, + " 20.8% Processing: Wallpaper/bigcollection/_E...ekorierter_Saguaro-Kaktus_.jpg\r" + ], + [ + 0.028446, + " 20.9% Processing: Wallpaper/bigcollection/_E...e__berquert_den_Luangwa-Fl.jpg\r" + ], + [ + 0.031808, + " 21.0% Processing: Wallpaper/bigcollection/_E...ngstunnel_zur_Felsenkirche.jpg\r" + ], + [ + 0.031065, + " 22.7% Processing: Wallpaper/bigcollection/_E...n_Koblenz_and_Trier__Germa.jpg\r" + ], + [ + 0.033059, + " 22.8% Processing: Wallpaper/bigcollection/_E...n_Angola_und_Namibia____Fr.jpg\r" + ], + [ + 0.035115, + " 22.9% Processing: Wallpaper/bigcollection/_E...r_Olympischen_Spiele_1896_.jpg\r" + ], + [ + 0.032507, + " 23.0% Processing: Wallpaper/bigcollection/_E..._Fr_hlingskrokus__Almwiese.jpg\r" + ], + [ + 0.028219, + " 23.1% Processing: Wallpaper/bigcollection/_E...in_der_Meeresbucht_Cathedr.jpg\r" + ], + [ + 0.029551, + " 23.2% Processing: Wallpaper/bigcollection/_E..._Nationalpark_Bayerischer_.jpg\r" + ], + [ + 0.02746, + " 23.3% Processing: Wallpaper/bigcollection/_E...im_Nationalpark_Bayerische.jpg\r" + ], + [ + 0.028081, + " 23.4% Processing: Wallpaper/bigcollection/_E...im__umava-Nationalpark__Ts.jpg\r" + ], + [ + 0.027796, + " 23.5% Processing: Wallpaper/bigcollection/_E..._Emsland__Germany____Erh 1.jpg\r" + ], + [ + 0.026053, + " 25.4% Processing: Wallpaper/bigcollection/_F...chersee_J_kuls_rl_n__Islan.jpg\r" + ], + [ + 0.029312, + " 25.5% Processing: Wallpaper/bigcollection/_F...Yellowstone_Nationalpark__.jpg\r" + ], + [ + 0.029189, + " 25.6% Processing: Wallpaper/bigcollection/_F...yi__Provinz_Phang-nga__Tha.jpg\r" + ], + [ + 0.029535, + " 25.7% Processing: Wallpaper/bigcollection/_F..._Tree_River__Kitikmeot_Reg.jpg\r" + ], + [ + 0.031935, + " 25.8% Processing: Wallpaper/bigcollection/_F...ystad__Niederlande____Erns.jpg\r" + ], + [ + 0.034076, + " 25.9% Processing: Wallpaper/bigcollection/_F...kyline_von_Baku__Aserbaids.jpg\r" + ], + [ + 0.028655, + " 26.0% Processing: Wallpaper/bigcollection/_F..._New_York_City__Bundesstaa.jpg\r" + ], + [ + 0.030152, + " 26.1% Processing: Wallpaper/bigcollection/_F...wals__Cierva_Cove__Antarkt.jpg\r" + ], + [ + 0.030983, + " 26.2% Processing: Wallpaper/bigcollection/_F..._des_Norman_River__Queensl.jpg\r" + ], + [ + 0.027019, + " 27.4% Processing: Wallpaper/bigcollection/_G..._Ger_llhang_im_Rondane-Nat.jpg\r" + ], + [ + 0.027058, + " 27.5% Processing: Wallpaper/bigcollection/_G...tzgebiet_Sacramento_Nation.jpg\r" + ], + [ + 0.038515, + " 27.6% Processing: Wallpaper/bigcollection/_G...Villandry__Loiretal__Frank.jpg\r" + ], + [ + 0.024219, + " 27.7% Processing: Wallpaper/bigcollection/_G...Villandry__Loiretal__Frank.jpg\r" + ], + [ + 0.028063, + " 27.8% Processing: Wallpaper/bigcollection/_G...__Champion-Insel__Floreana.jpg\r" + ], + [ + 0.030237, + " 27.9% Processing: Wallpaper/bigcollection/_G...__R_bida__Gal_pagosinseln_.jpg\r" + ], + [ + 0.031455, + " 28.0% Processing: Wallpaper/bigcollection/_G...c-Nationalpark__Alaska__US.jpg\r" + ], + [ + 0.028409, + " 28.1% Processing: Wallpaper/bigcollection/_G...um_Bridge__Newcastle_upon_.jpg\r" + ], + [ + 0.031595, + " 28.2% Processing: Wallpaper/bigcollection/_G..._Kanal_in_Venedig__Italien.jpg\r" + ], + [ + 0.031079, + " 28.3% Processing: Wallpaper/bigcollection/_G..._Rock_Canyon__Waterton-Lak.jpg\r" + ], + [ + 0.028272, + " 30.5% Processing: Wallpaper/bigcollection/_G...oos_in_der_Gro_aufnahme___.jpg\r" + ], + [ + 0.034208, + " 30.6% Processing: Wallpaper/bigcollection/_G...iesel__Bayern__Deutschland.jpg\r" + ], + [ + 0.034016, + " 30.7% Processing: Wallpaper/bigcollection/_G...__ber_dem_Thunersee__Berne.jpg\r" + ], + [ + 0.0292, + " 30.8% Processing: Wallpaper/bigcollection/_G...ell-St.-Elias-Nationalpark.jpg\r" + ], + [ + 0.024942, + " 32.8% Processing: Wallpaper/bigcollection/_G..._bei_Mettlach__Saarland__D.jpg\r" + ], + [ + 0.031677, + " 32.9% Processing: Wallpaper/bigcollection/_G...ngxia-Zuchtstation__Ya_an_.jpg\r" + ], + [ + 0.031108, + " 33.9% Processing: Wallpaper/bigcollection/_H...kaido__Japan____JTB_Media_.jpg\r" + ], + [ + 0.030964, + " 35.9% Processing: Wallpaper/bigcollection/_H...ew_RussellVisuals_Unlimite.jpg\r" + ], + [ + 0.026577, + " 38.9% Processing: Wallpaper/bigcollection/_I...eutschen_Doms__Gendarmenma.jpg\r" + ], + [ + 0.031898, + " 39.0% Processing: Wallpaper/bigcollection/_I...ukuoka_Tower__Fukuoka__Jap.jpg\r" + ], + [ + 0.031693, + " 39.1% Processing: Wallpaper/bigcollection/_I...__Bermeo__Provinz_Bizkaia_.jpg\r" + ], + [ + 0.026825, + " 39.2% Processing: Wallpaper/bigcollection/_I...P_tzcuaro-See__Bundesstaat.jpg\r" + ], + [ + 0.030749, + " 41.0% Processing: Wallpaper/bigcollection/_J...ia-Nationalpark__Maine__US.jpg\r" + ], + [ + 0.032301, + " 41.1% Processing: Wallpaper/bigcollection/_J..._im_Moremi_Game_Reserve__O.jpg\r" + ], + [ + 0.031689, + " 42.2% Processing: Wallpaper/bigcollection/_K...n_in_der_Antarktis____Jan_.jpg\r" + ], + [ + 0.029222, + " 42.3% Processing: Wallpaper/bigcollection/_K...e_Washington__Antarktis___.jpg\r" + ], + [ + 0.174039, + " 42.4% Processing: Wallpaper/bigcollection/_K...K_ken__Snow_Hill_Island__A.jpg\r" + ], + [ + 0.03322, + " 42.5% Processing: Wallpaper/bigcollection/_K...SCO-Welterbest_tte__Trier_.jpg\r" + ], + [ + 0.031657, + " 43.4% Processing: Wallpaper/bigcollection/_K...ufort__South_Carolina__USA.jpg\r" + ], + [ + 0.026738, + " 43.5% Processing: Wallpaper/bigcollection/_K...chfang_vor_Port_St._Johns_.jpg\r" + ], + [ + 0.033834, + " 44.4% Processing: Wallpaper/bigcollection/_K...eide__Schottisches_Hochlan.jpg\r" + ], + [ + 0.034061, + " 44.5% Processing: Wallpaper/bigcollection/_K...m_Schlossgarten_Schwetzing.jpg\r" + ], + [ + 0.033845, + " 44.6% Processing: Wallpaper/bigcollection/_K...dscha__Kachetien__Georgien.jpg\r" + ], + [ + 0.031383, + " 44.7% Processing: Wallpaper/bigcollection/_K..._Baden-W_rttemberg__Deutsc.jpg\r" + ], + [ + 0.027515, + " 44.8% Processing: Wallpaper/bigcollection/_K..._Zanskar__Region_Ladakh__B.jpg\r" + ], + [ + 0.031935, + " 44.9% Processing: Wallpaper/bigcollection/_K...Meteora__Griechenland____S.jpg\r" + ], + [ + 0.030994, + " 45.0% Processing: Wallpaper/bigcollection/_K...-Ville__Belgien____Patty_P.jpg\r" + ], + [ + 0.031632, + " 46.8% Processing: Wallpaper/bigcollection/_L...ionalpark__Simbabwe____Jer.jpg\r" + ], + [ + 0.032645, + " 46.9% Processing: Wallpaper/bigcollection/_L...Hochland_von_Cuenca__Auton.jpg\r" + ], + [ + 0.028682, + " 47.0% Processing: Wallpaper/bigcollection/_L...Hochland_von_Cuenca__Auton.jpg\r" + ], + [ + 0.030087, + " 47.1% Processing: Wallpaper/bigcollection/_L...__Axel_Flasbarth500px_____.jpg\r" + ], + [ + 0.030684, + " 47.2% Processing: Wallpaper/bigcollection/_L...athedrale_von_Chartres__Fr.jpg\r" + ], + [ + 0.029522, + " 47.3% Processing: Wallpaper/bigcollection/_L...und_Aiguilles_de_Chamonix_.jpg\r" + ], + [ + 0.032174, + " 47.4% Processing: Wallpaper/bigcollection/_L...Nutthavood_Punpeng500px___.jpg\r" + ], + [ + 0.029075, + " 47.5% Processing: Wallpaper/bigcollection/_L...nd__Great_Barrier_Reef__Au.jpg\r" + ], + [ + 0.028973, + " 47.6% Processing: Wallpaper/bigcollection/_L...__Insel_Corvo__Portugal___.jpg\r" + ], + [ + 0.030047, + " 47.7% Processing: Wallpaper/bigcollection/_L...ationalpark__British_Colum.jpg\r" + ], + [ + 0.031497, + " 49.3% Processing: Wallpaper/bigcollection/_L...hof__Great_Court__des_Brit.jpg\r" + ], + [ + 0.029466, + " 49.4% Processing: Wallpaper/bigcollection/_L...em_Wald_auf_der_Insel_Shik.jpg\r" + ], + [ + 0.025178, + " 49.5% Processing: Wallpaper/bigcollection/_L...er_K_ste_von_Ixtapa_Zihuat.jpg\r" + ], + [ + 0.030228, + " 49.6% Processing: Wallpaper/bigcollection/_L...e_Itapu__in_Salvador__Bahi.jpg\r" + ], + [ + 0.027644, + " 49.7% Processing: Wallpaper/bigcollection/_L...l_Point_in_der_N_he_von_Po.jpg\r" + ], + [ + 0.026513, + " 49.8% Processing: Wallpaper/bigcollection/_L...eversand__Westerhever__Sch.jpg\r" + ], + [ + 0.032316, + " 49.9% Processing: Wallpaper/bigcollection/_L...i__Provinz_Jiangsu__Volksr.jpg\r" + ], + [ + 0.026983, + " 50.0% Processing: Wallpaper/bigcollection/_L...g__aufgenommen_von_der_Int.jpg\r" + ], + [ + 0.03107, + " 51.7% Processing: Wallpaper/bigcollection/_M..._Cay__Exuma__Bahamas____Ji.jpg\r" + ], + [ + 0.028123, + " 51.8% Processing: Wallpaper/bigcollection/_M...ationalpark_Jardines_de_la.jpg\r" + ], + [ + 0.028547, + " 51.9% Processing: Wallpaper/bigcollection/_M...au____WaterFrameAlamy_____.jpg\r" + ], + [ + 0.030092, + " 53.1% Processing: Wallpaper/bigcollection/_M...ands-Nationalpark__Utah__U.jpg\r" + ], + [ + 0.027589, + " 53.2% Processing: Wallpaper/bigcollection/_M...useum_in_den_Wolken__Monte.jpg\r" + ], + [ + 0.029779, + " 53.3% Processing: Wallpaper/bigcollection/_M...Plaza_de_la_Encarnaci_n__S.jpg\r" + ], + [ + 0.031154, + " 54.6% Processing: Wallpaper/bigcollection/_M...lmie_National_Forest__Bund.jpg\r" + ], + [ + 0.03317, + " 54.7% Processing: Wallpaper/bigcollection/_M...t_Edziza_Provincial_Park__.jpg\r" + ], + [ + 0.031631, + " 54.8% Processing: Wallpaper/bigcollection/_M...__Washington__USA____Diane.jpg\r" + ], + [ + 0.025722, + " 56.1% Processing: Wallpaper/bigcollection/_N..._K_ste_des_Atlantischen_Oz.jpg\r" + ], + [ + 0.029888, + " 56.2% Processing: Wallpaper/bigcollection/_N...hee__Schiras__Iran____R.Cr.jpg\r" + ], + [ + 0.022761, + " 57.5% Processing: Wallpaper/bigcollection/_N...Fischotter_im_Yellowstone-.jpg\r" + ], + [ + 0.030469, + " 57.6% Processing: Wallpaper/bigcollection/_N..._Baumstachler____Minden_Pi.jpg\r" + ], + [ + 0.032258, + " 58.9% Processing: Wallpaper/bigcollection/_O...-Park__Bomarzo__Italien___.jpg\r" + ], + [ + 0.028556, + " 59.0% Processing: Wallpaper/bigcollection/_O...-Park__Bomarzo__Italien___.jpg\r" + ], + [ + 0.029665, + " 60.4% Processing: Wallpaper/bigcollection/_P..._der_Boardman_Tree_Farm__B.jpg\r" + ], + [ + 0.030072, + " 60.5% Processing: Wallpaper/bigcollection/_P...o-Ebene__Italien____Eddy_G.jpg\r" + ], + [ + 0.034601, + " 60.6% Processing: Wallpaper/bigcollection/_P...nem_Karnevalswagen_beim_Ro.jpg\r" + ], + [ + 0.029305, + " 61.9% Processing: Wallpaper/bigcollection/_P...der_argentinischen_Atlanti.jpg\r" + ], + [ + 0.03045, + " 62.0% Processing: Wallpaper/bigcollection/_P...m__Pilsum__Niedersachsen__.jpg\r" + ], + [ + 0.02941, + " 63.4% Processing: Wallpaper/bigcollection/_P...rk_Torres_del_Paine__Chile.jpg\r" + ], + [ + 0.033345, + " 63.5% Processing: Wallpaper/bigcollection/_P...i-Nationalpark__New_South_.jpg\r" + ], + [ + 0.031818, + " 64.9% Processing: Wallpaper/bigcollection/_R...ationalpark_Sarek__Schwede.jpg\r" + ], + [ + 0.025656, + " 65.0% Processing: Wallpaper/bigcollection/_R...ationalpark_Sarek__Schwede.jpg\r" + ], + [ + 0.030751, + " 66.6% Processing: Wallpaper/bigcollection/_R...nyang__Provinz_Yunnan__Chi.jpg\r" + ], + [ + 0.030313, + " 66.7% Processing: Wallpaper/bigcollection/_R...n_Ludwig_XIV._auf_dem_Plac.jpg\r" + ], + [ + 0.032915, + " 68.6% Processing: Wallpaper/bigcollection/_R...r____Getty_Images______Bin.jpg\r" + ], + [ + 0.029504, + " 70.1% Processing: Wallpaper/bigcollection/_S...tional_Park__Germany____ 3.jpg\r" + ], + [ + 0.026571, + " 70.2% Processing: Wallpaper/bigcollection/_S...tional_Park__Germany____ 4.jpg\r" + ], + [ + 0.032136, + " 71.7% Processing: Wallpaper/bigcollection/_S...e_t_sich_als_Wasserfall_vo.jpg\r" + ], + [ + 0.032883, + " 72.6% Processing: Wallpaper/bigcollection/_S...riehunde_im_Wind_Cave_Nati.jpg\r" + ], + [ + 0.031602, + " 72.7% Processing: Wallpaper/bigcollection/_S...erkstatt__Hexenlochm_hle__.jpg\r" + ], + [ + 0.030634, + " 73.6% Processing: Wallpaper/bigcollection/_S...en_in_der_Son_Doong-H_hle_.jpg\r" + ], + [ + 0.027026, + " 74.5% Processing: Wallpaper/bigcollection/_S..._at_sunset__Attendorn__Sau.jpg\r" + ], + [ + 0.038777, + " 75.4% Processing: Wallpaper/bigcollection/_S..._Dartmoor-Nationalpark__De.jpg\r" + ], + [ + 0.027422, + " 75.5% Processing: Wallpaper/bigcollection/_S..._der_Halong-Bucht__Vietnam.jpg\r" + ], + [ + 0.027539, + " 76.3% Processing: Wallpaper/bigcollection/_S...em_See__Bergpark_Wilhelmsh.jpg\r" + ], + [ + 0.031058, + " 76.4% Processing: Wallpaper/bigcollection/_S...ge_in_den_Ausl_ufern_der_R.jpg\r" + ], + + [ + 0.036506, + " 77.6% Processing: Wallpaper/bigcollection/_S..._Geothermalgebiet_Haukadal.jpg\r" + ], + [ + 0.025063, + " 77.7% Processing: Wallpaper/bigcollection/_S...ampagne-Ardennes__Frankrei.jpg\r" + ], + [ + 0.029054, + " 77.8% Processing: Wallpaper/bigcollection/_S...r__ber_West_Point__Nebrask.jpg\r" + ], + [ + 0.028908, + " 77.9% Processing: Wallpaper/bigcollection/_S...n-Bodenstation__Longyearby.jpg\r" + ], + [ + 0.029276, + " 78.0% Processing: Wallpaper/bigcollection/_S..._und_Solidarit_t_Kerzen__K.jpg\r" + ], + [ + 0.024812, + " 79.0% Processing: Wallpaper/bigcollection/_T..._Blatt_eines_Per_ckenstrau.jpg\r" + ], + [ + 0.031898, + " 80.0% Processing: Wallpaper/bigcollection/_T...er_Bavaria__Germany____F 3.jpg\r" + ], + [ + 0.029189, + " 80.1% Processing: Wallpaper/bigcollection/_T...pper_Bavaria__Germany____F.jpg\r" + ], + [ + 0.028065, + " 81.2% Processing: Wallpaper/bigcollection/_T...Image_BrokerRex_Features 1.jpg\r" + ], + [ + 0.03116, + " 81.3% Processing: Wallpaper/bigcollection/_T...n__Baden-W_rttemberg__Deut.jpg\r" + ], + [ + 0.026524, + " 82.3% Processing: Wallpaper/bigcollection/_U..._seltene_Blattschwanzgecko.jpg\r" + ], + [ + 0.028383, + " 82.4% Processing: Wallpaper/bigcollection/_V...en_und_Altstadt_von_Chania.jpg\r" + ], + [ + 0.032476, + " 83.5% Processing: Wallpaper/bigcollection/_V..._Hirta__St._Kilda__Schottl.jpg\r" + ], + [ + 0.030701, + " 84.7% Processing: Wallpaper/bigcollection/_W...wald__Insel_Sula__Solund__.jpg\r" + ], + [ + 0.034129, + " 84.8% Processing: Wallpaper/bigcollection/_W...nschafe__Kanton_Wallis__ 1.jpg\r" + ], + [ + 0.03033, + " 85.8% Processing: Wallpaper/bigcollection/_W...ionalpark_Plitvicer_Seen__.jpg\r" + ], + + [ + 0.031761, + " 87.2% Processing: Wallpaper/bigcollection/_W..._N_he_von_Ca_amares__Provi.jpg\r" + ], + [ + 0.031627, + " 87.3% Processing: Wallpaper/bigcollection/_W..._N_he_von_Cuenca__Spanien_.jpg\r" + ], + [ + 0.024242, + " 88.3% Processing: Wallpaper/bigcollection/_W...jeu__D_partement_Rh_ne__Re.jpg\r" + ], + [ + 0.027362, + " 89.3% Processing: Wallpaper/bigcollection/_W...guna_Colorada__Bolivien___.jpg\r" + ], + [ + 0.031448, + " 90.5% Processing: Wallpaper/bigcollection/_Z..._von_Autobahnen_in_Bangkok.jpg\r" + ], + [ + 0.027535, + " 90.6% Processing: Wallpaper/bigcollection/_Z...abara-Bucht__Rio_de_Janeir.jpg\r" + ], + [ + 0.025329, + " 92.1% Processing: Wallpaper/bigcollection/__...ptur_der_Landart-K_nstleri.jpg\r" + ], + [ + 0.044106, + " 92.2% Processing: Wallpaper/bigcollection/__...__Magic_Mountain_-Landmark.jpg\r" + ], + [ + 0.03068, + " 93.5% Processing: Wallpaper/bigcollection/_F...rte_Marina_Bay_zum_50._Nat.jpg\r" + ], + [ + 0.031039, + " 93.6% Processing: Wallpaper/bigcollection/_H...ing_Crane_Pond_Conservancy.jpg\r" + ], + [ + 0.020685, + " 95.0% Processing: Wallpaper/2048example/Palo...t_by_Beth___Jeremy_Jonkman.jpg\r" + ], + [ + 0.019863, + " 96.3% Processing: Wallpaper/evenmore/ChipDE ...jpg \r" + ], + [ + 0.056069, + " 96.4% Processing: Wallpaper/evenmore/ChipDE ...jpg \r" + ], + [ + 0.049869, + " 97.4% Processing: Wallpaper/evenmore/ChipDE 06.jpg \r" + ], + [ + 0.021021, + " 97.5% Processing: Wallpaper/evenmore/ChipDE ...jpg \r" + ], + [ + 0.019135, + " 98.4% Processing: Wallpaper/evenmore/ChipDE ...jpg \r" + ], + [ + 0.021483, + " 99.6% Processing: Wallpaper/deer.jpg ... \r" + ], + [ + 0.021593, + " 99.7% Processing: Wallpaper/deer.jpg ... \r" + ], + [ + 0.02037, + " 99.8% Processing: Wallpaper/deer.jpg ... \r" + ], + [ + 0.027858, + " 99.9% Processing: Wallpaper/deer.jpg ... \r" + ], + [ + 0.020864, + " \r" + ], + [ + 0.077955, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001068, + "\u001b]7;\u0007" + ], + [ + 0.000836, + "\u001b]7;\u0007" + ], + [ + 0.000104, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 8.3e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000234, + "\u001b[?2004h" + ], + [ + 2.471911, + "\u001b[32ml\u001b[39m" + ], + [ + 0.102688, + "\b\u001b[32ml\u001b[32ms\u001b[39m" + ], + [ + 0.272296, + " " + ], + [ + 0.220114, + "-" + ], + [ + 0.157165, + "l" + ], + [ + 0.074368, + "a" + ], + [ + 0.353976, + "\u001b[?1l\u001b>" + ], + [ + 0.000755, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000778, + "\u001b]2;ls --color=tty -la\u0007\u001b]1;ls\u0007" + ], + [ + 0.001633, + "total 573616\r\n" + ], + [ + 1.9e-05, + "drwxr-xr-x. 4 rugk rugk 4096 Jul 16 18:56 \u001b[0m\u001b[38;5;33m.\u001b[0m\r\ndrwxr-x---. 55 rugk rugk 4096 Jul 16 18:57 \u001b[38;5;33m..\u001b[0m\r\ndrwx------. 2 rugk rugk 4096 Jul 14 21:57 \u001b[38;5;33mWallpaper\u001b[0m\r\ndrwxr-xr-x. 6 rugk rugk 4096 Jul 14 21:55 \u001b[38;5;33mWallpaper.orig\u001b[0m\r\n-rw-------. 1 rugk rugk 587361454 Jul 16 18:57 \u001b[38;5;9mbackup.tar.gz\u001b[0m\r\n" + ], + [ + 0.000404, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001103, + "\u001b]7;\u0007" + ], + [ + 0.000992, + "\u001b]7;\u0007" + ], + [ + 7.3e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 8.9e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000158, + "\u001b[?2004h" + ], + [ + 3.04506, + "\u001b[?1l\u001b>" + ], + [ + 0.000385, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000485, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001512, + "\u001b]7;\u0007" + ], + [ + 0.001245, + "\u001b]7;\u0007" + ], + [ + 6.9e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000129, + "\u001b[?1h\u001b=" + ], + [ + 0.000247, + "\u001b[?2004h" + ], + [ + 0.325892, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.228892, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.186392, + "\b\b\u001b[1m#\u001b[1m \u001b[1mY\u001b[0m\u001b[39m" + ], + [ + 0.112073, + "\b\u001b[1mY\u001b[1mo\u001b[0m\u001b[39m" + ], + [ + 0.139024, + "\b\u001b[1mo\u001b[1mu\u001b[0m\u001b[39m" + ], + [ + 0.151793, + "\b\u001b[1mu\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.106484, + "\b\u001b[1m \u001b[1mc\u001b[0m\u001b[39m" + ], + [ + 0.147932, + "\b\u001b[1mc\u001b[1ma\u001b[0m\u001b[39m" + ], + [ + 0.181458, + "\b\u001b[1ma\u001b[1mn\u001b[0m\u001b[39m" + ], + [ + 0.137456, + "\b\u001b[1mn\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.21885, + "\b\u001b[1m \u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.170788, + "\b\u001b[1me\u001b[1mv\u001b[0m\u001b[39m" + ], + [ + 0.133285, + "\b\u001b[1mv\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.28717, + "\b\u001b[1me\u001b[1mn\u001b[0m\u001b[39m" + ], + [ + 0.485291, + "\b\u001b[1mn\u001b[1m mount an archive or even the whole repository:\u001b[0m\u001b[39m" + ], + [ + 1.036008, + "\u001b[?1l\u001b>" + ], + [ + 0.001535, + "\u001b[?2004l\r\r\n" + ], + [ + 0.001777, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.002934, + "\u001b]7;\u0007" + ], + [ + 0.002695, + "\u001b]7;\u0007" + ], + [ + 0.00014, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 9.4e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000881, + "\u001b[?2004h" + ], + [ + 1.264493, + "\u001b[1m\u001b[31mm\u001b[0m\u001b[39m" + ], + [ + 0.158178, + "\b\u001b[1m\u001b[31mm\u001b[1m\u001b[31mk\u001b[0m\u001b[39m" + ], + [ + 0.129344, + "\b\b\u001b[1m\u001b[31mm\u001b[1m\u001b[31mk\u001b[1m\u001b[31md\u001b[0m\u001b[39m" + ], + [ + 0.153746, + "\b\u001b[1m\u001b[31md\u001b[1m\u001b[31mi\u001b[0m\u001b[39m" + ], + [ + 0.106254, + "\b\b\b\b\u001b[0m\u001b[32mm\u001b[0m\u001b[32mk\u001b[0m\u001b[32md\u001b[0m\u001b[32mi\u001b[32mr\u001b[39m" + ], + [ + 0.178794, + " " + ], + [ + 0.328222, + "\u001b[4m/\u001b[24m" + ], + [ + 0.202794, + "\b\u001b[4m/\u001b[4mt\u001b[24m" + ], + [ + 0.246443, + "\b\u001b[4mt\u001b[4mm\u001b[24m" + ], + [ + 0.207634, + "\b\u001b[4mm\u001b[4mp\u001b[24m" + ], + [ + 0.88273, + "\b\u001b[4mp\u001b[4m/\u001b[24m" + ], + [ + 0.339887, + "\b\u001b[4m/\u001b[4mm\u001b[24m" + ], + [ + 0.210076, + "\b\u001b[4mm\u001b[4mo\u001b[24m" + ], + [ + 0.16667, + "\b\b\b\b\b\b\b\u001b[24m/\u001b[24mt\u001b[24mm\u001b[24mp\u001b[24m/\u001b[24mm\u001b[24mou" + ], + [ + 0.141564, + "n" + ], + [ + 0.184, + "t" + ], + [ + 1.4607, + "\u001b[?1l\u001b>" + ], + [ + 0.001306, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000794, + "\u001b]2;mkdir /tmp/mount\u0007" + ], + [ + 6.6e-05, + "\u001b]1;mkdir\u0007" + ], + [ + 0.00176, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.00142, + "\u001b]7;\u0007" + ], + [ + 0.001308, + "\u001b]7;\u0007" + ], + [ + 7.2e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ " + ], + [ + 1.3e-05, + "\u001b[K" + ], + [ + 9.1e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000273, + "\u001b[?2004h" + ], + [ + 1.09686, + "\u001b[4mb\u001b[24m" + ], + [ + 0.187046, + "\b\u001b[24m\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[0m\u001b[39m" + ], + [ + 0.10907, + "\b\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[1m\u001b[31mr\u001b[0m\u001b[39m" + ], + [ + 0.12414, + "\b\b\b\u001b[0m\u001b[32mb\u001b[0m\u001b[32mo\u001b[0m\u001b[32mr\u001b[32mg\u001b[39m" + ], + [ + 0.187573, + " " + ], + [ + 0.229364, + "m" + ], + [ + 0.195942, + "o" + ], + [ + 0.183861, + "u" + ], + [ + 0.138559, + "n" + ], + [ + 0.207537, + "t" + ], + [ + 1.01571, + " " + ], + [ + 0.55086, + ":" + ], + [ + 0.110713, + ":" + ], + [ + 0.27265, + " " + ], + [ + 0.462869, + "\u001b[4m/\u001b[24m" + ], + [ + 0.795464, + "\b\u001b[4m/\u001b[4mt\u001b[24m" + ], + [ + 0.295092, + "\b\u001b[4mt\u001b[4mm\u001b[24m" + ], + [ + 0.200509, + "\b\u001b[4mm\u001b[4mp\u001b[24m" + ], + [ + 0.878464, + "\b\u001b[4mp\u001b[4m/\u001b[24m" + ], + [ + 0.306666, + "\b\u001b[4m/\u001b[4mm\u001b[24m" + ], + [ + 0.24341, + "\b\u001b[4mm\u001b[4mo\u001b[24m" + ], + [ + 0.166203, + "\b\u001b[4mo\u001b[4mu\u001b[24m" + ], + [ + 0.138953, + "\b\u001b[4mu\u001b[4mn\u001b[24m" + ], + [ + 0.177723, + "\b\u001b[4mn\u001b[4mt\u001b[24m" + ], + [ + 1.371278, + "\u001b[?1l\u001b>" + ], + [ + 0.001184, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000603, + "\u001b]2;borg mount :: /tmp/mount\u0007\u001b]1;borg\u0007" + ], + [ + 0.651025, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001453, + "\u001b]7;\u0007" + ], + [ + 0.000984, + "\u001b]7;\u0007" + ], + [ + 7.4e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 2.6e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.0002, + "\u001b[?2004h" + ], + [ + 1.860515, + "\u001b[32ml\u001b[39m" + ], + [ + 0.107896, + "\b\u001b[32ml\u001b[32ms\u001b[39m" + ], + [ + 0.253911, + " " + ], + [ + 0.203092, + "-" + ], + [ + 0.178525, + "l" + ], + [ + 0.111795, + "a" + ], + [ + 0.200138, + " " + ], + [ + 0.353001, + "\u001b[4m/\u001b[24m" + ], + [ + 0.264827, + "\b\u001b[4m/\u001b[4mt\u001b[24m" + ], + [ + 0.205749, + "\b\u001b[4mt\u001b[4mm\u001b[24m" + ], + [ + 0.168679, + "\b\u001b[4mm\u001b[4mp\u001b[24m" + ], + [ + 0.016649, + "\b\b\b\b\u001b[24m/\u001b[24mt\u001b[24mm\u001b[24mp" + ], + [ + 0.712108, + "\b\b\b\b\u001b[4m/\u001b[4mt\u001b[4mm\u001b[4mp\u001b[24m \b" + ], + [ + 0.383057, + "\b\u001b[4mp\u001b[4m/\u001b[24m" + ], + [ + 0.159994, + "\b\u001b[4m/\u001b[4mm\u001b[24m" + ], + [ + 0.187645, + "\b\u001b[4mm\u001b[4mo\u001b[24m" + ], + [ + 0.168813, + "\b\u001b[4mo\u001b[4mu\u001b[24m" + ], + [ + 0.12933, + "\b\u001b[4mu\u001b[4mn\u001b[24m" + ], + [ + 0.421583, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.018359, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mls\u001b[39m -la \u001b[4m/tmp/mount\u001b[1m\u001b[4m/\u001b[0m\u001b[24m\u001b[K" + ], + [ + 0.602087, + "\b\b\u001b[4mt\u001b[24m\u001b[0m\u001b[24m \b" + ], + [ + 2.5e-05, + "\u001b[?1l\u001b>" + ], + [ + 0.000874, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000682, + "\u001b]2;ls --color=tty -la /tmp/mount\u0007\u001b]1;ls\u0007" + ], + [ + 0.002495, + "total 0\r\n" + ], + [ + 4.4e-05, + "drwxr-xr-x. 1 rugk rugk 0 Jul 16 18:58 \u001b[0m\u001b[38;5;33m.\u001b[0m\r\ndrwxrwxrwt. 27 root root 660 Jul 16 18:58 \u001b[48;5;10;38;5;16m..\u001b[0m\r\ndrwxr-xr-x. 1 rugk rugk 0 Jul 16 18:58 \u001b[38;5;33mbackup-block-device\u001b[0m\r\ndrwxr-xr-x. 1 rugk rugk 0 Jul 16 18:58 \u001b[38;5;33mbackup1\u001b[0m\r\ndrwxr-xr-x. 1 rugk rugk 0 Jul 16 18:58 \u001b[38;5;33mbackup2\u001b[0m\r\ndrwxr-xr-x. 1 rugk rugk 0 Jul 16 18:58 \u001b[38;5;33mbackup3\u001b[0m\r\ndrwxr-xr-x. 1 rugk rugk 0 Jul 16 18:58 \u001b[38;5;33mrugk-2017-07-16T18:51:34\u001b[0m\r\ndrwxr-xr-x. 1 rugk rugk 0 Jul 16 18:58 \u001b[38;5;33mrugk-2017-07-16T18:52:19\u001b[0m\r\n" + ], + [ + 0.000169, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.000984, + "\u001b]7;\u0007" + ], + [ + 0.00097, + "\u001b]7;\u0007" + ], + [ + 2.4e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000122, + "\u001b[?1h\u001b=" + ], + [ + 0.000251, + "\u001b[?2004h" + ], + [ + 0.482339, + "\u001b[4mb\u001b[24m" + ], + [ + 0.179808, + "\b\u001b[24m\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[0m\u001b[39m" + ], + [ + 0.105817, + "\b\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[1m\u001b[31mr\u001b[0m\u001b[39m" + ], + [ + 0.116974, + "\b\b\b\u001b[0m\u001b[32mb\u001b[0m\u001b[32mo\u001b[0m\u001b[32mr\u001b[32mg\u001b[39m" + ], + [ + 0.173004, + " " + ], + [ + 0.216291, + "u" + ], + [ + 0.168732, + "m" + ], + [ + 0.228004, + "o" + ], + [ + 0.19268, + "u" + ], + [ + 0.14303, + "n" + ], + [ + 0.175557, + "t" + ], + [ + 0.406596, + " " + ], + [ + 0.69215, + "\u001b[4m/\u001b[24m" + ], + [ + 0.242216, + "\b\u001b[4m/\u001b[4mt\u001b[24m" + ], + [ + 0.260453, + "\b\u001b[4mt\u001b[4mm\u001b[24m" + ], + [ + 0.2605, + "\b\u001b[4mm\u001b[4mp\u001b[24m" + ], + [ + 0.014483, + "\b\b\b\b\u001b[24m/\u001b[24mt\u001b[24mm\u001b[24mp" + ], + [ + 0.597766, + "\b\b\b\b\u001b[4m/\u001b[4mt\u001b[4mm\u001b[4mp\u001b[24m \b" + ], + [ + 0.482551, + "\b\u001b[4mp\u001b[4m/\u001b[24m" + ], + [ + 0.236361, + "\b\u001b[4m/\u001b[4mm\u001b[24m" + ], + [ + 0.212317, + "\b\u001b[4mm\u001b[4mo\u001b[24m" + ], + [ + 0.160611, + "\b\u001b[4mo\u001b[4mu\u001b[24m" + ], + [ + 0.142036, + "\b\u001b[4mu\u001b[4mn\u001b[24m" + ], + [ + 0.335664, + "\b\u001b[4mn\u001b[4mt\u001b[24m" + ], + [ + 1.159614, + "\u001b[?1l\u001b>" + ], + [ + 0.001057, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000642, + "\u001b]2;borg umount /tmp/mount\u0007\u001b]1;borg\u0007" + ], + [ + 0.596849, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.001387, + "\u001b]7;\u0007" + ], + [ + 0.001067, + "\u001b]7;\u0007" + ], + [ + 9.1e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 7.6e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000187, + "\u001b[?2004h" + ], + [ + 1.467084, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.264583, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.277487, + "\b\b\u001b[1m#\u001b[1m \u001b[1mT\u001b[0m\u001b[39m" + ], + [ + 0.12184, + "\b\u001b[1mT\u001b[1mh\u001b[0m\u001b[39m" + ], + [ + 0.103403, + "\b\u001b[1mh\u001b[1ma\u001b[0m\u001b[39m" + ], + [ + 0.125651, + "\b\u001b[1ma\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.465663, + "\b\u001b[1mt\u001b[1m'\u001b[0m\u001b[39m" + ], + [ + 0.298764, + "\b\u001b[1m'\u001b[1ms it, but of course there is more to explore, so have a look at the d\u001b[1mo\u001b[1mcs.\u001b[0m\u001b[39m\u001b[K\r\r\n\u001b[K\u001b[A\u001b[4C" + ], + [ + 1.453815, + "\u001b[1B\r\u001b[K\u001b[A\u001b[4C" + ], + [ + 2e-05, + "\u001b[?1l\u001b>" + ], + [ + 0.000725, + "\u001b[?2004l\u001b[1B\r\r\n" + ], + [ + 0.00054, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.00118, + "\u001b]7;\u0007" + ], + [ + 0.000909, + "\u001b]7;\u0007" + ], + [ + 7.8e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 3e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000524, + "\u001b[?2004h" + ], + [ + 1.74407, + "\u001b[?2004l\r\r\n" + ] + ] +} diff --git a/docs/misc/asciinema/advanced.sh b/docs/misc/asciinema/advanced.sh new file mode 100644 index 0000000000..e09d7bcb0c --- /dev/null +++ b/docs/misc/asciinema/advanced.sh @@ -0,0 +1,65 @@ +# For the pro users, here are some advanced features of borg, so you can impress your friends. ;) +# Note: This screencast was made with borg version 1.1.0 – older or newer borg versions may behave differently. + +# First of all, we can use several environment variables for borg. +# E.g. we do not want to type in our repo path and password again and again… +export BORG_REPO='/media/backup/borgdemo' +export BORG_PASSPHRASE='1234' +# Problem solved, borg will use this automatically… :) +# We'll use this right away… + +## ADVANCED CREATION ## + +# We can also use some placeholders in our archive name… +borg create --stats --progress --compression lz4 ::{user}-{now} Wallpaper +# Notice the backup name. + +# And we can put completely different data, with different backup settings, in our backup. It will be deduplicated, anyway: +borg create --stats --progress --compression zlib,6 --exclude ~/Downloads/big ::{user}-{now} ~/Downloads + +# Or let's backup a device via STDIN. +sudo dd if=/dev/loop0 bs=10M | borg create --progress --stats ::specialbackup - + +# Let's continue with some simple things: +## USEFUL COMMANDS ## +# You can show some information about an archive. You can even do it without needing to specify the archive name: +borg info :: --last 1 + +# So let's rename our last archive: +borg rename ::specialbackup backup-block-device + +borg info :: --last 1 + +# A very important step if you choose keyfile mode (where the keyfile is only saved locally) is to export your keyfile and possibly print it, etc. +borg key export :: --qr-code file.html # this creates a nice HTML, but when you want something simpler… +< remove comment > +< let there: borg check > --paper # this is a "manual input"-only backup (but it is also included in the --qr-code option) + +## MAINTENANCE ## +# Sometimes backups get broken or we want a regular "checkup" that everything is okay… +borg check -v :: + +# Next problem: Usually you do not have infinite disk space. So you may need to prune your archive… +# You can tune this in every detail. See the docs for details. Here only a simple example: +borg prune --list --keep-last 1 --dry-run +# When actually executing it in a script, you have to use it without the --dry-run option, of course. + +## RESTORE ## + +# When you want to see the diff between two archives use this command. +# E.g. what happened between the first two backups? +borg diff ::backup1 backup2 +# Ah, we added a file, right… + +# There are also other ways to extract the data. +# E.g. as a tar archive. +borg export-tar --progress ::backup2 backup.tar.gz +ls -l + +# You can mount an archive or even the whole repository: +mkdir /tmp/mount +borg mount :: /tmp/mount +ls -la /tmp/mount +borg umount /tmp/mount + +# That's it, but of course there is more to explore, so have a look at the docs. diff --git a/docs/misc/asciinema/basic.json b/docs/misc/asciinema/basic.json new file mode 100644 index 0000000000..9eaf89356b --- /dev/null +++ b/docs/misc/asciinema/basic.json @@ -0,0 +1,4862 @@ +{ + "version": 1, + "width": 78, + "height": 25, + "duration": 379.234504, + "command": null, + "title": null, + "env": { + "TERM": "xterm-256color", + "SHELL": "/bin/zsh" + }, + "stdout": [ + [ + 0.000155, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000133, + "\u001b[?1h\u001b=" + ], + [ + 0.000183, + "\u001b[?2004h" + ], + [ + 0.468833, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.413214, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.440799, + "\b\b\u001b[1m#\u001b[1m \u001b[1mH\u001b[0m\u001b[39m" + ], + [ + 0.155436, + "\b\u001b[1mH\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.153888, + "\b\u001b[1me\u001b[1mr\u001b[0m\u001b[39m" + ], + [ + 0.145046, + "\b\u001b[1mr\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.191005, + "\b\u001b[1me\u001b[1m you'll see some basic commands to start working with borg.\u001b[0m\u001b[39m" + ], + [ + 0.328571, + "\u001b[?1l\u001b>" + ], + [ + 0.000462, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000787, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 7.2e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 1.4e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.0003, + "\u001b[?2004h" + ], + [ + 0.553943, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.254153, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.205346, + "\b\b\u001b[1m#\u001b[1m \u001b[1mN\u001b[0m\u001b[39m" + ], + [ + 0.164037, + "\b\u001b[1mN\u001b[1mo\u001b[0m\u001b[39m" + ], + [ + 0.198817, + "\b\u001b[1mo\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.157487, + "\b\u001b[1mt\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.348855, + "\b\u001b[1me\u001b[1m:\u001b[0m\u001b[39m" + ], + [ + 0.308837, + "\b\u001b[1m:\u001b[1m This teaser screencast was made with borg version 1.1.0 – older or n\u001b[1me\u001b[1mwer borg versions may behave differently.\u001b[0m\u001b[39m\u001b[K" + ], + [ + 0.760183, + "\u001b[?1l\u001b>" + ], + [ + 0.001229, + "\u001b[?2004l\r\r\n" + ], + [ + 0.001043, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.000111, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 8.6e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000699, + "\u001b[?2004h" + ], + [ + 0.617302, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.269944, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.231147, + "\b\b\u001b[1m#\u001b[1m \u001b[1mB\u001b[0m\u001b[39m" + ], + [ + 0.157768, + "\b\u001b[1mB\u001b[1mu\u001b[0m\u001b[39m" + ], + [ + 0.145012, + "\b\u001b[1mu\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.360132, + "\b\u001b[1mt\u001b[1m let's start.\u001b[0m\u001b[39m" + ], + [ + 0.808076, + "\u001b[?1l\u001b>" + ], + [ + 0.000384, + "\u001b[?2004l\r\r\n" + ], + [ + 0.001063, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 1e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 8.2e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000271, + "\u001b[?2004h" + ], + [ + 1.213811, + "\u001b[?1l\u001b>" + ], + [ + 0.000271, + "\u001b[?2004l\r\r\n" + ], + [ + 0.001041, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 7e-06, + "\u001b]1;~/Pictures\u0007" + ], + [ + 4.2e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 5.3e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000167, + "\u001b[?2004h" + ], + [ + 0.326924, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.245919, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.173421, + "\b\b\u001b[1m#\u001b[1m \u001b[1mF\u001b[0m\u001b[39m" + ], + [ + 0.121947, + "\b\u001b[1mF\u001b[1mi\u001b[0m\u001b[39m" + ], + [ + 0.196316, + "\b\u001b[1mi\u001b[1mr\u001b[1ms\u001b[0m\u001b[39m" + ], + [ + 0.224037, + "\b\u001b[1ms\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.323925, + "\b\u001b[1mt\u001b[1m of all, you can always get help:\u001b[0m\u001b[39m" + ], + [ + 0.738987, + "\u001b[?1l\u001b>" + ], + [ + 0.000395, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000643, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.000107, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 3.8e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.00031, + "\u001b[?2004h" + ], + [ + 1.268663, + "\u001b[1m\u001b[31mb\u001b[0m\u001b[39m" + ], + [ + 0.19562, + "\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[0m\u001b[39m" + ], + [ + 0.100091, + "\b\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[1m\u001b[31mr\u001b[0m\u001b[39m" + ], + [ + 0.157538, + "\b\b\b\u001b[0m\u001b[32mb\u001b[0m\u001b[32mo\u001b[0m\u001b[32mr\u001b[32mg\u001b[39m" + ], + [ + 0.196595, + " " + ], + [ + 0.210071, + "h" + ], + [ + 0.124892, + "e" + ], + [ + 0.177906, + "l" + ], + [ + 0.121006, + "p" + ], + [ + 0.314487, + "\u001b[?1l\u001b>" + ], + [ + 0.000695, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000777, + "\u001b]2;borg help\u0007\u001b]1;borg\u0007" + ], + [ + 0.538908, + "usage: borg [-V] [-h] [--critical] [--error] [--warning] [--info] [--debug]\r\n [--debug-topic TOPIC] [-p] [--log-json] [--lock-wait N]\r\n [--show-version] [--show-rc] [--no-files-cache] [--umask M]\r\n [--remote-path PATH] [--remote-ratelimit rate]\r\n [--consider-part-files] [--debug-profile FILE]\r\n ...\r\n\r\nBorg - Deduplicated Backups\r\n\r\noptional arguments:\r\n -V, --version show version number and exit\r\n\r\nCommon options:\r\n -h, --help show this help message and exit\r\n --critical work on log level CRITICAL\r\n --error work on log level ERROR\r\n --warning work on log level WARNING (default)\r\n --info, -v, --verbose\r\n work on log level INFO\r\n --debug enable debug output, work on log level DEBUG\r\n --debug-topic TOPIC enable TOPIC debugging (can be specified multiple\r\n times). The logger path is borg.debug. if TOPIC\r\n " + ], + [ + 4.1e-05, + " is not fully qualified.\r\n -p, --progress show progress information\r\n --log-json Output one JSON object per log line instead of\r\n formatted text.\r\n --lock-wait N wait for the lock, but max. N seconds (default: 1).\r\n --show-version show/log the borg version\r\n --show-rc show/log the return code (rc)\r\n --no-files-cache do not load/update the file metadata cache used to\r\n detect unchanged files\r\n --umask M set umask to M (local and remote, default: 0077)\r\n --remote-path PATH use PATH as borg executable on the remote (default:\r\n \"borg\")\r\n --remote-ratelimit rate\r\n set remote network upload rate limit in kiByte/s\r\n (default: 0=unlimited)\r\n --consider-part-files\r\n treat part files like normal files (e.g. to\r\n list/extract them)\r\n --debug-profile FILE Write execution profile" + ], + [ + 1.6e-05, + " in Borg format into FILE. For\r\n local use a Python-compatible file can be generated by\r\n suffixing FILE with \".pyprof\".\r\n\r\nrequired arguments:\r\n \r\n serve start repository server process\r\n init initialize empty repository\r\n check verify repository\r\n key manage repository key\r\n change-passphrase change repository passphrase\r\n create create backup\r\n extract extract archive contents\r\n export-tar create tarball from archive\r\n diff find differences in archive contents\r\n rename rename archive\r\n delete delete archive\r\n list list archive or repository contents\r\n mount mount repository\r\n umount umount repository\r\n info show repository or archive information\r\n break-lock break repository and cache locks\r\n prune " + ], + [ + 2e-05, + " prune archives\r\n upgrade upgrade repository format\r\n recreate Re-create archives\r\n with-lock run user command with lock held\r\n debug debugging command (not intended for normal use)\r\n benchmark benchmark command\r\n" + ], + [ + 0.043747, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 8e-06, + "\u001b]1;~/Pictures\u0007" + ], + [ + 5.2e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 4.6e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000163, + "\u001b[?2004h" + ], + [ + 1.509225, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.593308, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.334291, + "\b\b\u001b[1m#\u001b[1m \u001b[1mT\u001b[0m\u001b[39m" + ], + [ + 0.170683, + "\b\u001b[1mT\u001b[1mh\u001b[0m\u001b[39m" + ], + [ + 0.07295, + "\b\u001b[1mh\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.184509, + "\b\u001b[1me\u001b[1ms\u001b[0m\u001b[39m" + ], + [ + 0.136032, + "\b\u001b[1ms\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.250718, + "\b\u001b[1me\u001b[1m are a lot of commands, so better we start with a few:\u001b[0m\u001b[39m" + ], + [ + 1.088446, + "\u001b[?1l\u001b>" + ], + [ + 0.000396, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000604, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.000101, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 4.5e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000296, + "\u001b[?2004h" + ], + [ + 0.921744, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.276219, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.204903, + "\b\b\u001b[1m#\u001b[1m \u001b[1mL\u001b[0m\u001b[39m" + ], + [ + 0.137064, + "\b\u001b[1mL\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.16386, + "\b\u001b[1me\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.340061, + "\b\u001b[1mt\u001b[1m'\u001b[0m\u001b[39m" + ], + [ + 0.115905, + "\b\u001b[1m'\u001b[1ms\u001b[0m\u001b[39m" + ], + [ + 0.213255, + "\b\u001b[1ms\u001b[1m create a repo on an external drive:\u001b[0m\u001b[39m" + ], + [ + 1.086717, + "\u001b[?1l\u001b>" + ], + [ + 0.000391, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000606, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.000133, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 7.5e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000274, + "\u001b[?2004h" + ], + [ + 1.935612, + "\u001b[1m\u001b[31mb\u001b[0m\u001b[39m" + ], + [ + 0.184978, + "\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[0m\u001b[39m" + ], + [ + 0.115803, + "\b\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[1m\u001b[31mr\u001b[0m\u001b[39m" + ], + [ + 0.134282, + "\b\b\b\u001b[0m\u001b[32mb\u001b[0m\u001b[32mo\u001b[0m\u001b[32mr\u001b[32mg\u001b[39m" + ], + [ + 0.266061, + " " + ], + [ + 0.599046, + "i" + ], + [ + 0.183493, + "n" + ], + [ + 0.181453, + "i" + ], + [ + 0.258375, + "t" + ], + [ + 0.712329, + " " + ], + [ + 0.381053, + "" + ], + [ + 0.381053, + "-" + ], + [ + 0.119206, + "-" + ], + [ + 0.18993, + "e" + ], + [ + 0.175168, + "n" + ], + [ + 0.258977, + "c" + ], + [ + 0.139364, + "r" + ], + [ + 0.111012, + "y" + ], + [ + 0.55406, + "p" + ], + [ + 0.261667, + "t" + ], + [ + 0.284611, + "i" + ], + [ + 0.142087, + "o" + ], + [ + 0.195185, + "n" + ], + [ + 0.23882, + "=" + ], + [ + 0.31059, + "r" + ], + [ + 0.151355, + "e" + ], + [ + 0.165925, + "p" + ], + [ + 0.132833, + "o" + ], + [ + 0.253402, + "k" + ], + [ + 0.174711, + "e" + ], + [ + 0.245888, + "y" + ], + [ + 0.759586, + " " + ], + [ + 0.383355, + "\u001b[4m/\u001b[24m" + ], + [ + 0.189694, + "\b\u001b[4m/\u001b[4mm\u001b[24m" + ], + [ + 0.16364, + "\b\u001b[4mm\u001b[4me\u001b[24m" + ], + [ + 0.151451, + "\b\u001b[4me\u001b[4md\u001b[24m" + ], + [ + 0.239109, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.006487, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mborg\u001b[39m init --encryption=repokey \u001b[4m/media\u001b[1m\u001b[4m/\u001b[0m\u001b[24m\u001b[K" + ], + [ + 0.268216, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.003429, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mborg\u001b[39m init --encryption=repokey \u001b[4m/media/backup\u001b[1m\u001b[4m/\u001b[0m\u001b[24m\u001b[K" + ], + [ + 0.232352, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.003575, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mborg\u001b[39m init --encryption=repokey \u001b[4m/media/backup/borgdemo\u001b[1m\u001b[4m/\u001b[0m\u001b[24m\u001b[K" + ], + [ + 0.492094, + "\b\b\u001b[4mo\u001b[24m\u001b[0m\u001b[24m \b" + ], + [ + 0.748712, + "\u001b[?1l\u001b>" + ], + [ + 0.001017, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000712, + "\u001b]2;borg init --encryption=repokey /media/backup/borgdemo\u0007\u001b]1;borg\u0007" + ], + [ + 0.548105, + "Enter new passphrase: " + ], + [ + 2.119749, + "\r\n" + ], + [ + 0.000155, + "Enter same passphrase again: " + ], + [ + 1.606761, + "\r\n" + ], + [ + 5.8e-05, + "Do you want your passphrase to be displayed for verification? [yN]: " + ], + [ + 0.901237, + "\r\n" + ], + [ + 0.362453, + "\r\nBy default repositories initialized with this version will produce security\r\nerrors if written to with an older version (up to and including Borg 1.0.8).\r\n\r\nIf you want to use these older versions, you can disable the check by running:\r\nborg upgrade --disable-tam '/media/backup/borgdemo'\r\n\r\nSee https://borgbackup.readthedocs.io/en/stable/changes.html#pre-1-0-9-manifest-spoofing-vulnerability for details about the security implications.\r\n" + ], + [ + 0.050488, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 5e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 5.9e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000166, + "\u001b[?2004h" + ], + [ + 2.49308, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.308744, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.256774, + "\b\b\u001b[1m#\u001b[1m \u001b[1mT\u001b[0m\u001b[39m" + ], + [ + 0.157732, + "\b\u001b[1mT\u001b[1mh\u001b[0m\u001b[39m" + ], + [ + 0.127107, + "\b\u001b[1mh\u001b[1mi\u001b[0m\u001b[39m" + ], + [ + 0.178449, + "\b\u001b[1mi\u001b[1ms\u001b[0m\u001b[39m" + ], + [ + 0.179372, + "\b\u001b[1ms\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.383584, + "\b\u001b[1m \u001b[1mu\u001b[0m\u001b[39m" + ], + [ + 0.103361, + "\b\u001b[1mu\u001b[1ms\u001b[0m\u001b[39m" + ], + [ + 0.155066, + "\b\u001b[1ms\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.133308, + "\b\u001b[1me\u001b[1ms\u001b[0m\u001b[39m" + ], + [ + 0.23615, + "\b\u001b[1ms\u001b[1m the repokey encryption. You may look at \"borg help init\" or the \u001b[1mo\u001b[1mnline doc at https://borgbackup.readthedocs.io/ for other modes.\u001b[0m\u001b[39m\u001b[K" + ], + [ + 1.159159, + "\u001b[?1l\u001b>" + ], + [ + 0.0004, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000738, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.000111, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 8.5e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000286, + "\u001b[?2004h" + ], + [ + 1.645569, + "\u001b[?1l\u001b>" + ], + [ + 0.000452, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000619, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.0001, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 7.3e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000161, + "\u001b[?2004h" + ], + [ + 1.17234, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.575706, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.205759, + "\b\b\u001b[1m#\u001b[1m \u001b[1mS\u001b[0m\u001b[39m" + ], + [ + 0.343517, + "\b\u001b[1mS\u001b[1mo\u001b[0m\u001b[39m" + ], + [ + 0.245497, + "\b\u001b[1mo\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.218486, + "\b\u001b[1m \u001b[1mn\u001b[0m\u001b[39m" + ], + [ + 0.171258, + "\b\u001b[1mn\u001b[1mo\u001b[0m\u001b[39m" + ], + [ + 0.146364, + "\b\u001b[1mo\u001b[1mw\u001b[0m\u001b[39m" + ], + [ + 0.25775, + "\b\u001b[1mw\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.271708, + "\b\b\u001b[1mw\u001b[0m\u001b[39m\u001b[0m\u001b[39m \b" + ], + [ + 0.213838, + "\b\u001b[1mw\u001b[1m,\u001b[0m\u001b[39m" + ], + [ + 0.422324, + "\b\u001b[1m,\u001b[1m let's create our first (compressed) backup.\u001b[0m\u001b[39m" + ], + [ + 0.561514, + "\u001b[?1l\u001b>" + ], + [ + 0.000855, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000773, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 4.8e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 6.2e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000411, + "\u001b[?2004h" + ], + [ + 1.326196, + "\u001b[1m\u001b[31mb\u001b[0m\u001b[39m" + ], + [ + 0.191851, + "\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[0m\u001b[39m" + ], + [ + 0.136657, + "\b\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[1m\u001b[31mr\u001b[0m\u001b[39m" + ], + [ + 0.142499, + "\b\b\b\u001b[0m\u001b[32mb\u001b[0m\u001b[32mo\u001b[0m\u001b[32mr\u001b[32mg\u001b[39m" + ], + [ + 0.173217, + " " + ], + [ + 0.294445, + "c" + ], + [ + 0.200519, + "r" + ], + [ + 0.153078, + "e" + ], + [ + 0.133383, + "a" + ], + [ + 0.12891, + "t" + ], + [ + 0.151491, + "e" + ], + [ + 0.728709, + " " + ], + [ + 0.592118, + "-" + ], + [ + 0.118108, + "-" + ], + [ + 0.277349, + "s" + ], + [ + 0.134588, + "t" + ], + [ + 0.148057, + "a" + ], + [ + 0.090202, + "t" + ], + [ + 0.150971, + "s" + ], + [ + 0.307217, + " " + ], + [ + 0.481688, + "-" + ], + [ + 0.112243, + "-" + ], + [ + 0.234317, + "p" + ], + [ + 0.12453, + "r" + ], + [ + 0.116446, + "o" + ], + [ + 0.213657, + "g" + ], + [ + 0.12239, + "r" + ], + [ + 0.165156, + "e" + ], + [ + 0.256082, + "s" + ], + [ + 0.175158, + "s" + ], + [ + 0.302493, + " " + ], + [ + 0.490303, + "-" + ], + [ + 0.117279, + "-" + ], + [ + 0.130499, + "c" + ], + [ + 0.146261, + "o" + ], + [ + 0.139848, + "m" + ], + [ + 0.156108, + "p" + ], + [ + 0.190058, + "r" + ], + [ + 0.166862, + "e" + ], + [ + 0.261225, + "s" + ], + [ + 0.157133, + "s" + ], + [ + 0.281205, + "i" + ], + [ + 0.142487, + "o" + ], + [ + 0.179023, + "n" + ], + [ + 0.854723, + " " + ], + [ + 0.580178, + "l" + ], + [ + 0.29757, + "z" + ], + [ + 0.3111, + "4" + ], + [ + 1.085772, + " " + ], + [ + 0.635539, + "\u001b[4m/\u001b[24m" + ], + [ + 0.268857, + "\b\u001b[4m/\u001b[4mm\u001b[24m" + ], + [ + 0.121341, + "\b\u001b[4mm\u001b[4me\u001b[24m" + ], + [ + 0.141645, + "\b\u001b[4me\u001b[4md\u001b[24m" + ], + [ + 0.230858, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.010346, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mborg\u001b[39m create --stats --progress --compression lz4 \u001b[4m/media\u001b[1m\u001b[4m/\u001b[0m\u001b[24m\u001b[K" + ], + [ + 0.416084, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.004048, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mborg\u001b[39m create --stats --progress --compression lz4 \u001b[4m/media/backup\u001b[1m\u001b[4m/\u001b[0m\u001b[24m\u001b[K" + ], + [ + 0.346657, + "\u001b[?7l" + ], + [ + 2.7e-05, + "\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.003996, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mborg\u001b[39m create --stats --progress --compression lz4 \u001b[4m/media/backup/borgdemo\u001b[1m\u001b[4m/\u001b[0m\u001b[24m\u001b[K" + ], + [ + 1.068791, + "\b\b\u001b[4mo\u001b[24m\u001b[0m\u001b[24m \b" + ], + [ + 1.210608, + "\u001b[22D\u001b[24m/\u001b[24mm\u001b[24me\u001b[24md\u001b[24mi\u001b[24ma\u001b[24m/\u001b[24mb\u001b[24ma\u001b[24mc\u001b[24mk\u001b[24mu\u001b[24mp\u001b[24m/\u001b[24mb\u001b[24mo\u001b[24mr\u001b[24mg\u001b[24md\u001b[24me\u001b[24mm\u001b[24mo:" + ], + [ + 0.125995, + ":" + ], + [ + 0.376036, + "b" + ], + [ + 0.101011, + "a" + ], + [ + 0.178171, + "c \r\u001b[K" + ], + [ + 0.133561, + "k" + ], + [ + 0.162923, + "\rku" + ], + [ + 0.241519, + "p" + ], + [ + 1.426974, + "1" + ], + [ + 0.432275, + " " + ], + [ + 0.295102, + "\u001b[4mW\u001b[24m" + ], + [ + 0.158768, + "\b\u001b[4mW\u001b[4ma\u001b[24m" + ], + [ + 0.270666, + "\b\u001b[4ma\u001b[4ml\u001b[24m" + ], + [ + 0.13015, + "\b\u001b[4ml\u001b[4ml\u001b[24m" + ], + [ + 0.267749, + "\b\u001b[4ml\u001b[4mp\u001b[24m" + ], + [ + 0.173461, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.003997, + "\u001b[A\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ " + ], + [ + 3.5e-05, + "\u001b[32mborg\u001b[39m create --stats --progress --compression lz4 /media/backup/borgdemo::backup1 \u001b[4mWallpaper\u001b[1m\u001b[4m/\u001b[0m\u001b[24m\u001b[K" + ], + [ + 0.997225, + "\b\b\u001b[4mr\u001b[24m\u001b[K" + ], + [ + 0.447022, + "\u001b[?1l\u001b>" + ], + [ + 0.002978, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000917, + "\u001b]2;borg create --stats --progress --compression lz4 Wallpaper\u0007\u001b]1;borg\u0007" + ], + [ + 0.630228, + "Enter passphrase for key /media/backup/borgdemo: " + ], + [ + 2.264647, + "\r\n" + ], + [ + 0.108689, + "0 B O 0 B C 0 B D 0 N Wallpaper \r" + ], + [ + 0.024194, + "Initializing cache transaction: Reading config \r" + ], + [ + 0.000234, + "Initializing cache transaction: Reading chunks \r" + ], + [ + 0.000225, + "Initializing cache transaction: Reading files \r" + ], + [ + 0.000215, + " \r" + ], + [ + 0.179719, + "5.21 MB O 5.21 MB C 5.21 MB D 8 N Wallpaper/bigcollec...em_Wasserloch_im_Q.jpg\r" + ], + [ + 0.206362, + "17.55 MB O 17.51 MB C 15.18 MB D 31 N Wallpaper/bigcoll...ckr_OpenGetty_Im.jpg\r" + ], + [ + 0.202173, + "28.63 MB O 28.59 MB C 26.26 MB D 49 N Wallpaper/bigcoll...ortugal____Stefa.jpg\r" + ], + [ + 0.201105, + "41.30 MB O 41.26 MB C 38.94 MB D 71 N Wallpaper/bigcoll...e_in_der_Gro_auf.jpg\r" + ], + [ + 0.205913, + "53.63 MB O 53.54 MB C 51.21 MB D 93 N Wallpaper/bigcoll...De_Janeiro__Bras.jpg\r" + ], + [ + 0.201657, + "66.10 MB O 65.99 MB C 63.66 MB D 115 N Wallpaper/bigcol...Kenai-Fjords-Nat.jpg\r" + ], + [ + 0.222663, + "78.06 MB O 77.92 MB C 75.59 MB D 135 N Wallpaper/bigcol...ien____Nora_De_A.jpg\r" + ], + [ + 0.206809, + "89.99 MB O 89.82 MB C 85.43 MB D 155 N Wallpaper/bigcol...__De_Pere__Wisco.jpg\r" + ], + [ + 0.204475, + "101.51 MB O 101.32 MB C 96.93 MB D 175 N Wallpaper/bigco..._Silver_Falls_S.jpg\r" + ], + [ + 0.206201, + "115.08 MB O 114.89 MB C 110.50 MB D 199 N Wallpaper/bigco..._Garret_Suhrie.jpg\r" + ], + [ + 0.202147, + "126.51 MB O 126.28 MB C 119.47 MB D 220 N Wallpaper/bigco...fenmesserfisch.jpg\r" + ], + [ + 0.206629, + "138.74 MB O 138.50 MB C 131.69 MB D 243 N Wallpaper/bigco...tswana____Mich.jpg\r" + ], + [ + 0.214855, + "152.84 MB O 152.60 MB C 142.74 MB D 269 N Wallpaper/bigco...fest__Munich__.jpg\r" + ], + [ + 0.200083, + "163.05 MB O 162.80 MB C 152.94 MB D 288 N Wallpaper/bigco..._Marco_RomaniG.jpg\r" + ], + [ + 0.208535, + "175.85 MB O 175.57 MB C 164.47 MB D 308 N Wallpaper/bigco...gway__Colorado.jpg\r" + ], + [ + 0.21234, + "184.65 MB O 184.36 MB C 173.25 MB D 324 N Wallpaper/bigco...nstanz__Baden-.jpg\r" + ], + [ + 0.200087, + "194.92 MB O 194.59 MB C 183.49 MB D 343 N Wallpaper/bigco...op__Caledon__P.jpg\r" + ], + [ + 0.201257, + "204.71 MB O 204.38 MB C 191.68 MB D 361 N Wallpaper/bigco...izian_in_Jamni.jpg\r" + ], + [ + 0.213355, + "217.22 MB O 216.88 MB C 202.98 MB D 382 N Wallpaper/bigco...appadokien__T_.jpg\r" + ], + [ + 0.202274, + "230.56 MB O 230.16 MB C 212.45 MB D 404 N Wallpaper/bigco...eleiGetty_Imag.jpg\r" + ], + [ + 0.204836, + "242.95 MB O 242.53 MB C 224.34 MB D 426 N Wallpaper/bigco...g__Thailand___.jpg\r" + ], + [ + 0.205093, + "254.42 MB O 254.02 MB C 232.75 MB D 446 N Wallpaper/bigco...ame_Reserve__O.jpg\r" + ], + [ + 0.201488, + "265.77 MB O 265.39 MB C 242.76 MB D 466 N Wallpaper/bigco...e_Republik____.jpg\r" + ], + [ + 0.20036, + "278.64 MB O 278.26 MB C 254.62 MB D 488 N Wallpaper/bigco...ien____Patty_P.jpg\r" + ], + [ + 0.209301, + "288.82 MB O 288.45 MB C 264.81 MB D 505 N Wallpaper/bigco...Ruhpolding__Ch.jpg\r" + ], + [ + 0.214561, + "298.04 MB O 297.68 MB C 274.04 MB D 520 N Wallpaper/bigco...wo__Landkreis_.jpg\r" + ], + [ + 0.222111, + "311.03 MB O 310.66 MB C 287.02 MB D 543 N Wallpaper/bigco...a__Portugal___.jpg\r" + ], + [ + 0.204945, + "319.53 MB O 319.17 MB C 295.53 MB D 558 N Wallpaper/bigco...hinos__Hondura.jpg\r" + ], + [ + 0.213928, + "328.19 MB O 327.77 MB C 304.13 MB D 574 N Wallpaper/bigco...ndon__Gro_brit.jpg\r" + ], + [ + 0.206827, + "338.25 MB O 337.81 MB C 314.17 MB D 591 N Wallpaper/bigco...l_Forest__Bund.jpg\r" + ], + [ + 0.209094, + "347.40 MB O 346.96 MB C 323.32 MB D 606 N Wallpaper/bigco...tlantischen_Oz.jpg\r" + ], + [ + 0.200671, + "361.16 MB O 360.71 MB C 334.04 MB D 628 N Wallpaper/bigco...lpark__British.jpg\r" + ], + [ + 0.208778, + "375.20 MB O 374.77 MB C 348.09 MB D 650 N Wallpaper/bigco...swagen_beim_Ro.jpg\r" + ], + [ + 0.2023, + "385.94 MB O 385.47 MB C 358.79 MB D 669 N Wallpaper/bigco...-Bessin__Frank.jpg\r" + ], + [ + 0.201448, + "396.55 MB O 396.10 MB C 368.89 MB D 687 N Wallpaper/bigco...nian_Switzerla.jpg\r" + ], + [ + 0.200229, + "411.96 MB O 411.41 MB C 373.94 MB D 711 N Wallpaper/bigco...CREATISTAGetty.jpg\r" + ], + [ + 0.202083, + "420.92 MB O 420.38 MB C 382.91 MB D 727 N Wallpaper/bigco...LLCCorbisVCG_G.jpg\r" + ], + [ + 0.202677, + "430.76 MB O 430.21 MB C 392.74 MB D 745 N Wallpaper/bigco...r__Tansania___.jpg\r" + ], + [ + 0.206733, + "441.45 MB O 440.87 MB C 400.76 MB D 763 N Wallpaper/bigco...andenburg__Deu.jpg\r" + ], + [ + 0.205541, + "449.42 MB O 448.83 MB C 408.72 MB D 776 N Wallpaper/bigco...Wind_Cave_Nati.jpg\r" + ], + [ + 0.201764, + "458.56 MB O 457.97 MB C 417.20 MB D 792 N Wallpaper/bigco...dney_Harbour_B.jpg\r" + ], + [ + 0.206272, + "470.73 MB O 470.08 MB C 428.74 MB D 815 N Wallpaper/bigco...hland____Patri.jpg\r" + ], + [ + 0.210875, + "485.80 MB O 485.15 MB C 443.01 MB D 843 N Wallpaper/bigco...Hokkaido__Japa.jpg\r" + ], + [ + 0.227162, + "498.93 MB O 498.27 MB C 450.80 MB D 867 N Wallpaper/bigco...topher_Collins.jpg\r" + ], + [ + 0.206293, + "510.73 MB O 510.07 MB C 462.15 MB D 887 N Wallpaper/bigco...itzeinschlag_i.jpg\r" + ], + [ + 0.200265, + "520.54 MB O 519.86 MB C 471.39 MB D 903 N Wallpaper/bigco..._zwischen_Boli.jpg\r" + ], + [ + 0.204067, + "528.01 MB O 527.33 MB C 478.86 MB D 916 N Wallpaper/bigco...jall__Island__.jpg\r" + ], + [ + 0.209223, + "539.61 MB O 538.94 MB C 490.47 MB D 934 N Wallpaper/bigco..._amares__Provi.jpg\r" + ], + [ + 0.215843, + "551.16 MB O 550.49 MB C 501.50 MB D 952 N Wallpaper/bigco...tionalpark__Ut.jpg\r" + ], + [ + 0.212909, + "561.29 MB O 560.60 MB C 511.22 MB D 970 N Wallpaper/bigco..._Inseln__Niede.jpg\r" + ], + [ + 0.209655, + "571.59 MB O 570.86 MB C 520.92 MB D 989 N Wallpaper/bigco...rbeskopf__Huns.jpg\r" + ], + [ + 0.232431, + "582.52 MB O 581.80 MB C 525.99 MB D 1006 N Wallpaper/bigc...n__an_art_in 2.jpg\r" + ], + [ + 0.201199, + "593.36 MB O 592.12 MB C 536.31 MB D 1036 N Wallpaper/more/Green_Curves.jpg \r" + ], + [ + 0.205747, + "604.80 MB O 603.52 MB C 547.71 MB D 1044 N Wallpaper/evenmore/ChipDE 06.jpg \r" + ], + [ + 0.23016, + "Compacting segments 0% \r" + ], + [ + 0.174726, + "Compacting segments 50% \r" + ], + [ + 4.5e-05, + " \r" + ], + [ + 0.04695, + "Saving files cache \r" + ], + [ + 0.005688, + "Saving chunks cache \r" + ], + [ + 0.000299, + "Saving cache config \r" + ], + [ + 0.107527, + " \r" + ], + [ + 3.7e-05, + " \r" + ], + [ + 0.000355, + "------------------------------------------------------------------------------\r\n" + ], + [ + 3.7e-05, + "Archive name: backup1\r\n" + ], + [ + 1.4e-05, + "Archive fingerprint: 9758c7db339a066360bffad17b2ffac4fb368c6722c0be3a47a7a9b631f06407\r\n" + ], + [ + 0.000106, + "Time (start): Fri, 2017-07-14 21:54:06\r\nTime (end): Fri, 2017-07-14 21:54:17\r\n" + ], + [ + 3.9e-05, + "Duration: 11.40 seconds\r\n" + ], + [ + 3.4e-05, + "Number of files: 1050\r\n" + ], + [ + 7.2e-05, + "Utilization of maximum supported archive size: 0%\r\n------------------------------------------------------------------------------\r\n" + ], + [ + 4.7e-05, + " Original size Compressed size Deduplicated size\r\n" + ], + [ + 1.1e-05, + "This archive: 618.96 MB 617.47 MB 561.67 MB\r\n" + ], + [ + 2.7e-05, + "All archives: 618.96 MB 617.47 MB 561.67 MB\r\n" + ], + [ + 2.4e-05, + "\r\n" + ], + [ + 2.3e-05, + " Unique chunks Total chunks\r\n" + ], + [ + 1.3e-05, + "Chunk index: 999 1093\r\n" + ], + [ + 2.4e-05, + "------------------------------------------------------------------------------\r\n" + ], + [ + 0.04885, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.000195, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 7.4e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000196, + "\u001b[?2004h" + ], + [ + 1.403148, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.918581, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.361872, + "\b\b\u001b[1m#\u001b[1m \u001b[1mT\u001b[0m\u001b[39m" + ], + [ + 0.12148, + "\b\u001b[1mT\u001b[1mh\u001b[0m\u001b[39m" + ], + [ + 0.21559, + "\b\u001b[1mh\u001b[1ma\u001b[0m\u001b[39m" + ], + [ + 0.152309, + "\b\u001b[1ma\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.941741, + "\b\u001b[1mt\u001b[1m's nice, so far.\u001b[0m\u001b[39m" + ], + [ + 1.005262, + "\u001b[?1l\u001b>" + ], + [ + 0.00039, + "\u001b[?2004l\r\r\n" + ], + [ + 0.001061, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 8.3e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 6.1e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000287, + "\u001b[?2004h" + ], + [ + 2.564637, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.34769, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.160447, + "\b\b\u001b[1m#\u001b[1m \u001b[1mS\u001b[0m\u001b[39m" + ], + [ + 0.153165, + "\b\u001b[1mS\u001b[1mo\u001b[0m\u001b[39m" + ], + [ + 0.17514, + "\b\u001b[1mo\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.198658, + "\b\u001b[1m \u001b[1ml\u001b[0m\u001b[39m" + ], + [ + 0.204631, + "\b\u001b[1ml\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.250815, + "\b\u001b[1me\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 1.190059, + "\b\u001b[1mt\u001b[1m's add a new file…\u001b[0m\u001b[39m" + ], + [ + 1.216941, + "\u001b[?1l\u001b>" + ], + [ + 0.000401, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000756, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 2.1e-05, + "\u001b]1;~/Pictures\u0007" + ], + [ + 8.1e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 5.3e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000277, + "\u001b[?2004h" + ], + [ + 1.289557, + "\u001b[1m\u001b[31me\u001b[0m\u001b[39m" + ], + [ + 0.216875, + "\b\u001b[1m\u001b[31me\u001b[1m\u001b[31mc\u001b[0m\u001b[39m" + ], + [ + 0.184187, + "\b\b\u001b[1m\u001b[31me\u001b[1m\u001b[31mc\u001b[1m\u001b[31mh\u001b[0m\u001b[39m" + ], + [ + 0.177444, + "\b\b\b\u001b[0m\u001b[32me\u001b[0m\u001b[32mc\u001b[0m\u001b[32mh\u001b[32mo\u001b[39m" + ], + [ + 0.226152, + " " + ], + [ + 0.320216, + "\u001b[33m\"\u001b[39m" + ], + [ + 0.404454, + "\b\u001b[33m\"\u001b[33ma\u001b[39m" + ], + [ + 0.267657, + "\b\u001b[33ma\u001b[33md\u001b[39m" + ], + [ + 0.130258, + "\b\u001b[33md\u001b[33md\u001b[39m" + ], + [ + 1.613237, + "\b\u001b[33md\u001b[33me\u001b[39m" + ], + [ + 0.175381, + "\b\u001b[33me\u001b[33md\u001b[39m" + ], + [ + 0.404248, + "\b\u001b[33md\u001b[33m \u001b[39m" + ], + [ + 0.669276, + "\b\u001b[33m \u001b[33mn\u001b[39m" + ], + [ + 0.128663, + "\b\u001b[33mn\u001b[33me\u001b[39m" + ], + [ + 0.132483, + "\b\u001b[33me\u001b[33mw\u001b[39m" + ], + [ + 0.175823, + "\b\u001b[33mw\u001b[33m \u001b[39m" + ], + [ + 0.220023, + "\b\u001b[33m \u001b[33mn\u001b[39m" + ], + [ + 0.156931, + "\b\u001b[33mn\u001b[33mi\u001b[39m" + ], + [ + 0.10604, + "\b\u001b[33mi\u001b[33mc\u001b[39m" + ], + [ + 0.166585, + "\b\u001b[33mc\u001b[33me\u001b[39m" + ], + [ + 0.306911, + "\b\u001b[33me\u001b[33m \u001b[39m" + ], + [ + 0.228895, + "\b\u001b[33m \u001b[33mf\u001b[39m" + ], + [ + 0.160772, + "\b\u001b[33mf\u001b[33mi\u001b[39m" + ], + [ + 0.144448, + "\b\u001b[33mi\u001b[33ml\u001b[39m" + ], + [ + 0.125193, + "\b\u001b[33ml\u001b[33me\u001b[39m" + ], + [ + 0.828758, + "\b\u001b[33me\u001b[33m\"\u001b[39m" + ], + [ + 0.566156, + " " + ], + [ + 0.349791, + ">" + ], + [ + 0.577663, + " " + ], + [ + 0.28936, + "\u001b[4mW\u001b[24m" + ], + [ + 0.157708, + "\b\u001b[4mW\u001b[4ma\u001b[24m" + ], + [ + 0.226616, + "\b\u001b[4ma\u001b[4ml\u001b[24m" + ], + [ + 0.106124, + "\b\u001b[4ml\u001b[4ml\u001b[24m" + ], + [ + 0.099397, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.00361, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mecho\u001b[39m \u001b[33m\"added new nice file\"\u001b[39m > \u001b[4mWallpaper\u001b[1m\u001b[4m/\u001b[0m\u001b[24m\u001b[K" + ], + [ + 0.822747, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.003743, + "\r\r\u001b[40C\u001b[0m\u001b[4m/\u001b[24m" + ], + [ + 0.00018, + "\r\r\n\u001b[J" + ], + [ + 5.1e-05, + "\u001b[38;5;33m2048example\u001b[0m/ \u001b[38;5;13mdeer.jpg\u001b[0m \u001b[38;5;33mmore\u001b[0m/ \r\n\u001b[J\u001b[38;5;33mbigcollection\u001b[0m/ \u001b[J\u001b[38;5;33mevenmore\u001b[0m/ \u001b[J \u001b[A\u001b[A\u001b[0m\u001b[27m\u001b[24m\r\u001b[2C\u001b[32mecho\u001b[39m \u001b[33m\"added new nice file\"\u001b[39m > \u001b[4mWallpaper/\u001b[24m\u001b[K" + ], + [ + 1.173525, + "\u001b[10D\u001b[24mW\u001b[24ma\u001b[24ml\u001b[24ml\u001b[24mp\u001b[24ma\u001b[24mp\u001b[24me\u001b[24mr\u001b[24m/n" + ], + [ + 0.118482, + "e" + ], + [ + 0.130187, + "w" + ], + [ + 0.499912, + "f" + ], + [ + 0.161863, + "i" + ], + [ + 0.13679, + "l" + ], + [ + 0.093681, + "e" + ], + [ + 0.261183, + "." + ], + [ + 0.312651, + "t" + ], + [ + 0.10665, + "x" + ], + [ + 0.131562, + "t" + ], + [ + 0.79879, + "\u001b[?1l\u001b>" + ], + [ + 0.001397, + "\u001b[?2004l\r\r\n\u001b[J" + ], + [ + 0.000679, + "\u001b]2;echo \"added new nice file\" > Wallpaper/newfile.txt\u0007\u001b]1;echo\u0007" + ], + [ + 0.000151, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 5.4e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 8.6e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000183, + "\u001b[?2004h" + ], + [ + 2.785656, + "\u001b[32mecho\u001b[39m \u001b[33m\"added new nice file\"\u001b[39m > \u001b[4mWallpaper/newfile.txt\u001b[24m" + ], + [ + 0.206019, + "\u001b[50D\u001b[1m#\u001b[1m \u001b[1mS\u001b[1mo\u001b[1m \u001b[1ml\u001b[1me\u001b[1mt\u001b[1m'\u001b[1ms\u001b[1m \u001b[1ma\u001b[1md\u001b[1md\u001b[1m \u001b[1ma\u001b[1m \u001b[1mn\u001b[1me\u001b[1mw\u001b[1m \u001b[1mf\u001b[1mi\u001b[1ml\u001b[1me\u001b[1m…\u001b[0m\u001b[39m \u001b[24m \u001b[24m \u001b[24m \u001b[24m \u001b[24m \u001b[24m \u001b[24m \u001b[24m \u001b[24m \u001b[24m \u001b[24m \u001b[24m \u001b[24m \u001b[24m \u001b[24m \u001b[24m \u001b[24m \u001b[24m \u001b[24m \u001b[24m \u001b[24m \u001b[24D" + ], + [ + 0.251309, + "\u001b[24D\u001b[1mT\u001b[1mh\u001b[1ma\u001b[1mt\u001b[1m'\u001b[1ms\u001b[1m \u001b[1mn\u001b[1mi\u001b[1mc\u001b[1me\u001b[1m,\u001b[1m \u001b[1ms\u001b[1mo\u001b[1m \u001b[1mf\u001b[1ma\u001b[1mr\u001b[1m.\u001b[0m\u001b[39m\u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \b\b\b\b" + ], + [ + 0.372268, + "\u001b[22D\u001b[0m\u001b[32mb\u001b[0m\u001b[32mo\u001b[0m\u001b[32mr\u001b[0m\u001b[32mg\u001b[39m\u001b[0m\u001b[39m \u001b[0m\u001b[39mc\u001b[0m\u001b[39mr\u001b[0m\u001b[39me\u001b[0m\u001b[39ma\u001b[0m\u001b[39mt\u001b[0m\u001b[39me\u001b[0m\u001b[39m \u001b[0m\u001b[39m-\u001b[0m\u001b[39m-\u001b[0m\u001b[39ms\u001b[0m\u001b[39mt\u001b[0m\u001b[39ma\u001b[0m\u001b[39mt\u001b[0m\u001b[39ms\u001b[0m\u001b[39m \u001b[0m\u001b[39m-\u001b[0m\u001b[39m-progress --compression lz4 /media/backup/borgdemo::backup1 \u001b[4mWallpaper\u001b[24m\u001b[K" + ], + [ + 0.686798, + "\b" + ], + [ + 0.49974, + "\b" + ], + [ + 0.029256, + "\b" + ], + [ + 0.030383, + "\b" + ], + [ + 0.030965, + "\b" + ], + [ + 0.02928, + "\b" + ], + [ + 0.030139, + "\b" + ], + [ + 0.029254, + "\b" + ], + [ + 0.03083, + "\b" + ], + [ + 0.030284, + "\b" + ], + [ + 0.030187, + "\b" + ], + [ + 0.030317, + "\b" + ], + [ + 0.439014, + "\u001b[1C" + ], + [ + 0.357869, + "\u001b[P\u001b[10C \u001b[11D" + ], + [ + 0.141225, + "2\u001b[24m \u001b[4mW\u001b[4ma\u001b[4ml\u001b[4ml\u001b[4mp\u001b[4ma\u001b[4mp\u001b[4me\u001b[4mr\u001b[24m\u001b[10D" + ], + [ + 0.615794, + "\u001b[?1l\u001b>" + ], + [ + 0.001653, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000779, + "\u001b]2;borg create --stats --progress --compression lz4 Wallpaper\u0007\u001b]1;borg\u0007" + ], + [ + 0.627474, + "Enter passphrase for key /media/backup/borgdemo: " + ], + [ + 3.666123, + "\r\n" + ], + [ + 0.128711, + "0 B O 0 B C 0 B D 0 N Wallpaper \r" + ], + [ + 0.006399, + "Initializing cache transaction: Reading config \r" + ], + [ + 0.000208, + "Initializing cache transaction: Reading chunks \r" + ], + [ + 0.000253, + "Initializing cache transaction: Reading files \r" + ], + [ + 0.000269, + " \r" + ], + [ + 0.247567, + "584.80 MB O 584.09 MB C 65 B D 1011 N Wallpaper/newfile.txt \r" + ], + [ + 0.264517, + "Compacting segments 0% \r" + ], + [ + 0.000942, + "Compacting segments 50% \r" + ], + [ + 4e-05, + " \r" + ], + [ + 0.0606, + "Saving files cache \r" + ], + [ + 0.005405, + "Saving chunks cache \r" + ], + [ + 0.000411, + "Saving cache config \r" + ], + [ + 0.079766, + " \r" + ], + [ + 4.7e-05, + " \r" + ], + [ + 0.000375, + "------------------------------------------------------------------------------\r\n" + ], + [ + 2.4e-05, + "Archive name: backup2\r\n" + ], + [ + 2.7e-05, + "Archive fingerprint: 5aaf03d1c710cf774f9c9ff1c6317b621c14e519c6bac459f6d64b31e3bbd200\r\n" + ], + [ + 0.000102, + "Time (start): Fri, 2017-07-14 21:54:56\r\n" + ], + [ + 2.1e-05, + "Time (end): Fri, 2017-07-14 21:54:56\r\nDuration: 0.33 seconds\r\n" + ], + [ + 7.4e-05, + "Number of files: 1051\r\n" + ], + [ + 8.3e-05, + "Utilization of maximum supported archive size: 0%\r\n------------------------------------------------------------------------------\r\n" + ], + [ + 7e-06, + " Original size Compressed size Deduplicated size\r\n" + ], + [ + 2.8e-05, + "This archive: 618.96 MB 617.47 MB 106.70 kB\r\n" + ], + [ + 2.2e-05, + "All archives: 1.24 GB 1.23 GB 561.77 MB\r\n" + ], + [ + 5.3e-05, + "\r\n" + ], + [ + 7e-06, + " Unique chunks Total chunks\r\n" + ], + [ + 2.2e-05, + "Chunk index: 1002 2187\r\n" + ], + [ + 2.3e-05, + "------------------------------------------------------------------------------\r\n" + ], + [ + 0.046167, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 8.1e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 8.5e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000212, + "\u001b[?2004h" + ], + [ + 1.922718, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.225243, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.166756, + "\b\b\u001b[1m#\u001b[1m \u001b[1mW\u001b[0m\u001b[39m" + ], + [ + 0.162323, + "\b\u001b[1mW\u001b[1mo\u001b[0m\u001b[39m" + ], + [ + 0.097757, + "\b\u001b[1mo\u001b[1mw\u001b[0m\u001b[39m" + ], + [ + 0.265877, + "\b\u001b[1mw\u001b[1m, this was a lot faster!\u001b[0m\u001b[39m" + ], + [ + 0.789811, + "\u001b[?1l\u001b>" + ], + [ + 0.000392, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000754, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 7.2e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 7.2e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000273, + "\u001b[?2004h" + ], + [ + 1.15181, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.234049, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.209548, + "\b\b\u001b[1m#\u001b[1m \u001b[1mN\u001b[0m\u001b[39m" + ], + [ + 0.168421, + "\b\u001b[1mN\u001b[1mo\u001b[0m\u001b[39m" + ], + [ + 0.232312, + "\b\u001b[1mo\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.201133, + "\b\u001b[1mt\u001b[1mi\u001b[0m\u001b[39m" + ], + [ + 0.338758, + "\b\u001b[1mi\u001b[1mce the \"Deduplicated size\" in \"This archive\"?\u001b[0m\u001b[39m" + ], + [ + 2.236964, + "\u001b[?1l\u001b>" + ], + [ + 0.000951, + "\u001b[?2004l\r\r\n" + ], + [ + 0.001084, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 9.7e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 9.6e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000505, + "\u001b[?2004h" + ], + [ + 2.51909, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.240091, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.216793, + "\b\b\u001b[1m#\u001b[1m \u001b[1mB\u001b[0m\u001b[39m" + ], + [ + 0.192027, + "\b\u001b[1mB\u001b[1mo\u001b[0m\u001b[39m" + ], + [ + 0.138706, + "\b\u001b[1mo\u001b[1mr\u001b[0m\u001b[39m" + ], + [ + 0.129501, + "\b\u001b[1mr\u001b[1mg\u001b[0m\u001b[39m" + ], + [ + 0.536844, + "\b\u001b[1mg\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.143314, + "\b\u001b[1m \u001b[1mr\u001b[0m\u001b[39m" + ], + [ + 0.138384, + "\b\u001b[1mr\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.197658, + "\b\u001b[1me\u001b[1mcognized that most files did not change and deduplicated them.\u001b[0m\u001b[39m" + ], + [ + 1.432604, + "\u001b[?1l\u001b>" + ], + [ + 0.000397, + "\u001b[?2004l\r\r\n" + ], + [ + 0.00069, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 6.4e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 7.1e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000319, + "\u001b[?2004h" + ], + [ + 1.153873, + "\u001b[?1l\u001b>" + ], + [ + 0.000537, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000623, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.000101, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000108, + "\u001b[?1h\u001b=" + ], + [ + 0.000309, + "\u001b[?2004h" + ], + [ + 0.447325, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.257975, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.210602, + "\b\b\u001b[1m#\u001b[1m \u001b[1mB\u001b[0m\u001b[39m" + ], + [ + 0.182148, + "\b\u001b[1mB\u001b[1mu\u001b[0m\u001b[39m" + ], + [ + 0.159923, + "\b\u001b[1mu\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.165905, + "\b\u001b[1mt\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.175925, + "\b\u001b[1m \u001b[1mw\u001b[0m\u001b[39m" + ], + [ + 0.116184, + "\b\u001b[1mw\u001b[1mh\u001b[0m\u001b[39m" + ], + [ + 0.125029, + "\b\u001b[1mh\u001b[1ma\u001b[0m\u001b[39m" + ], + [ + 0.110311, + "\b\u001b[1ma\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.26718, + "\b\u001b[1mt\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.393846, + "\b\u001b[1m \u001b[1mhappens, when we move a dir and create a new backup?\u001b[0m\u001b[39m" + ], + [ + 1.840157, + "\u001b[?1l\u001b>" + ], + [ + 0.000398, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000678, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.000105, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000102, + "\u001b[?1h\u001b=" + ], + [ + 0.000242, + "\u001b[?2004h" + ], + [ + 1.044202, + "\u001b[1m\u001b[31mm\u001b[0m\u001b[39m" + ], + [ + 0.167573, + "\b\u001b[0m\u001b[32mm\u001b[32mv\u001b[39m" + ], + [ + 0.203794, + " " + ], + [ + 0.199502, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.002962, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mmv\u001b[39m \u001b[4mWallpaper\u001b[1m\u001b[4m/\u001b[0m\u001b[24m\u001b[K" + ], + [ + 0.399299, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.004451, + "\r\r\u001b[14C\u001b[0m\u001b[4m/\u001b[24m" + ], + [ + 0.000168, + "\r\r\n\u001b[J" + ], + [ + 3.2e-05, + "\u001b[38;5;33m2048example\u001b[0m/ \u001b[38;5;13mdeer.jpg\u001b[0m \u001b[38;5;33mmore\u001b[0m/ \r\n\u001b[J\u001b[38;5;33mbigcollection\u001b[0m/ \u001b[J\u001b[38;5;33mevenmore\u001b[0m/ \u001b[Jnewfile.txt \u001b[J \u001b[A\u001b[A\u001b[0m\u001b[27m\u001b[24m\r\u001b[2C\u001b[32mmv\u001b[39m \u001b[4mWallpaper/\u001b[24m\u001b[K" + ], + [ + 0.416097, + "\u001b[?7l" + ], + [ + 1.3e-05, + "\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.002339, + "\u001b[10D\u001b[24mW\u001b[24ma\u001b[24ml\u001b[24ml\u001b[24mp\u001b[24ma\u001b[24mp\u001b[24me\u001b[24mr\u001b[24m/2048example\u001b[1m/\u001b[0m" + ], + [ + 0.000184, + "\r\r\n" + ], + [ + 0.000156, + "\u001b[7m2048example/ \u001b[0m \u001b[38;5;13mdeer.jpg\u001b[0m \u001b[38;5;33mmore\u001b[0m/ \u001b[K\r\n\u001b[J\u001b[38;5;33mbigcollection\u001b[0m/ \u001b[J\u001b[38;5;33mevenmore\u001b[0m/ \u001b[Jnewfile.txt \u001b[J \u001b[A\u001b[A\u001b[0m\u001b[27m\u001b[24m\r\u001b[2C\u001b[32mmv\u001b[39m Wallpaper/2048example\u001b[1m/\u001b[0m\u001b[K" + ], + [ + 0.23342, + "\r\r\n" + ], + [ + 1.4e-05, + "\u001b[7m2048example/ \u001b[0m \r\u001b[7m2048example/ \u001b[0m \r\u001b[A\u001b[0m\u001b[27m\u001b[24m\r\u001b[2C\u001b[32mmv\u001b[39m Wallpaper/2048example\u001b[1m/\u001b[0m\u001b[K\u001b[12Dbigcollecti\u001b[0mon\u001b[1m/\u001b[0m" + ], + [ + 0.000154, + "\r\r\n" + ], + [ + 2.5e-05, + "\u001b[38;5;33m2048example\u001b[0m/ \r\u001b[1B\u001b[7mbigcollection/\u001b[0m \r\u001b[A\u001b[A\u001b[0m\u001b[27m\u001b[24m\r\u001b[2C\u001b[32mmv\u001b[39m Wallpaper/bigcollection\u001b[1m/\u001b[0m\u001b[K" + ], + [ + 0.378809, + "\r\r\n\u001b[J\u001b[A\u001b[29C" + ], + [ + 0.002159, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mmv\u001b[39m \u001b[4mWallpaper/bigcollection\u001b[24m\u001b[K\u001b[1C" + ], + [ + 0.35586, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.007824, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mmv\u001b[39m \u001b[4mWallpaper/bigcollection\u001b[24m \u001b[4mWallpaper\u001b[1m\u001b[4m/\u001b[0m\u001b[24m\u001b[K" + ], + [ + 0.248908, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.002608, + "\r\r\u001b[38C\u001b[0m\u001b[4m/\u001b[24m" + ], + [ + 0.000171, + "\r\r\n\u001b[J" + ], + [ + 5.4e-05, + "\u001b[38;5;33m2048example\u001b[0m/ \u001b[38;5;13mdeer.jpg\u001b[0m \u001b[38;5;33mmore\u001b[0m/ \r\n\u001b[J\u001b[38;5;33mbigcollection\u001b[0m/ \u001b[J\u001b[38;5;33mevenmore\u001b[0m/ \u001b[Jnewfile.txt \u001b[J \u001b[A\u001b[A\u001b[0m\u001b[27m\u001b[24m\r\u001b[2C\u001b[32mmv\u001b[39m \u001b[4mWallpaper/bigcollection\u001b[24m \u001b[4mWallpaper/\u001b[24m\u001b[K" + ], + [ + 0.248788, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.004567, + "\u001b[10D\u001b[24mW\u001b[24ma\u001b[24ml\u001b[24ml\u001b[24mp\u001b[24ma\u001b[24mp\u001b[24me\u001b[24mr\u001b[24m/2048example\u001b[1m/\u001b[0m" + ], + [ + 0.000182, + "\r\r\n" + ], + [ + 9.1e-05, + "\u001b[7m2048example/ \u001b[0m \u001b[38;5;13mdeer.jpg\u001b[0m \u001b[38;5;33mmore\u001b[0m/ \u001b[K\r\n\u001b[J\u001b[38;5;33mbigcollection\u001b[0m/ \u001b[J\u001b[38;5;33mevenmore\u001b[0m/ \u001b[Jnewfile.txt \u001b[J \u001b[A\u001b[A\u001b[0m\u001b[27m\u001b[24m\r\u001b[2C\u001b[32mmv\u001b[39m \u001b[4mWallpaper/bigcollection\u001b[24m Wallpaper/2048example\u001b[1m/\u001b[0m\u001b[K" + ], + [ + 0.24704, + "\r\r\n" + ], + [ + 3.2e-05, + "\u001b[7m2048example/ \u001b[0m \r\u001b[7m2048example/ \u001b[0m \r\u001b[A\u001b[0m\u001b[27m\u001b[24m\r\u001b[2C\u001b[32mmv\u001b[39m \u001b[4mWallpaper/bigcollection\u001b[24m Wallpaper/2048example\u001b[1m/\u001b[0m\u001b[K\u001b[12Dbigcollecti\u001b[0mon\u001b[1m/\u001b[0m" + ], + [ + 0.000389, + "\r\r\n" + ], + [ + 3e-05, + "\u001b[38;5;33m2048example\u001b[0m/ \r\u001b[1B\u001b[7mbigcollection/\u001b[0m \r\u001b[A\u001b[A\u001b[0m\u001b[27m\u001b[24m\r\u001b[2C\u001b[32mmv\u001b[39m \u001b[4mWallpaper/bigcollection\u001b[24m Wallpaper/bigcollection\u001b[1m/\u001b[0m\u001b[K" + ], + [ + 0.595335, + "\r\r\n\u001b[J\u001b[A\u001b[53C" + ], + [ + 0.003755, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mmv\u001b[39m \u001b[4mWallpaper/bigcollection\u001b[24m \u001b[4mWallpaper/bigcollection\u001b[24m\u001b[K\u001b[1C" + ], + [ + 0.271014, + "\b" + ], + [ + 0.554135, + "\u001b[23D\u001b[24mW\u001b[24ma\u001b[24ml\u001b[24ml\u001b[24mp\u001b[24ma\u001b[24mp\u001b[24me\u001b[24mr\u001b[24m/\u001b[24mb\u001b[24mi\u001b[24mg\u001b[24mc\u001b[24mo\u001b[24ml\u001b[24ml\u001b[24me\u001b[24mc\u001b[24mt\u001b[24mi\u001b[24mo\u001b[24mn_" + ], + [ + 0.317529, + "N" + ], + [ + 0.104435, + "E" + ], + [ + 0.175308, + "W" + ], + [ + 0.956051, + "\u001b[?1l\u001b>" + ], + [ + 0.001192, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000754, + "\u001b]2;mv -i Wallpaper/bigcollection Wallpaper/bigcollection_NEW\u0007\u001b]1;mv\u0007" + ], + [ + 0.001182, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 9.8e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 7.7e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000188, + "\u001b[?2004h" + ], + [ + 1.857261, + "\u001b[32mmv\u001b[39m Wallpaper/bigcollection \u001b[4mWallpaper/bigcollection_NEW\u001b[24m" + ], + [ + 0.208181, + "\u001b[54D\u001b[1m#\u001b[1m \u001b[1mB\u001b[1mu\u001b[1mt\u001b[1m \u001b[1mw\u001b[1mh\u001b[1ma\u001b[1mt\u001b[1m \u001b[1mh\u001b[1ma\u001b[1mp\u001b[1mp\u001b[1me\u001b[1mn\u001b[1ms\u001b[1m,\u001b[1m \u001b[1mw\u001b[1mh\u001b[1me\u001b[1mn\u001b[1m \u001b[1mw\u001b[1me\u001b[24m\u001b[1m \u001b[24m\u001b[1mm\u001b[24m\u001b[1mo\u001b[24m\u001b[1mv\u001b[24m\u001b[1me\u001b[24m\u001b[1m \u001b[24m\u001b[1ma\u001b[24m\u001b[1m \u001b[24m\u001b[1md\u001b[24m\u001b[1mi\u001b[24m\u001b[1mr\u001b[24m\u001b[1m \u001b[24m\u001b[1ma\u001b[24m\u001b[1mn\u001b[24m\u001b[1md\u001b[24m\u001b[1m \u001b[24m\u001b[1mc\u001b[24m\u001b[1mr\u001b[24m\u001b[1me\u001b[24m\u001b[1ma\u001b[24m\u001b[1mt\u001b[24m\u001b[1me\u001b[24m\u001b[1m \u001b[24m\u001b[1ma\u001b[24m\u001b[1m \u001b[24m\u001b[1mn\u001b[24m\u001b[1me\u001b[1mw backup?\u001b[0m\u001b[39m" + ], + [ + 0.2399, + "\u001b[60D\u001b[1mo\u001b[1mr\u001b[1mg\u001b[1m \u001b[1mr\u001b[1me\u001b[1mc\u001b[1mo\u001b[1mg\u001b[1mn\u001b[1mi\u001b[1mz\u001b[1me\u001b[1md\u001b[1m \u001b[1mt\u001b[1mh\u001b[1ma\u001b[1mt\u001b[1m \u001b[1mm\u001b[1mo\u001b[1ms\u001b[1mt\u001b[1m \u001b[1mf\u001b[1mi\u001b[1ml\u001b[1me\u001b[1ms\u001b[1m \u001b[1md\u001b[1mi\u001b[1md\u001b[1m \u001b[1mn\u001b[1mo\u001b[1mt\u001b[1m \u001b[1mc\u001b[1mh\u001b[1ma\u001b[1mn\u001b[1mg\u001b[1me\u001b[1m \u001b[1ma\u001b[1mn\u001b[1md\u001b[1m \u001b[1md\u001b[1me\u001b[1md\u001b[1mu\u001b[1mp\u001b[1ml\u001b[1mi\u001b[1mc\u001b[1ma\u001b[1mt\u001b[1med them.\u001b[0m\u001b[39m" + ], + [ + 0.227963, + "\u001b[69D\u001b[1mN\u001b[1mo\u001b[1mt\u001b[1mi\u001b[1mc\u001b[1me\u001b[1m \u001b[1mt\u001b[1mh\u001b[1me\u001b[1m \u001b[1m\"\u001b[1mD\u001b[2C\u001b[0m\u001b[39m\u001b[39P\u001b[10C\u001b[1ms\u001b[1mi\u001b[1mz\u001b[1me\u001b[1m\"\u001b[1m in \"This archive\"?\u001b[0m\u001b[39m \u001b[20D" + ], + [ + 0.344233, + "\u001b[49D\u001b[1mW\u001b[1mo\u001b[1mw\u001b[1m,\u001b[1m \u001b[1mt\u001b[1mh\u001b[1mi\u001b[1ms\u001b[1m \u001b[1mw\u001b[1ma\u001b[1ms\u001b[1m \u001b[1ma\u001b[1m \u001b[1ml\u001b[1mo\u001b[1mt\u001b[1m \u001b[1mf\u001b[1ma\u001b[1ms\u001b[1mt\u001b[1me\u001b[1mr\u001b[1m!\u001b[0m\u001b[39m\u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[0m\u001b[39m \u001b[22D" + ], + [ + 0.396096, + "\u001b[29D\u001b[0m\u001b[32mb\u001b[0m\u001b[32mo\u001b[0m\u001b[32mr\u001b[0m\u001b[32mg\u001b[39m\u001b[0m\u001b[39m \u001b[0m\u001b[39mc\u001b[0m\u001b[39mr\u001b[0m\u001b[39me\u001b[0m\u001b[39ma\u001b[0m\u001b[39mt\u001b[0m\u001b[39me\u001b[0m\u001b[39m \u001b[0m\u001b[39m-\u001b[0m\u001b[39m-\u001b[0m\u001b[39ms\u001b[0m\u001b[39mt\u001b[0m\u001b[39ma\u001b[0m\u001b[39mt\u001b[0m\u001b[39ms\u001b[0m\u001b[39m \u001b[0m\u001b[39m-\u001b[0m\u001b[39m-\u001b[0m\u001b[39mp\u001b[0m\u001b[39mr\u001b[0m\u001b[39mo\u001b[0m\u001b[39mg\u001b[0m\u001b[39mr\u001b[0m\u001b[39me\u001b[0m\u001b[39mss --compression lz4 /media/backup/borgdemo::backup2 \u001b[4mWallpaper\u001b[24m\u001b[K" + ], + [ + 0.854343, + "\b" + ], + [ + 0.192067, + "\b" + ], + [ + 0.161921, + "\b" + ], + [ + 0.152949, + "\b" + ], + [ + 0.158914, + "\b" + ], + [ + 0.150013, + "\b" + ], + [ + 0.168061, + "\b" + ], + [ + 0.170964, + "\b" + ], + [ + 0.156237, + "\b" + ], + [ + 0.161813, + "\b" + ], + [ + 0.698972, + "\b\u001b[P\u001b[10C \u001b[11D" + ], + [ + 0.185005, + "3\u001b[24m \u001b[4mW\u001b[4ma\u001b[4ml\u001b[4ml\u001b[4mp\u001b[4ma\u001b[4mp\u001b[4me\u001b[4mr\u001b[24m\u001b[10D" + ], + [ + 0.670037, + "\u001b[?1l\u001b>" + ], + [ + 0.002029, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000793, + "\u001b]2;borg create --stats --progress --compression lz4 Wallpaper\u0007\u001b]1;borg\u0007" + ], + [ + 0.621587, + "Enter passphrase for key /media/backup/borgdemo: " + ], + [ + 5.034162, + "\r\n" + ], + [ + 0.136527, + "0 B O 0 B C 0 B D 0 N Wallpaper \r" + ], + [ + 0.028491, + "Initializing cache transaction: Reading config \r" + ], + [ + 0.000245, + "Initializing cache transaction: Reading chunks \r" + ], + [ + 0.000278, + "Initializing cache transaction: Reading files \r" + ], + [ + 0.000296, + " \r" + ], + [ + 0.173817, + "10.07 MB O 10.04 MB C 0 B D 17 N Wallpaper/bigcollec...rland__England____A.jpg\r" + ], + [ + 0.20311, + "29.10 MB O 29.05 MB C 0 B D 50 N Wallpaper/bigcollec...Creek_Redwoods_Stat.jpg\r" + ], + [ + 0.202422, + "47.67 MB O 47.62 MB C 0 B D 83 N Wallpaper/bigcollec...rson-Wildschutzgebi.jpg\r" + ], + [ + 0.216811, + "64.30 MB O 64.19 MB C 0 B D 112 N Wallpaper/bigcollec..._Planten_un_Blomen.jpg\r" + ], + [ + 0.214409, + "80.89 MB O 80.75 MB C 0 B D 140 N Wallpaper/bigcollec...g__Cologne__German.jpg\r" + ], + [ + 0.202244, + "100.45 MB O 100.26 MB C 0 B D 173 N Wallpaper/bigcolle..._Menorca__Spanien.jpg\r" + ], + [ + 0.202027, + "116.80 MB O 116.61 MB C 0 B D 202 N Wallpaper/bigcolle...artenkirchen__Bay.jpg\r" + ], + [ + 0.202003, + "130.38 MB O 130.15 MB C 0 B D 227 N Wallpaper/bigcolle..._zur_Felsenkirche.jpg\r" + ], + [ + 0.234918, + "143.32 MB O 143.09 MB C 0 B D 251 N Wallpaper/bigcolle...land__Antarktis__.jpg\r" + ], + [ + 0.204976, + "156.31 MB O 156.07 MB C 0 B D 275 N Wallpaper/bigcolle...-Stadion__Rio_de_.jpg\r" + ], + [ + 0.205408, + "173.36 MB O 173.09 MB C 0 B D 304 N Wallpaper/bigcolle...lpark__Alaska__US.jpg\r" + ], + [ + 0.221776, + "183.65 MB O 183.35 MB C 0 B D 322 N Wallpaper/bigcolle...lmeer____Pasquale.jpg\r" + ], + [ + 0.201052, + "195.95 MB O 195.63 MB C 0 B D 345 N Wallpaper/bigcolle...Schutzgebiet_Mary.jpg\r" + ], + [ + 0.240687, + "217.22 MB O 216.88 MB C 0 B D 382 N Wallpaper/bigcolle...__Kappadokien__T_.jpg\r" + ], + [ + 0.20767, + "233.09 MB O 232.68 MB C 0 B D 409 N Wallpaper/bigcolle...epublic_ImagesShu.jpg\r" + ], + [ + 0.210433, + "250.21 MB O 249.81 MB C 0 B D 439 N Wallpaper/bigcolle...ter__Pr_fektur_Fu.jpg\r" + ], + [ + 0.200954, + "268.90 MB O 268.51 MB C 0 B D 472 N Wallpaper/bigcolle...uth_Carolina__USA.jpg\r" + ], + [ + 0.212828, + "286.72 MB O 286.35 MB C 0 B D 502 N Wallpaper/bigcolle...l_Park__Cobham__E.jpg\r" + ], + [ + 0.206527, + "296.84 MB O 296.47 MB C 0 B D 518 N Wallpaper/bigcolle...entAlamy______Bin.jpg\r" + ], + [ + 0.205003, + "310.38 MB O 310.00 MB C 0 B D 542 N Wallpaper/bigcolle...ationalpark__Flor.jpg\r" + ], + [ + 0.209538, + "320.38 MB O 320.03 MB C 0 B D 559 N Wallpaper/bigcolle...ma__Bahamas____Ji.jpg\r" + ], + [ + 0.201896, + "331.76 MB O 331.35 MB C 0 B D 580 N Wallpaper/bigcolle...rd_Bay__Eyre-Halb.jpg\r" + ], + [ + 0.207585, + "347.40 MB O 346.96 MB C 0 B D 606 N Wallpaper/bigcolle...s_Atlantischen_Oz.jpg\r" + ], + [ + 0.200781, + "369.05 MB O 368.62 MB C 0 B D 640 N Wallpaper/bigcolle...ankreich____John_.jpg\r" + ], + [ + 0.202326, + "379.22 MB O 378.78 MB C 0 B D 657 N Wallpaper/bigcolle...chtanemone__Insel.jpg\r" + ], + [ + 0.211929, + "389.83 MB O 389.36 MB C 0 B D 676 N Wallpaper/bigcolle...ugal____Mikael_Sv.jpg\r" + ], + [ + 0.219553, + "402.12 MB O 401.68 MB C 0 B D 695 N Wallpaper/bigcolle...rk_Sarek__Schwede.jpg\r" + ], + [ + 0.20375, + "416.03 MB O 415.48 MB C 0 B D 718 N Wallpaper/bigcolle...em_taubenetzten_G.jpg\r" + ], + [ + 0.201474, + "428.93 MB O 428.38 MB C 0 B D 742 N Wallpaper/bigcolle...Francisco_Bay__Ka.jpg\r" + ], + [ + 0.200248, + "437.92 MB O 437.35 MB C 0 B D 756 N Wallpaper/bigcolle..._der_N_he_von_Tro.jpg\r" + ], + [ + 0.215254, + "446.04 MB O 445.46 MB C 0 B D 770 N Wallpaper/bigcolle...enver__Colorado__.jpg\r" + ], + [ + 0.202133, + "455.95 MB O 455.36 MB C 0 B D 787 N Wallpaper/bigcolle..._Son_Doong-H_hle_.jpg\r" + ], + [ + 0.208499, + "471.36 MB O 470.71 MB C 0 B D 816 N Wallpaper/bigcolle...ly_National_Monum.jpg\r" + ], + [ + 0.205116, + "491.46 MB O 490.81 MB C 0 B D 853 N Wallpaper/bigcolle...ted_during_the_ 1.jpg\r" + ], + [ + 0.220215, + "510.73 MB O 510.07 MB C 0 B D 887 N Wallpaper/bigcolle..._Blitzeinschlag_i.jpg\r" + ], + [ + 0.201825, + "522.32 MB O 521.65 MB C 0 B D 906 N Wallpaper/bigcolle...vador__Santiago__.jpg\r" + ], + [ + 0.202937, + "534.02 MB O 533.34 MB C 0 B D 925 N Wallpaper/bigcolle...doah_National_Par.jpg\r" + ], + [ + 0.202635, + "550.50 MB O 549.83 MB C 0 B D 951 N Wallpaper/bigcolle...liffs_National_Mo.jpg\r" + ], + [ + 0.202296, + "564.18 MB O 563.47 MB C 0 B D 976 N Wallpaper/bigcolle...n_in_Aktion____Va.jpg\r" + ], + [ + 0.203791, + "576.43 MB O 575.71 MB C 0 B D 996 N Wallpaper/bigcolle...______WRIGHTSuper.jpg\r" + ], + [ + 0.439796, + "Compacting segments 0% \r" + ], + [ + 0.000919, + "Compacting segments 50% \r" + ], + [ + 3.7e-05, + " \r" + ], + [ + 0.040817, + "Saving files cache \r" + ], + [ + 0.010023, + "Saving chunks cache \r" + ], + [ + 0.000278, + "Saving cache config \r" + ], + [ + 0.093829, + " \r" + ], + [ + 1.6e-05, + " \r" + ], + [ + 0.000308, + "------------------------------------------------------------------------------\r\n" + ], + [ + 9e-06, + "Archive name: backup3\r\n" + ], + [ + 3.8e-05, + "Archive fingerprint: 36cd8fdf9b8b2e3bbb3fc2bb600acd48609efaf3a0880f900e0701a47ff69d4d\r\n" + ], + [ + 2e-05, + "Time (start): Fri, 2017-07-14 21:55:37\r\n" + ], + [ + 2.4e-05, + "Time (end): Fri, 2017-07-14 21:55:46\r\n" + ], + [ + 2.2e-05, + "Duration: 8.58 seconds\r\n" + ], + [ + 2.6e-05, + "Number of files: 1051\r\n" + ], + [ + 2.6e-05, + "Utilization of maximum supported archive size: 0%\r\n" + ], + [ + 2.1e-05, + "------------------------------------------------------------------------------\r\n" + ], + [ + 2.6e-05, + " Original size Compressed size Deduplicated size\r\n" + ], + [ + 2.4e-05, + "This archive: 618.96 MB 617.47 MB 107.55 kB\r\n" + ], + [ + 2.1e-05, + "All archives: 1.86 GB 1.85 GB 561.88 MB\r\n" + ], + [ + 2.5e-05, + "\r\n" + ], + [ + 3.9e-05, + " Unique chunks Total chunks\r\n" + ], + [ + 1.1e-05, + "Chunk index: 1006 3283\r\n" + ], + [ + 4.8e-05, + "------------------------------------------------------------------------------\r\n" + ], + [ + 0.048607, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 1.9e-05, + "\u001b]1;~/Pictures\u0007" + ], + [ + 7.7e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 6.2e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.00017, + "\u001b[?2004h" + ], + [ + 1.509372, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.261334, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.25826, + "\b\b\u001b[1m#\u001b[1m \u001b[1mS\u001b[0m\u001b[39m" + ], + [ + 0.162616, + "\b\u001b[1mS\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.27891, + "\b\u001b[1mt\u001b[1mi\u001b[0m\u001b[39m" + ], + [ + 0.174723, + "\b\u001b[1mi\u001b[1ml\u001b[0m\u001b[39m" + ], + [ + 0.124142, + "\b\u001b[1ml\u001b[1ml\u001b[0m\u001b[39m" + ], + [ + 1.012371, + "\b\u001b[1ml\u001b[1m quite fast…\u001b[0m\u001b[39m" + ], + [ + 0.74493, + "\u001b[?1l\u001b>" + ], + [ + 0.000416, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000686, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 6e-06, + "\u001b]1;~/Pictures\u0007" + ], + [ + 8.8e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000111, + "\u001b[?1h\u001b=" + ], + [ + 0.000271, + "\u001b[?2004h" + ], + [ + 2.038818, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.861519, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 2.235116, + "\b\b\u001b[1m#\u001b[1m \u001b[1mB\u001b[0m\u001b[39m" + ], + [ + 0.20981, + "\b\u001b[1mB\u001b[1mu\u001b[0m\u001b[39m" + ], + [ + 0.216676, + "\b\u001b[1mu\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.72822, + "\b\u001b[1mt\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 1.094756, + "\b\u001b[1m \u001b[1mw\u001b[0m\u001b[39m" + ], + [ + 0.315528, + "\b\u001b[1mw\u001b[1mh\u001b[0m\u001b[39m" + ], + [ + 0.23713, + "\b\u001b[1mh\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.286805, + "\b\u001b[1me\u001b[1mn\u001b[0m\u001b[39m" + ], + [ + 0.638764, + "\b\u001b[1mn\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.81778, + "\b\u001b[1m \u001b[1my\u001b[0m\u001b[39m" + ], + [ + 0.245269, + "\b\u001b[1my\u001b[1mou look at the \"deduplicated file size\" again, you see that borg\u001b[1m \u001b[1malso recognized that only the dir and not the files changed in this backup.\u001b[0m\u001b[39m\u001b[K" + ], + [ + 2.34618, + "\u001b[?1l\u001b>" + ], + [ + 0.000453, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000631, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.00011, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000113, + "\u001b[?1h\u001b=" + ], + [ + 0.000262, + "\u001b[?2004h" + ], + [ + 3.418707, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.275819, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.2004, + "\b\b\u001b[1m#\u001b[1m \u001b[1mN\u001b[0m\u001b[39m" + ], + [ + 0.172829, + "\b\u001b[1mN\u001b[1mo\u001b[0m\u001b[39m" + ], + [ + 0.308378, + "\b\u001b[1mo\u001b[1mw\u001b[0m\u001b[39m" + ], + [ + 0.703684, + "\b\u001b[1mw\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.8183, + "\b\u001b[1m \u001b[1ml\u001b[0m\u001b[39m" + ], + [ + 0.193322, + "\b\u001b[1ml\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.18438, + "\b\u001b[1me\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.389996, + "\b\u001b[1mt\u001b[1m's look into a repo.\u001b[0m\u001b[39m" + ], + [ + 0.857879, + "\u001b[?1l\u001b>" + ], + [ + 0.000349, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000564, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 2.9e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 7.9e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000246, + "\u001b[?2004h" + ], + [ + 1.60039, + "\u001b[1m\u001b[31mb\u001b[0m\u001b[39m" + ], + [ + 0.177554, + "\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[0m\u001b[39m" + ], + [ + 0.117613, + "\b\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[1m\u001b[31mr\u001b[0m\u001b[39m" + ], + [ + 0.12982, + "\b\b\b\u001b[0m\u001b[32mb\u001b[0m\u001b[32mo\u001b[0m\u001b[32mr\u001b[32mg\u001b[39m" + ], + [ + 0.145309, + " " + ], + [ + 0.256078, + "l" + ], + [ + 0.145029, + "i" + ], + [ + 0.100415, + "s" + ], + [ + 0.137667, + "t" + ], + [ + 0.172051, + " " + ], + [ + 0.490083, + "\u001b[4m/\u001b[24m" + ], + [ + 0.190449, + "\b\u001b[4m/\u001b[4mm\u001b[24m" + ], + [ + 0.216676, + "\b\u001b[4mm\u001b[4me\u001b[24m" + ], + [ + 0.174909, + "\b\u001b[4me\u001b[4md\u001b[24m" + ], + [ + 0.242368, + "\u001b[?7l\u001b[31m......\u001b[39m" + ], + [ + 3.2e-05, + "\u001b[?7h" + ], + [ + 0.00599, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mborg\u001b[39m list \u001b[4m/media\u001b[1m\u001b[4m/\u001b[0m\u001b[24m\u001b[K" + ], + [ + 0.345758, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.003294, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mborg\u001b[39m list \u001b[4m/media/backup\u001b[1m\u001b[4m/\u001b[0m\u001b[24m\u001b[K" + ], + [ + 0.253376, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.003389, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mborg\u001b[39m list \u001b[4m/media/backup/borgdemo\u001b[1m\u001b[4m/\u001b[0m\u001b[24m\u001b[K" + ], + [ + 1.036958, + "\b\b\u001b[4mo\u001b[24m\u001b[0m\u001b[24m \b" + ], + [ + 2.6e-05, + "\u001b[?1l\u001b>" + ], + [ + 0.000854, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000717, + "\u001b]2;borg list /media/backup/borgdemo\u0007\u001b]1;borg\u0007" + ], + [ + 0.624291, + "Enter passphrase for key /media/backup/borgdemo: " + ], + [ + 2.363577, + "\r\n" + ], + [ + 0.158203, + "backup1 Fri, 2017-07-14 21:54:06 [9758c7db339a066360bffad17b2ffac4fb368c6722c0be3a47a7a9b631f06407]\r\nbackup2 Fri, 2017-07-14 21:54:56 [5aaf03d1c710cf774f9c9ff1c6317b621c14e519c6bac459f6d64b31e3bbd200]\r\nbackup3 Fri, 2017-07-14 21:55:37 [36cd8fdf9b8b2e3bbb3fc2bb600acd48609efaf3a0880f900e0701a47ff69d4d]\r\n" + ], + [ + 0.044143, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 5.2e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 8.4e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000207, + "\u001b[?2004h" + ], + [ + 5.582312, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.371134, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.184918, + "\b\b\u001b[1m#\u001b[1m \u001b[1mY\u001b[0m\u001b[39m" + ], + [ + 0.177123, + "\b\u001b[1mY\u001b[1mo\u001b[0m\u001b[39m" + ], + [ + 0.148041, + "\b\u001b[1mo\u001b[1mu\u001b[0m\u001b[39m" + ], + [ + 0.461676, + "\b\u001b[1mu\u001b[1m'\u001b[0m\u001b[39m" + ], + [ + 0.668888, + "\b\u001b[1m'\u001b[1mll see a list of all backups.\u001b[0m\u001b[39m" + ], + [ + 0.876235, + "\u001b[?1l\u001b>" + ], + [ + 0.000363, + "\u001b[?2004l\r\r\n" + ], + [ + 0.001075, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 8.2e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 8.8e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000297, + "\u001b[?2004h" + ], + [ + 2.475491, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.382591, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.23474, + "\b\b\u001b[1m#\u001b[1m \u001b[1mY\u001b[0m\u001b[39m" + ], + [ + 0.210269, + "\b\u001b[1mY\u001b[1mo\u001b[0m\u001b[39m" + ], + [ + 0.196151, + "\b\u001b[1mo\u001b[1mu\u001b[0m\u001b[39m" + ], + [ + 0.460253, + "\b\u001b[1mu\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.305764, + "\b\u001b[1m \u001b[1mc\u001b[0m\u001b[39m" + ], + [ + 0.184098, + "\b\u001b[1mc\u001b[1ma\u001b[0m\u001b[39m" + ], + [ + 0.212534, + "\b\u001b[1ma\u001b[1mn\u001b[0m\u001b[39m" + ], + [ + 0.305097, + "\b\u001b[1mn\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.163485, + "\b\u001b[1m \u001b[1ma\u001b[0m\u001b[39m" + ], + [ + 0.194803, + "\b\u001b[1ma\u001b[1ml\u001b[0m\u001b[39m" + ], + [ + 0.282791, + "\b\u001b[1ml\u001b[1mso use the same command to look into an archive. But we better f\u001b[1mi\u001b[1mlter the output here:\u001b[0m\u001b[39m\u001b[K" + ], + [ + 2.679252, + "\u001b[?1l\u001b>" + ], + [ + 0.000434, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000646, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 3e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000107, + "\u001b[?1h\u001b=" + ], + [ + 0.000302, + "\u001b[?2004h" + ], + [ + 1.162094, + "\u001b[1m\u001b[31mb\u001b[0m\u001b[39m" + ], + [ + 0.184756, + "\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[0m\u001b[39m" + ], + [ + 0.114887, + "\b\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[1m\u001b[31mr\u001b[0m\u001b[39m" + ], + [ + 0.143983, + "\b\b\b\u001b[0m\u001b[32mb\u001b[0m\u001b[32mo\u001b[0m\u001b[32mr\u001b[32mg\u001b[39m" + ], + [ + 0.230507, + " " + ], + [ + 0.414382, + "l" + ], + [ + 0.153591, + "i" + ], + [ + 0.044178, + "s" + ], + [ + 0.236299, + "t" + ], + [ + 0.330148, + " " + ], + [ + 0.70018, + "\u001b[4m/\u001b[24m" + ], + [ + 0.193582, + "\b\u001b[4m/\u001b[4mm\u001b[24m" + ], + [ + 0.172118, + "\b\u001b[4mm\u001b[4me\u001b[24m" + ], + [ + 0.134283, + "\b\u001b[4me\u001b[4md\u001b[24m" + ], + [ + 0.250757, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.006227, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mborg\u001b[39m list \u001b[4m/media\u001b[1m\u001b[4m/\u001b[0m\u001b[24m\u001b[K" + ], + [ + 0.374078, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.003992, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mborg\u001b[39m list \u001b[4m/media/backup\u001b[1m\u001b[4m/\u001b[0m\u001b[24m\u001b[K" + ], + [ + 0.2609, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.003434, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mborg\u001b[39m list \u001b[4m/media/backup/borgdemo\u001b[1m\u001b[4m/\u001b[0m\u001b[24m\u001b[K" + ], + [ + 0.237963, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.003371, + "\r\r\u001b[34C\u001b[0m\u001b[4m/\u001b[24m" + ], + [ + 0.000178, + "\r\r\n\u001b[J" + ], + [ + 4.2e-05, + "\u001b[0mREADME \u001b[38;5;33mdata\u001b[0m/ index.14 nonce \r\n\u001b[Jconfig \u001b[Jhints.14 \u001b[Jintegrity.14 \u001b[J \u001b[A\u001b[A\u001b[0m\u001b[27m\u001b[24m\r\u001b[2C\u001b[32mborg\u001b[39m list \u001b[4m/media/backup/borgdemo/\u001b[24m\u001b[K" + ], + [ + 0.833604, + "\b\b\u001b[4mo\u001b[24m\u001b[24m \b" + ], + [ + 1.042199, + "\u001b[22D\u001b[24m/\u001b[24mm\u001b[24me\u001b[24md\u001b[24mi\u001b[24ma\u001b[24m/\u001b[24mb\u001b[24ma\u001b[24mc\u001b[24mk\u001b[24mu\u001b[24mp\u001b[24m/\u001b[24mb\u001b[24mo\u001b[24mr\u001b[24mg\u001b[24md\u001b[24me\u001b[24mm\u001b[24mo:" + ], + [ + 0.139477, + ":" + ], + [ + 0.711096, + "b" + ], + [ + 0.099664, + "a" + ], + [ + 0.149912, + "c" + ], + [ + 0.16888, + "k" + ], + [ + 0.923931, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.005451, + "\r\r\r\r\n\u001b[J\u001b[A\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mborg\u001b[39m list /media/backup/borgdemo::back\u001b[K" + ], + [ + 0.885297, + "u" + ], + [ + 0.29853, + "p" + ], + [ + 0.456244, + "3" + ], + [ + 1.061844, + " " + ], + [ + 0.589511, + "|" + ], + [ + 0.527539, + " " + ], + [ + 0.343662, + "\u001b[32mg\u001b[39m" + ], + [ + 0.117117, + "\b\u001b[32mg\u001b[32mr\u001b[39m" + ], + [ + 0.124331, + "\b\b\u001b[1m\u001b[31mg\u001b[1m\u001b[31mr\u001b[1m\u001b[31me\u001b[0m\u001b[39m" + ], + [ + 0.726149, + "\b\b\b\u001b[0m\u001b[32mg\u001b[0m\u001b[32mr\u001b[0m\u001b[32me\u001b[32mp\u001b[39m" + ], + [ + 0.198601, + " " + ], + [ + 0.476336, + "\u001b[33m'\u001b[39m" + ], + [ + 0.392009, + "\b\u001b[33m'\u001b[33md\u001b[39m" + ], + [ + 0.627529, + "\b\u001b[33md\u001b[33me\u001b[39m" + ], + [ + 0.142332, + "\b\u001b[33me\u001b[33me\u001b[39m" + ], + [ + 0.322681, + "\b\u001b[33me\u001b[33mr\u001b[39m" + ], + [ + 0.916328, + "\b\u001b[33mr\u001b[33m.\u001b[39m" + ], + [ + 0.50653, + "\b\u001b[33m.\u001b[33mj\u001b[39m" + ], + [ + 0.242318, + "\b\u001b[33mj\u001b[33mp\u001b[39m" + ], + [ + 0.272214, + "\b\u001b[33mp\u001b[33mg\u001b[39m" + ], + [ + 0.581098, + "\b\u001b[33mg\u001b[33m'\u001b[39m" + ], + [ + 2.559186, + "\u001b[?1l\u001b>" + ], + [ + 0.001382, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000773, + "\u001b]2;borg list /media/backup/borgdemo::backup3 | grep --color 'deer.jpg'\u0007\u001b]1;borg\u0007" + ], + [ + 0.628501, + "Enter passphrase for key /media/backup/borgdemo: " + ], + [ + 2.584332, + "\r\n" + ], + [ + 0.141205, + "-rw-rw-r-- rugk rugk 3781749 Fri, 2017-07-14 17:01:45 Wallpaper/\u001b[01;31m\u001b[Kdeer.jpg\u001b[m\u001b[K\r\n" + ], + [ + 0.054041, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.000135, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 2.7e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.00017, + "\u001b[?2004h" + ], + [ + 2.222435, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.269828, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.211035, + "\b\b\u001b[1m#\u001b[1m \u001b[1mO\u001b[0m\u001b[39m" + ], + [ + 0.184712, + "\b\u001b[1mO\u001b[1mh\u001b[0m\u001b[39m" + ], + [ + 0.374912, + "\b\u001b[1mh\u001b[1m, we found our picture. Now extract it:\u001b[0m\u001b[39m" + ], + [ + 1.545747, + "\u001b[?1l\u001b>" + ], + [ + 0.000418, + "\u001b[?2004l\r\r\n" + ], + [ + 0.00063, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 3.1e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000135, + "\u001b[?1h\u001b=" + ], + [ + 0.000463, + "\u001b[?2004h" + ], + [ + 1.638625, + "\u001b[1m\u001b[31mm\u001b[0m\u001b[39m" + ], + [ + 0.156977, + "\b\u001b[0m\u001b[32mm\u001b[32mv\u001b[39m" + ], + [ + 0.220013, + " " + ], + [ + 0.151118, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.002944, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mmv\u001b[39m \u001b[4mWallpaper\u001b[1m\u001b[4m/\u001b[0m\u001b[24m\u001b[K" + ], + [ + 0.668654, + "\b\b\u001b[4mr\u001b[24m\u001b[0m\u001b[24m " + ], + [ + 0.297169, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.005693, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mmv\u001b[39m \u001b[4mWallpaper\u001b[24m \u001b[4mWallpaper\u001b[1m\u001b[4m/\u001b[0m\u001b[24m\u001b[K" + ], + [ + 0.672973, + "\b\b\u001b[4mr\u001b[24m\u001b[0m\u001b[24m \b" + ], + [ + 0.263416, + "\u001b[9D\u001b[24mW\u001b[24ma\u001b[24ml\u001b[24ml\u001b[24mp\u001b[24ma\u001b[24mp\u001b[24me\u001b[24mr." + ], + [ + 0.334671, + "o" + ], + [ + 0.19768, + "r" + ], + [ + 0.142283, + "i" + ], + [ + 0.17833, + "g" + ], + [ + 0.688576, + "\u001b[?1l\u001b>" + ], + [ + 0.001806, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000954, + "\u001b]2;mv -i Wallpaper Wallpaper.orig\u0007\u001b]1;mv\u0007" + ], + [ + 0.002076, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 5.7e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 7e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000153, + "\u001b[?2004h" + ], + [ + 1.864942, + "\u001b[1m\u001b[31mb\u001b[0m\u001b[39m" + ], + [ + 0.18048, + "\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[0m\u001b[39m" + ], + [ + 0.143872, + "\b\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[1m\u001b[31mr\u001b[0m\u001b[39m" + ], + [ + 0.161829, + "\b\b\b\u001b[0m\u001b[32mb\u001b[0m\u001b[32mo\u001b[0m\u001b[32mr\u001b[32mg\u001b[39m" + ], + [ + 0.170439, + " " + ], + [ + 0.248909, + "c" + ], + [ + 0.365319, + "\b \b" + ], + [ + 0.142233, + "e" + ], + [ + 0.157272, + "x" + ], + [ + 0.166861, + "t" + ], + [ + 0.115114, + "r" + ], + [ + 0.103674, + "a" + ], + [ + 0.102162, + "c" + ], + [ + 0.163264, + "t" + ], + [ + 0.308166, + " " + ], + [ + 1.386497, + "\u001b[4m/\u001b[24m" + ], + [ + 0.183134, + "\b\u001b[4m/\u001b[4mm\u001b[24m" + ], + [ + 0.115533, + "\b\u001b[4mm\u001b[4me\u001b[24m" + ], + [ + 0.12416, + "\b\u001b[4me\u001b[4md\u001b[24m" + ], + [ + 0.206989, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.003179, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mborg\u001b[39m extract \u001b[4m/media\u001b[1m\u001b[4m/\u001b[0m\u001b[24m\u001b[K" + ], + [ + 0.241808, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.003324, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mborg\u001b[39m extract \u001b[4m/media/backup\u001b[1m\u001b[4m/\u001b[0m\u001b[24m\u001b[K" + ], + [ + 0.193552, + "\u001b[?7l" + ], + [ + 2.6e-05, + "\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.003368, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mborg\u001b[39m extract \u001b[4m/media/backup/borgdemo\u001b[1m\u001b[4m/\u001b[0m\u001b[24m\u001b[K" + ], + [ + 0.700774, + "\b\b\u001b[4mo\u001b[24m\u001b[0m\u001b[24m \b" + ], + [ + 1.151074, + "\u001b[22D\u001b[24m/\u001b[24mm\u001b[24me\u001b[24md\u001b[24mi\u001b[24ma\u001b[24m/\u001b[24mb\u001b[24ma\u001b[24mc\u001b[24mk\u001b[24mu\u001b[24mp\u001b[24m/\u001b[24mb\u001b[24mo\u001b[24mr\u001b[24mg\u001b[24md\u001b[24me\u001b[24mm\u001b[24mo:" + ], + [ + 0.146222, + ":" + ], + [ + 0.579644, + "b" + ], + [ + 0.102789, + "a" + ], + [ + 0.178851, + "c" + ], + [ + 0.133936, + "k" + ], + [ + 0.124089, + "u" + ], + [ + 0.229823, + "p" + ], + [ + 0.174738, + "3" + ], + [ + 0.306821, + " " + ], + [ + 4.287483, + "\u001b[45D\u001b[39mb\u001b[39mo\u001b[39mr\u001b[39mg\u001b[41C\u001b[7mWallpaper/deer.jpg\u001b[27m" + ], + [ + 1.718396, + "\u001b[63D\u001b[32mb\u001b[32mo\u001b[32mr\u001b[32mg\u001b[39m\u001b[41C\u001b[27mW\u001b[27ma\u001b[27ml\u001b[27ml\u001b[27mp\u001b[27ma\u001b[27mp\u001b[27me\u001b[27mr\u001b[27m/\u001b[27md\u001b[27me\u001b[27me\u001b[27mr\u001b[27m.\u001b[27mj\u001b[27mp\u001b[27mg" + ], + [ + 6.4e-05, + "\u001b[?1l\u001b>" + ], + [ + 0.001749, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000991, + "\u001b]2;borg extract /media/backup/borgdemo::backup3 Wallpaper/deer.jpg\u0007\u001b]1;borg\u0007" + ], + [ + 0.633044, + "Enter passphrase for key /media/backup/borgdemo: " + ], + [ + 2.659432, + "\r\n" + ], + [ + 0.198939, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.000134, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 7.9e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000169, + "\u001b[?2004h" + ], + [ + 4.506682, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.287992, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.13604, + "\b\b\u001b[1m#\u001b[1m \u001b[1mA\u001b[0m\u001b[39m" + ], + [ + 0.132241, + "\b\u001b[1mA\u001b[1mn\u001b[0m\u001b[39m" + ], + [ + 0.115152, + "\b\u001b[1mn\u001b[1md\u001b[0m\u001b[39m" + ], + [ + 0.190449, + "\b\u001b[1md\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.168765, + "\b\u001b[1m \u001b[1mc\u001b[0m\u001b[39m" + ], + [ + 0.248816, + "\b\u001b[1mc\u001b[1mheck that it's the same:\u001b[0m\u001b[39m" + ], + [ + 1.093037, + "\u001b[?1l\u001b>" + ], + [ + 0.000401, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000745, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 7.7e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.0001, + "\u001b[?1h\u001b=" + ], + [ + 0.000321, + "\u001b[?2004h" + ], + [ + 1.350298, + "\u001b[32md\u001b[39m" + ], + [ + 0.181769, + "\b\u001b[1m\u001b[31md\u001b[1m\u001b[31mi\u001b[0m\u001b[39m" + ], + [ + 0.148155, + "\b\b\u001b[1m\u001b[31md\u001b[1m\u001b[31mi\u001b[1m\u001b[31mf\u001b[0m\u001b[39m" + ], + [ + 0.13874, + "\b\b\b\u001b[0m\u001b[32md\u001b[0m\u001b[32mi\u001b[0m\u001b[32mf\u001b[32mf\u001b[39m" + ], + [ + 0.321772, + " " + ], + [ + 0.410311, + "-" + ], + [ + 0.160707, + "s" + ], + [ + 0.223167, + " " + ], + [ + 0.856546, + "\u001b[4mW\u001b[24m" + ], + [ + 0.184551, + "\b\u001b[4mW\u001b[4ma\u001b[24m" + ], + [ + 0.211734, + "\b\u001b[4ma\u001b[4ml\u001b[24m" + ], + [ + 0.115481, + "\b\u001b[4ml\u001b[4ml\u001b[24m" + ], + [ + 0.13804, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.007132, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mdiff\u001b[39m -s \u001b[4mWallpaper\u001b[24m\u001b[K" + ], + [ + 0.620064, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.004082, + "\r\r\u001b[19C" + ], + [ + 0.000148, + "\r\r\n\u001b[J\u001b[J\u001b[38;5;33mWallpaper\u001b[0m/ \u001b[J\u001b[38;5;33mWallpaper.orig\u001b[0m/\u001b[J\u001b[A\u001b[0m\u001b[27m\u001b[24m\r\u001b[2C\u001b[32mdiff\u001b[39m -s \u001b[4mWallpaper\u001b[24m\u001b[K" + ], + [ + 0.83944, + "\u001b[?7l" + ], + [ + 2.4e-05, + "\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.003487, + "\u001b[9D\u001b[24mW\u001b[24ma\u001b[24ml\u001b[24ml\u001b[24mp\u001b[24ma\u001b[24mp\u001b[24me\u001b[24mr\u001b[1m/\u001b[0m" + ], + [ + 0.000166, + "\r\r\n\u001b[J\u001b[7mWallpaper/ \u001b[0m \u001b[J\u001b[38;5;33mWallpaper.orig\u001b[0m/\u001b[J\u001b[A\u001b[0m\u001b[27m\u001b[24m\r\u001b[2C\u001b[32mdiff\u001b[39m -s Wallpaper\u001b[1m/\u001b[0m\u001b[K" + ], + [ + 0.488495, + "\r\r\n" + ], + [ + 1.6e-05, + "\u001b[7mWallpaper/ \u001b[0m \r\u001b[7mWallpaper/ \u001b[0m \r\u001b[A\u001b[0m\u001b[27m\u001b[24m\r\u001b[2C\u001b[32mdiff\u001b[39m -s Wallpaper\u001b[1m/\u001b[0m\u001b[K\r\r\n\u001b[J\u001b[A\u001b[20C" + ], + [ + 0.001959, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mdiff\u001b[39m -s \u001b[4mWallpaper\u001b[24m\u001b[K\u001b[1C" + ], + [ + 0.285593, + "\b" + ], + [ + 0.303988, + "\b\u001b[4mr\u001b[4m/\u001b[24m" + ], + [ + 0.798187, + "\b\u001b[4m/\u001b[4md\u001b[24m" + ], + [ + 0.241007, + "\b\u001b[4md\u001b[4me\u001b[24m" + ], + [ + 0.21286, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.00579, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mdiff\u001b[39m -s \u001b[4mWallpaper/deer.jpg\u001b[24m\u001b[1m \u001b[0m\u001b[K" + ], + [ + 1.289271, + "\b\u001b[0m \u001b[4mW\u001b[24m" + ], + [ + 0.148557, + "\b\u001b[4mW\u001b[4ma\u001b[24m" + ], + [ + 0.16621, + "\b\u001b[4ma\u001b[4ml\u001b[24m" + ], + [ + 0.097599, + "\b\u001b[4ml\u001b[4ml\u001b[24m" + ], + [ + 0.111176, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.005059, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mdiff\u001b[39m -s \u001b[4mWallpaper/deer.jpg\u001b[24m \u001b[4mWallpaper\u001b[24m\u001b[K" + ], + [ + 0.431538, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.005176, + "\r\r\u001b[38C" + ], + [ + 0.000155, + "\r\r\n\u001b[J\u001b[J\u001b[38;5;33mWallpaper\u001b[0m/ \u001b[J\u001b[38;5;33mWallpaper.orig\u001b[0m/\u001b[J\u001b[A\u001b[0m\u001b[27m\u001b[24m\r\u001b[2C\u001b[32mdiff\u001b[39m -s \u001b[4mWallpaper/deer.jpg\u001b[24m \u001b[4mWallpaper\u001b[24m\u001b[K" + ], + [ + 0.389092, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.004561, + "\u001b[9D\u001b[24mW\u001b[24ma\u001b[24ml\u001b[24ml\u001b[24mp\u001b[24ma\u001b[24mp\u001b[24me\u001b[24mr\u001b[1m/\u001b[0m" + ], + [ + 0.000155, + "\r\r\n\u001b[J\u001b[7mWallpaper/ \u001b[0m \u001b[J\u001b[38;5;33mWallpaper.orig\u001b[0m/\u001b[J\u001b[A\u001b[0m\u001b[27m\u001b[24m\r\u001b[2C" + ], + [ + 1.3e-05, + "\u001b[32mdiff\u001b[39m -s \u001b[4mWallpaper/deer.jpg\u001b[24m Wallpaper\u001b[1m/\u001b[0m\u001b[K" + ], + [ + 0.260844, + "\r\r\n" + ], + [ + 3.6e-05, + "\u001b[7mWallpaper/ \u001b[0m \r\u001b[7mWallpaper/ \u001b[0m \r\u001b[A\u001b[0m\u001b[27m\u001b[24m\r\u001b[2C\u001b[32mdiff\u001b[39m -s \u001b[4mWallpaper/deer.jpg\u001b[24m Wallpaper\u001b[1m/\u001b[0m\u001b[K\b\u001b[0m.orig\u001b[1m/\u001b[0m" + ], + [ + 0.000163, + "\r\r\n\u001b[17C\u001b[7mWallpaper.orig/\u001b[0m\r\u001b[38;5;33mWallpaper\u001b[0m/ \r\u001b[A\u001b[0m\u001b[27m\u001b[24m\r\u001b[2C\u001b[32mdiff\u001b[39m -s \u001b[4mWallpaper/deer.jpg\u001b[24m Wallpaper.orig\u001b[1m/\u001b[0m\u001b[K" + ], + [ + 0.598634, + "\r\r\n\u001b[J\u001b[A\u001b[44C" + ], + [ + 0.002461, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mdiff\u001b[39m -s \u001b[4mWallpaper/deer.jpg\u001b[24m \u001b[4mWallpaper.orig\u001b[24m\u001b[K\u001b[1C" + ], + [ + 0.275896, + "\b" + ], + [ + 0.321512, + "\b\u001b[4mg\u001b[4m/\u001b[24m" + ], + [ + 1.499007, + "\b\u001b[4m/\u001b[4md\u001b[24m" + ], + [ + 0.165243, + "\b\u001b[4md\u001b[4me\u001b[24m" + ], + [ + 0.260397, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.005274, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mdiff\u001b[39m -s \u001b[4mWallpaper/deer.jpg\u001b[24m \u001b[4mWallpaper.orig/deer.jpg\u001b[24m\u001b[1m \u001b[0m\u001b[K" + ], + [ + 1.658125, + "\b\u001b[0m \b" + ], + [ + 1.5e-05, + "\u001b[?1l\u001b>" + ], + [ + 0.001138, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000783, + "\u001b]2;diff -s Wallpaper/deer.jpg Wallpaper.orig/deer.jpg\u0007\u001b]1;diff\u0007" + ], + [ + 0.057035, + "Files Wallpaper/deer.jpg and Wallpaper.orig/deer.jpg are identical\r\n" + ], + [ + 0.000183, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.000114, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 9.1e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000199, + "\u001b[?2004h" + ], + [ + 3.579542, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.624347, + "\b\u001b[0m\u001b[39m \b" + ], + [ + 0.353186, + "\u001b[?1l\u001b>" + ], + [ + 0.000351, + "\u001b[?2004l\r\r\n" + ], + [ + 0.0006, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 2.1e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.00013, + "\u001b[?1h\u001b=" + ], + [ + 0.000185, + "\u001b[?2004h" + ], + [ + 0.726522, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.358332, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.183839, + "\b\b\u001b[1m#\u001b[1m \u001b[1mA\u001b[0m\u001b[39m" + ], + [ + 0.150451, + "\b\u001b[1mA\u001b[1mn\u001b[0m\u001b[39m" + ], + [ + 0.128839, + "\b\u001b[1mn\u001b[1md\u001b[0m\u001b[39m" + ], + [ + 0.583652, + "\b\u001b[1md\u001b[1m,\u001b[0m\u001b[39m" + ], + [ + 0.152149, + "\b\u001b[1m,\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.240696, + "\b\u001b[1m \u001b[1mo\u001b[0m\u001b[39m" + ], + [ + 0.130032, + "\b\u001b[1mo\u001b[1mf\u001b[0m\u001b[39m" + ], + [ + 0.306901, + "\b\u001b[1mf\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.181176, + "\b\u001b[1m \u001b[1mc\u001b[0m\u001b[39m" + ], + [ + 0.271007, + "\b\u001b[1mc\u001b[1mourse, we can also create remote repos via ssh when borg is setup\u001b[1m \u001b[1mthere. This command creates a new remote repo in a subdirectory called \"demo\"\u001b[1m:\u001b[0m\u001b[39m\u001b[K" + ], + [ + 2.040444, + "\u001b[?1l\u001b>" + ], + [ + 0.000423, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000711, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 6.8e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 6.8e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000297, + "\u001b[?2004h" + ], + [ + 1.613372, + "\u001b[1m\u001b[31mb\u001b[0m\u001b[39m" + ], + [ + 0.204618, + "\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[0m\u001b[39m" + ], + [ + 0.121257, + "\b\b\u001b[1m\u001b[31mb\u001b[1m\u001b[31mo\u001b[1m\u001b[31mr\u001b[0m\u001b[39m" + ], + [ + 0.228506, + "\b\b\b\u001b[0m\u001b[32mb\u001b[0m\u001b[32mo\u001b[0m\u001b[32mr\u001b[32mg\u001b[39m" + ], + [ + 0.469213, + " " + ], + [ + 0.23811, + "i" + ], + [ + 0.139149, + "n" + ], + [ + 0.157285, + "i" + ], + [ + 0.219101, + "t" + ], + [ + 0.389153, + " " + ], + [ + 0.633813, + "-" + ], + [ + 0.102895, + "-" + ], + [ + 0.267338, + "e" + ], + [ + 0.244036, + "n" + ], + [ + 0.303722, + "c" + ], + [ + 0.117325, + "r" + ], + [ + 0.112606, + "y" + ], + [ + 0.250891, + "p" + ], + [ + 0.258828, + "t" + ], + [ + 0.276877, + "i" + ], + [ + 0.131491, + "o" + ], + [ + 0.206852, + "n" + ], + [ + 0.966102, + "=" + ], + [ + 0.388021, + "r" + ], + [ + 0.146133, + "e" + ], + [ + 0.176939, + "p" + ], + [ + 0.139187, + "o" + ], + [ + 0.273188, + "k" + ], + [ + 0.172429, + "e" + ], + [ + 0.306306, + "y" + ], + [ + 0.851125, + " " + ], + [ + 0.868971, + "b" + ], + [ + 0.261136, + "o" + ], + [ + 0.12143, + "r" + ], + [ + 0.15507, + "g" + ], + [ + 0.186684, + "d" + ], + [ + 0.141974, + "e" + ], + [ + 0.13004, + "m" + ], + [ + 0.172673, + "o" + ], + [ + 1.041475, + "@" + ], + [ + 0.536019, + "r" + ], + [ + 0.02293, + "e" + ], + [ + 0.223755, + "m" + ], + [ + 0.152859, + "o" + ], + [ + 0.222368, + "t" + ], + [ + 0.095106, + "e" + ], + [ + 0.33914, + "s" + ], + [ + 0.213902, + "e" + ], + [ + 0.136448, + "r" + ], + [ + 0.196228, + "v" + ], + [ + 0.171447, + "e" + ], + [ + 0.154296, + "r" + ], + [ + 1.151168, + "." + ], + [ + 0.198973, + "e" + ], + [ + 0.195428, + "x" + ], + [ + 0.163512, + "a" + ], + [ + 0.157805, + "m" + ], + [ + 0.174865, + "p" + ], + [ + 0.103133, + "l" + ], + [ + 0.145276, + "e" + ], + [ + 2.109373, + ":" + ], + [ + 0.494126, + "." + ], + [ + 0.315325, + "/" + ], + [ + 0.182218, + "d" + ], + [ + 0.138815, + "e" + ], + [ + 0.143066, + "m" + ], + [ + 0.17136, + "o" + ], + [ + 1.831712, + "\u001b[?1l\u001b>" + ], + [ + 0.001025, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000824, + "\u001b]2;borg init --encryption=repokey borgdemo@remoteserver.example:./demo\u0007\u001b]1;borg\u0007" + ], + [ + 6.069586, + "Enter new passphrase: " + ], + [ + 2.598936, + "\r\n" + ], + [ + 0.000189, + "Enter same passphrase again: " + ], + [ + 2.044707, + "\r\n" + ], + [ + 0.000198, + "Do you want your passphrase to be displayed for verification? [yN]: " + ], + [ + 1.415539, + "\r\n" + ], + [ + 1.950077, + "\r\nBy default repositories initialized with this version will produce security\r\nerrors if written to with an older version (up to and including Borg 1.0.8).\r\n\r\nIf you want to use these older versions, you can disable the check by running:\r\nborg upgrade --disable-tam 'ssh://borgdemo@remoteserver.example/./demo'\r\n\r\nSee https://borgbackup.readthedocs.io/en/stable/changes.html#pre-1-0-9-manifest-spoofing-vulnerability for details about the security implications.\r\n" + ], + [ + 0.548386, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 9.5e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000105, + "\u001b[?1h\u001b=" + ], + [ + 0.000221, + "\u001b[?2004h" + ], + [ + 0.82377, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.662248, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.610999, + "\b\b\u001b[1m#\u001b[1m \u001b[1mE\u001b[0m\u001b[39m" + ], + [ + 0.267513, + "\b\u001b[1mE\u001b[1ma\u001b[0m\u001b[39m" + ], + [ + 0.185698, + "\b\u001b[1ma\u001b[1ms\u001b[0m\u001b[39m" + ], + [ + 0.161855, + "\b\u001b[1ms\u001b[1my\u001b[0m\u001b[39m" + ], + [ + 0.46273, + "\b\u001b[1my\u001b[1m, isn't it? That's all you need to know for basic usage.\u001b[0m\u001b[39m" + ], + [ + 1.861984, + "\u001b[?1l\u001b>" + ], + [ + 0.001044, + "\u001b[?2004l\r\r\n" + ], + [ + 0.001525, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 6.3e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 8e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000316, + "\u001b[?2004h" + ], + [ + 1.009133, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.240205, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.217287, + "\b\b\u001b[1m#\u001b[1m \u001b[1mI\u001b[0m\u001b[39m" + ], + [ + 0.163888, + "\b\u001b[1mI\u001b[1mf\u001b[0m\u001b[39m" + ], + [ + 0.349458, + "\b\u001b[1mf\u001b[1m you want to see more, have a look at the screencast showing the \"advanc\u001b[1me\u001b[1md usage\".\u001b[0m\u001b[39m\u001b[K" + ], + [ + 2.780664, + "\u001b[?1l\u001b>" + ], + [ + 0.000734, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000812, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 4.4e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000113, + "\u001b[?1h\u001b=" + ], + [ + 0.000299, + "\u001b[?2004h" + ], + [ + 1.119856, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.281915, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.244389, + "\b\b\u001b[1m#\u001b[1m \u001b[1mI\u001b[0m\u001b[39m" + ], + [ + 0.143064, + "\b\u001b[1mI\u001b[1mn\u001b[0m\u001b[39m" + ], + [ + 0.171731, + "\b\u001b[1mn\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.139438, + "\b\u001b[1m \u001b[1ma\u001b[0m\u001b[39m" + ], + [ + 0.388834, + "\b\u001b[1ma\u001b[1mny case, enjoy using borg!\u001b[0m\u001b[39m" + ], + [ + 1.502218, + "\u001b[?1l\u001b>" + ], + [ + 0.000883, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000735, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 9.9e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000113, + "\u001b[?1h\u001b=" + ], + [ + 0.000498, + "\u001b[?2004h" + ], + [ + 1.273251, + "\u001b[?2004l\r\r\n" + ] + ] +} diff --git a/docs/misc/asciinema/basic.sh b/docs/misc/asciinema/basic.sh new file mode 100644 index 0000000000..743a506d2a --- /dev/null +++ b/docs/misc/asciinema/basic.sh @@ -0,0 +1,53 @@ +# Here you'll see some basic commands to start working with borg. +# Note: This teaser screencast was made with borg version 1.1.0 – older or newer borg versions may behave differently. +# But let's start. + +# First of all, you can always get help: +borg help +# These are a lot of commands, so better we start with a few: +# Let's create a repo on an external drive… +borg init --encryption=repokey /media/backup/borgdemo +# This uses the repokey encryption. You may look at "borg help init" or the online doc at https://borgbackup.readthedocs.io/ for other modes. + +# So now, let's create our first (compressed) backup. +borg create --stats --progress --compression lz4 /media/backup/borgdemo::backup1 Wallpaper + +# That's nice, so far. +# So let's add a new file… +echo "new nice file" > Wallpaper/newfile.txt + + +borg create --stats --progress --compression lz4 /media/backup/borgdemo::backup2 Wallpaper + +# Wow, this was a lot faster! +# Notice the "Deduplicated size" for "This archive"! +# Borg recognized that most files did not change and deduplicated them. + +# But what happens, when we move a dir and create a new backup? +mv … + +borg create --stats --progress --compression lz4 /media/backup/borgdemo::backup3 Wallpaper + +# Still quite fast… +# But when you look at the "deduplicated file size" again, you see that borg also recognized that only the dir and not the files changed in this backup. + +# Now lets look into a repo. +borg list /media/backup/borgdemo + +# You'll see a list of all backups. +# You can also use the same command to look into an archive. But we better filter the output here: +borg list /media/backup/borgdemo::backup3 | grep 'deer.jpg' + +# Oh, we found our picture. Now extract it… +mv Wallpaper Wallpaper.orig +borg extract /media/backup/borgdemo::backup3 + +# And check that it's the same: +diff -s Wallpaper/deer.jpg Wallpaper.orig/deer.jpg + +# And, of course, we can also create remote repos via ssh when borg is setup there. This command creates a new remote repo in a subdirectory called "demo": +borg init --encryption=repokey borgdemo@remoteserver.example:./demo + +# Easy, isn't it? That's all you need to know for basic usage. +# If you want to see more, have a look at the screencast showing the "advanced usage". +# In any case, enjoy using borg! diff --git a/docs/misc/asciinema/install.json b/docs/misc/asciinema/install.json new file mode 100644 index 0000000000..477b1ef276 --- /dev/null +++ b/docs/misc/asciinema/install.json @@ -0,0 +1,1354 @@ +{ + "version": 1, + "width": 78, + "height": 25, + "duration": 140.275038, + "command": null, + "title": null, + "env": { + "TERM": "xterm-256color", + "SHELL": "/bin/zsh" + }, + "stdout": [ + [ + 9.1e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000108, + "\u001b[?1h\u001b=" + ], + [ + 0.000182, + "\u001b[?2004h" + ], + [ + 0.45774, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.31515, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.220208, + "\b\b\u001b[1m#\u001b[1m \u001b[1mT\u001b[0m\u001b[39m" + ], + [ + 0.121752, + "\b\u001b[1mT\u001b[1mh\u001b[0m\u001b[39m" + ], + [ + 0.142781, + "\b\u001b[1mh\u001b[1mi\u001b[0m\u001b[39m" + ], + [ + 0.117367, + "\b\u001b[1mi\u001b[1ms\u001b[0m\u001b[39m" + ], + [ + 0.255471, + "\b\u001b[1ms\u001b[1m asciinema will show you the installation of borg as a standalone bina\u001b[1mr\u001b[1my. Usually you only need this if you want to have an up-to-date version of bo\u001b[1mr\u001b[1mg or no package is available for your distro/OS.\u001b[0m\u001b[39m\u001b[K" + ], + [ + 0.563803, + "\u001b[?1l\u001b>" + ], + [ + 0.000412, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000823, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 5.9e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 5.2e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.00027, + "\u001b[?2004h" + ], + [ + 2.191111, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.301924, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.21419, + "\b\b\u001b[1m#\u001b[1m \u001b[1mF\u001b[0m\u001b[39m" + ], + [ + 0.117654, + "\b\u001b[1mF\u001b[1mi\u001b[0m\u001b[39m" + ], + [ + 0.198616, + "\b\u001b[1mi\u001b[1mr\u001b[0m\u001b[39m" + ], + [ + 0.101113, + "\b\u001b[1mr\u001b[1ms\u001b[0m\u001b[39m" + ], + [ + 0.107485, + "\b\u001b[1ms\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.357443, + "\b\u001b[1mt\u001b[1m, we need to download the version, we'd like to install…\u001b[0m\u001b[39m" + ], + [ + 0.516614, + "\u001b[?1l\u001b>" + ], + [ + 0.000826, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000757, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 4.4e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000134, + "\u001b[?1h\u001b=" + ], + [ + 0.000598, + "\u001b[?2004h" + ], + [ + 1.411874, + "\u001b[32mw\u001b[39m" + ], + [ + 0.119593, + "\b\u001b[1m\u001b[31mw\u001b[1m\u001b[31mg\u001b[0m\u001b[39m" + ], + [ + 0.13329, + "\b\b\u001b[1m\u001b[31mw\u001b[1m\u001b[31mg\u001b[1m\u001b[31me\u001b[0m\u001b[39m" + ], + [ + 0.127861, + "\b\b\b\u001b[0m\u001b[32mw\u001b[0m\u001b[32mg\u001b[0m\u001b[32me\u001b[32mt\u001b[39m" + ], + [ + 0.324708, + " -q --show-progress https://github.com/borgbackup/borg/releases/download/1.1.0b6/borg-linux64\u001b[K" + ], + [ + 0.797801, + "\u001b[?1l\u001b>" + ], + [ + 0.000964, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000631, + "\u001b]2;wget -q --show-progress \u0007\u001b]1;wget\u0007" + ], + [ + 1.306534, + "\rborg-linux64 0%[ ] 0 --.-KB/s " + ], + [ + 0.23185, + "\rborg-linux64 0%[ ] 24.58K 106KB/s " + ], + [ + 0.341907, + "\rborg-linux64 0%[ ] 92.58K 161KB/s " + ], + [ + 0.230021, + "\rborg-linux64 1%[ ] 160.58K 200KB/s " + ], + [ + 0.22577, + "\rborg-linux64 1%[ ] 211.58K 206KB/s " + ], + [ + 0.229246, + "\rborg-linux64 2%[ ] 279.58K 222KB/s " + ], + [ + 0.347713, + "\rborg-linux64 2%[ ] 347.58K 216KB/s " + ], + [ + 0.224636, + "\rborg-linux64 98%[================> ] 12.41M 404KB/s eta 2s " + ], + [ + 0.205977, + "\rborg-linux64 99%[================> ] 12.50M 401KB/s eta 0s " + ], + [ + 0.137036, + "\rborg-linux64 100%[=================>] 12.56M 417KB/s in 39s \r\n" + ], + [ + 0.000872, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.000103, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 0.000117, + "\u001b[?1h\u001b=" + ], + [ + 0.000208, + "\u001b[?2004h" + ], + [ + 2.118269, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.266901, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.142975, + "\b\b\u001b[1m#\u001b[1m \u001b[1ma\u001b[0m\u001b[39m" + ], + [ + 0.074155, + "\b\u001b[1ma\u001b[1mn\u001b[0m\u001b[39m" + ], + [ + 0.167144, + "\b\u001b[1mn\u001b[1md\u001b[0m\u001b[39m" + ], + [ + 0.2241, + "\b\u001b[1md\u001b[1m do not forget the GPG signature…!\u001b[0m\u001b[39m" + ], + [ + 0.596854, + "\u001b[?1l\u001b>" + ], + [ + 0.000696, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000691, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 8.2e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 3e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000286, + "\u001b[?2004h" + ], + [ + 1.51737, + "\u001b[1m# and do not forget the GPG signature…!\u001b[0m\u001b[39m" + ], + [ + 0.314759, + "\u001b[39D\u001b[0m\u001b[32mw\u001b[0m\u001b[32mg\u001b[0m\u001b[32me\u001b[0m\u001b[32mt\u001b[39m\u001b[0m\u001b[39m \u001b[0m\u001b[39m-\u001b[0m\u001b[39mq\u001b[0m\u001b[39m \u001b[0m\u001b[39m-\u001b[0m\u001b[39m-\u001b[0m\u001b[39ms\u001b[0m\u001b[39mh\u001b[0m\u001b[39mo\u001b[0m\u001b[39mw\u001b[0m\u001b[39m-\u001b[0m\u001b[39mp\u001b[0m\u001b[39mr\u001b[0m\u001b[39mo\u001b[0m\u001b[39mg\u001b[0m\u001b[39mr\u001b[0m\u001b[39me\u001b[0m\u001b[39ms\u001b[0m\u001b[39ms\u001b[0m\u001b[39m \u001b[0m\u001b[39mh\u001b[0m\u001b[39mt\u001b[0m\u001b[39mt\u001b[0m\u001b[39mp\u001b[0m\u001b[39ms\u001b[0m\u001b[39m:\u001b[0m\u001b[39m/\u001b[0m\u001b[39m/\u001b[0m\u001b[39mg\u001b[0m\u001b[39mi\u001b[0m\u001b[39mt\u001b[0m\u001b[39mh\u001b[0m\u001b[39mu\u001b[0m\u001b[39mb\u001b[0m\u001b[39m.com/borgbackup/borg/releases/download/1.1.0b6/borg-linux64\u001b[K" + ], + [ + 1.043903, + "." + ], + [ + 0.207322, + "a" + ], + [ + 0.16952, + "s" + ], + [ + 0.19625, + "c" + ], + [ + 0.359073, + "\u001b[?1l\u001b>" + ], + [ + 0.001424, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000717, + "\u001b]2;wget -q --show-progress \u0007\u001b]1;wget\u0007" + ], + [ + 1.236785, + "\rborg-linux64.asc 0%[ ] 0 --.-KB/s " + ], + [ + 1.8e-05, + "\rborg-linux64.asc 100%[=================>] 819 --.-KB/s in 0s \r\n" + ], + [ + 0.00093, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 7.7e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 8.7e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000183, + "\u001b[?2004h" + ], + [ + 3.234458, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 1.023301, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.331266, + "\b\b\u001b[1m#\u001b[1m \u001b[1mI\u001b[0m\u001b[39m" + ], + [ + 0.166799, + "\b\u001b[1mI\u001b[1mn\u001b[0m\u001b[39m" + ], + [ + 0.34554, + "\b\u001b[1mn\u001b[1m this case, we have already imported the public key of a borg developer.\u001b[1m \u001b[1mSo we only need to verify it:\u001b[0m\u001b[39m\u001b[K" + ], + [ + 1.499971, + "\u001b[?1l\u001b>" + ], + [ + 0.001069, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000922, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.000159, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 3.1e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000451, + "\u001b[?2004h" + ], + [ + 2.31724, + "\u001b[32mg\u001b[39m" + ], + [ + 0.151243, + "\b\u001b[32mg\u001b[32mp\u001b[39m" + ], + [ + 0.074305, + "\b\b\u001b[32mg\u001b[32mp\u001b[32mg\u001b[39m" + ], + [ + 0.315686, + " " + ], + [ + 0.345624, + "-" + ], + [ + 0.100203, + "-" + ], + [ + 0.291673, + "v" + ], + [ + 0.11497, + "e" + ], + [ + 0.183055, + "r" + ], + [ + 0.146521, + "i" + ], + [ + 0.11872, + "f" + ], + [ + 0.309865, + "y" + ], + [ + 0.346758, + " " + ], + [ + 0.264902, + "\u001b[4mb\u001b[24m" + ], + [ + 0.307683, + "\u001b[?7l" + ], + [ + 2.1e-05, + "\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.011212, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mgpg\u001b[39m --verify \u001b[4mborg-linux64\u001b[24m\u001b[K" + ], + [ + 0.577848, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.015636, + "\r\r\u001b[27C" + ], + [ + 0.000193, + "\r\r\n\u001b[J" + ], + [ + 2e-05, + "\u001b[J\u001b[0mborg-linux64 \u001b[Jborg-linux64.asc\u001b[J\u001b[A\u001b[0m\u001b[27m\u001b[24m\r\u001b[2C\u001b[32mgpg\u001b[39m --verify \u001b[4mborg-linux64\u001b[24m\u001b[K" + ], + [ + 0.626316, + "\u001b[?7l\u001b[31m......\u001b[39m\u001b[?7h" + ], + [ + 0.012642, + "\u001b[12D\u001b[24mb\u001b[24mo\u001b[24mr\u001b[24mg\u001b[24m-\u001b[24ml\u001b[24mi\u001b[24mn\u001b[24mu\u001b[24mx\u001b[24m6\u001b[24m4" + ], + [ + 0.000154, + "\r\r\n" + ], + [ + 1.8e-05, + "\u001b[J\u001b[7mborg-linux64 \u001b[0m \u001b[Jborg-linux64.asc\u001b[J\u001b[A\u001b[0m\u001b[27m\u001b[24m\r\u001b[2C\u001b[32mgpg\u001b[39m --verify borg-linux64\u001b[K" + ], + [ + 0.189964, + "\r\r\n" + ], + [ + 1.7e-05, + "\u001b[7mborg-linux64 \u001b[0m \r\u001b[7mborg-linux64 \u001b[0m \r\u001b[A\u001b[0m\u001b[27m\u001b[24m\r\u001b[2C\u001b[32mgpg\u001b[39m --verify borg-linux64\u001b[K.asc" + ], + [ + 0.000225, + "\r\r\n" + ], + [ + 1.9e-05, + "\u001b[18C\u001b[7mborg-linux64.asc\u001b[0m\rborg-linux64 \r\u001b[A\u001b[0m\u001b[27m\u001b[24m\r\u001b[2C\u001b[32mgpg\u001b[39m --verify borg-linux64.asc\u001b[K" + ], + [ + 0.866638, + "\r\r\n\u001b[J\u001b[A\u001b[31C\u001b[1m \u001b[0m" + ], + [ + 0.001241, + "\r\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[32mgpg\u001b[39m --verify \u001b[4mborg-linux64.asc\u001b[24m\u001b[1m \u001b[0m\u001b[K" + ], + [ + 0.654098, + "\b\u001b[0m \b" + ], + [ + 2.7e-05, + "\u001b[?1l\u001b>" + ], + [ + 0.001361, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000737, + "\u001b]2;gpg --verify borg-linux64.asc\u0007" + ], + [ + 2.6e-05, + "\u001b]1;gpg\u0007" + ], + [ + 0.002478, + "gpg: assuming signed data in `borg-linux64'\r\n" + ], + [ + 0.082679, + "gpg: Signature made Sun Jun 18 16:54:19 2017 CEST\r\ngpg: using RSA key 0x243ACFA951F78E01\r\n" + ], + [ + 0.003947, + "gpg: Good signature from \"Thomas Waldmann \" [ultimate]\r\ngpg: aka \"Thomas Waldmann \" [ultimate]\r\ngpg: aka \"Thomas Waldmann \" [ultimate]\r\n" + ], + [ + 2.1e-05, + "gpg: aka \"Thomas Waldmann \"" + ], + [ + 1.5e-05, + " [ultimate]\r\n" + ], + [ + 0.001743, + "Primary key fingerprint: 6D5B EF9A DD20 7580 5747 B70F 9F88 FB52 FAF7 B393\r\n Subkey fingerprint: 2F81 AFFB AB04 E11F E8EE 65D4 243A CFA9 51F7 8E01\r\n" + ], + [ + 0.000384, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 9e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 9e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000155, + "\u001b[?2004h" + ], + [ + 4.627219, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.225001, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.213579, + "\b\b\u001b[1m#\u001b[1m \u001b[1mO\u001b[0m\u001b[39m" + ], + [ + 0.132218, + "\b\u001b[1mO\u001b[1mk\u001b[0m\u001b[39m" + ], + [ + 0.061577, + "\b\u001b[1mk\u001b[1ma\u001b[0m\u001b[39m" + ], + [ + 0.154786, + "\b\u001b[1ma\u001b[1my\u001b[0m\u001b[39m" + ], + [ + 0.172921, + "\b\u001b[1my\u001b[1m,\u001b[0m\u001b[39m" + ], + [ + 0.648978, + "\b\u001b[1m,\u001b[1m the binary is valid!\u001b[0m\u001b[39m" + ], + [ + 0.822303, + "\u001b[?1l\u001b>" + ], + [ + 0.000388, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000681, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.000113, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 4.6e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000252, + "\u001b[?2004h" + ], + [ + 2.048081, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.243659, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.174242, + "\b\b\u001b[1m#\u001b[1m \u001b[1mN\u001b[0m\u001b[39m" + ], + [ + 0.131485, + "\b\u001b[1mN\u001b[1mo\u001b[0m\u001b[39m" + ], + [ + 0.109555, + "\b\u001b[1mo\u001b[1mw\u001b[0m\u001b[39m" + ], + [ + 0.128309, + "\b\u001b[1mw\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.163064, + "\b\u001b[1m \u001b[1mi\u001b[0m\u001b[39m" + ], + [ + 0.138953, + "\b\u001b[1mi\u001b[1mn\u001b[0m\u001b[39m" + ], + [ + 0.050135, + "\b\u001b[1mn\u001b[1ms\u001b[0m\u001b[39m" + ], + [ + 0.095385, + "\b\u001b[1ms\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.114692, + "\b\u001b[1mt\u001b[1ma\u001b[0m\u001b[39m" + ], + [ + 0.155821, + "\b\u001b[1ma\u001b[1ml\u001b[0m\u001b[39m" + ], + [ + 0.118297, + "\b\u001b[1ml\u001b[1ml\u001b[0m\u001b[39m" + ], + [ + 0.165834, + "\b\u001b[1ml\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.231866, + "\b\u001b[1m \u001b[1mi\u001b[0m\u001b[39m" + ], + [ + 0.159893, + "\b\u001b[1mi\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.289328, + "\b\u001b[1mt\u001b[1m:\u001b[0m\u001b[39m" + ], + [ + 2.713706, + "\u001b[?1l\u001b>" + ], + [ + 0.000362, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000674, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 9e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 5.5e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000272, + "\u001b[?2004h" + ], + [ + 1.703796, + "\u001b[1m\u001b[31ms\u001b[0m\u001b[39m" + ], + [ + 0.12754, + "\b\u001b[0m\u001b[32ms\u001b[32mu\u001b[39m" + ], + [ + 0.149508, + "\b\b\u001b[1m\u001b[31ms\u001b[1m\u001b[31mu\u001b[1m\u001b[31md\u001b[0m\u001b[39m" + ], + [ + 0.121616, + "\b\b\b\u001b[0m\u001b[4m\u001b[32ms\u001b[0m\u001b[4m\u001b[32mu\u001b[0m\u001b[4m\u001b[32md\u001b[4m\u001b[32mo\u001b[24m\u001b[39m" + ], + [ + 0.321903, + " \u001b[32mcp\u001b[39m \u001b[4mborg-linux64\u001b[24m \u001b[4m/usr/local/bin/borg\u001b[24m" + ], + [ + 2.352378, + "\u001b[?1l\u001b>" + ], + [ + 0.001087, + "\u001b[?2004l\r\r\n" + ], + [ + 0.00091, + "\u001b]2;sudo cp borg-linux64 /usr/local/bin/borg\u0007\u001b]1;cp\u0007" + ], + [ + 0.013652, + "[sudo] password for rugk: " + ], + [ + 2.992379, + "\r\n" + ], + [ + 0.031173, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 4e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 6.3e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000214, + "\u001b[?2004h" + ], + [ + 5.400881, + "\u001b[1m\u001b[31ms\u001b[0m\u001b[39m" + ], + [ + 0.138474, + "\b\u001b[0m\u001b[32ms\u001b[32mu\u001b[39m" + ], + [ + 0.114266, + "\b\b\u001b[1m\u001b[31ms\u001b[1m\u001b[31mu\u001b[1m\u001b[31md\u001b[0m\u001b[39m" + ], + [ + 0.098068, + "\b\b\b\u001b[0m\u001b[4m\u001b[32ms\u001b[0m\u001b[4m\u001b[32mu\u001b[0m\u001b[4m\u001b[32md\u001b[4m\u001b[32mo\u001b[24m\u001b[39m" + ], + [ + 0.16926, + " " + ], + [ + 0.188874, + "\u001b[32mcp\u001b[39m \u001b[4mborg-linux64\u001b[24m \u001b[4m/usr/local/bin/borg\u001b[24m" + ], + [ + 0.413244, + "\u001b[34D\u001b[32mh\u001b[32mo\u001b[24m\u001b[32mw\u001b[24m\u001b[32mn\u001b[39m\u001b[24m \u001b[24mr\u001b[24mo\u001b[24mo\u001b[24mt\u001b[24m:\u001b[24mr\u001b[24mo\u001b[24mo\u001b[24mt\u001b[20C" + ], + [ + 1.397429, + "\u001b[?1l\u001b>" + ], + [ + 0.00132, + "\u001b[?2004l\r\r\n" + ], + [ + 0.00075, + "\u001b]2;sudo chown root:root /usr/local/bin/borg\u0007\u001b]1;chown\u0007" + ], + [ + 0.010539, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 5.7e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 6.8e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000284, + "\u001b[?2004h" + ], + [ + 2.229436, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.213191, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.099902, + "\b\b\u001b[1m#\u001b[1m \u001b[1ma\u001b[0m\u001b[39m" + ], + [ + 0.069437, + "\b\u001b[1ma\u001b[1mn\u001b[0m\u001b[39m" + ], + [ + 0.106463, + "\b\u001b[1mn\u001b[1md\u001b[0m\u001b[39m" + ], + [ + 0.080598, + "\b\u001b[1md\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.172381, + "\b\u001b[1m \u001b[1mm\u001b[0m\u001b[39m" + ], + [ + 0.096638, + "\b\u001b[1mm\u001b[1ma\u001b[0m\u001b[39m" + ], + [ + 0.078606, + "\b\u001b[1ma\u001b[1mk\u001b[0m\u001b[39m" + ], + [ + 0.106382, + "\b\u001b[1mk\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.110174, + "\b\u001b[1me\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.216964, + "\b\u001b[1m \u001b[1mi\u001b[0m\u001b[39m" + ], + [ + 0.183739, + "\b\u001b[1mi\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.150872, + "\b\u001b[1mt\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.188901, + "\b\u001b[1m \u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.099651, + "\b\u001b[1me\u001b[1mx\u001b[0m\u001b[39m" + ], + [ + 0.1893, + "\b\u001b[1mx\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.187999, + "\b\u001b[1me\u001b[1mc\u001b[0m\u001b[39m" + ], + [ + 0.128262, + "\b\u001b[1mc\u001b[1mu\u001b[0m\u001b[39m" + ], + [ + 0.144851, + "\b\u001b[1mu\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.091175, + "\b\u001b[1mt\u001b[1ma\u001b[0m\u001b[39m" + ], + [ + 0.135575, + "\b\u001b[1ma\u001b[1mb\u001b[0m\u001b[39m" + ], + [ + 0.18045, + "\b\u001b[1mb\u001b[1ml\u001b[0m\u001b[39m" + ], + [ + 0.110687, + "\b\u001b[1ml\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.360861, + "\b\u001b[1me\u001b[1m…\u001b[0m\u001b[39m" + ], + [ + 0.69896, + "\u001b[?1l\u001b>" + ], + [ + 0.000433, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000544, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 5.1e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 6.8e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000306, + "\u001b[?2004h" + ], + [ + 1.028139, + "\u001b[1m\u001b[31ms\u001b[0m\u001b[39m" + ], + [ + 0.136555, + "\b\u001b[0m\u001b[32ms\u001b[32mu\u001b[39m" + ], + [ + 0.115701, + "\b\b\u001b[1m\u001b[31ms\u001b[1m\u001b[31mu\u001b[1m\u001b[31md\u001b[0m\u001b[39m" + ], + [ + 0.151048, + "\b\b\b\u001b[0m\u001b[4m\u001b[32ms\u001b[0m\u001b[4m\u001b[32mu\u001b[0m\u001b[4m\u001b[32md\u001b[4m\u001b[32mo\u001b[24m\u001b[39m" + ], + [ + 0.276036, + " \u001b[32mchown\u001b[39m root:root \u001b[4m/usr/local/bin/borg\u001b[24m" + ], + [ + 0.284509, + "\u001b[34D\u001b[32mp\u001b[39m\u001b[39m \u001b[39m\u001b[4mb\u001b[39m\u001b[4mo\u001b[4mr\u001b[4mg\u001b[4m-\u001b[4ml\u001b[4mi\u001b[4mn\u001b[4mu\u001b[4mx\u001b[4m6\u001b[4m4\u001b[24m\u001b[20C" + ], + [ + 0.422112, + "\u001b[34D\u001b[32mh\u001b[32mo\u001b[24m\u001b[32mw\u001b[24m\u001b[32mn\u001b[39m\u001b[24m \u001b[24mr\u001b[24mo\u001b[24mo\u001b[24mt\u001b[24m:\u001b[24mr\u001b[24mo\u001b[24mo\u001b[24mt\u001b[20C" + ], + [ + 0.616462, + "\u001b[33D\u001b[32mm\u001b[32mo\u001b[32md\u001b[39m 755\u001b[6P\u001b[20C \b\b\b\b\b\b" + ], + [ + 1.090337, + "\u001b[?1l\u001b>" + ], + [ + 0.00101, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000655, + "\u001b]2;sudo chmod 755 /usr/local/bin/borg\u0007" + ], + [ + 1.8e-05, + "\u001b]1;chmod\u0007" + ], + [ + 0.009932, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.000124, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 5.9e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000237, + "\u001b[?2004h" + ], + [ + 3.613554, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.305561, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.451533, + "\b\b\u001b[1m#\u001b[1m \u001b[1mN\u001b[0m\u001b[39m" + ], + [ + 0.199295, + "\b\u001b[1mN\u001b[1mo\u001b[0m\u001b[39m" + ], + [ + 0.134017, + "\b\u001b[1mo\u001b[1mw\u001b[0m\u001b[39m" + ], + [ + 0.232574, + "\b\u001b[1mw\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.283449, + "\b\u001b[1m \u001b[1mc\u001b[0m\u001b[39m" + ], + [ + 0.156927, + "\b\u001b[1mc\u001b[1mh\u001b[0m\u001b[39m" + ], + [ + 0.100718, + "\b\u001b[1mh\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.145048, + "\b\u001b[1me\u001b[1mc\u001b[0m\u001b[39m" + ], + [ + 0.238223, + "\b\u001b[1mc\u001b[1mk\u001b[0m\u001b[39m" + ], + [ + 0.145393, + "\b\u001b[1mk\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.195514, + "\b\u001b[1m \u001b[1mi\u001b[0m\u001b[39m" + ], + [ + 0.190153, + "\b\u001b[1mi\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 1.202922, + "\b\u001b[1mt\u001b[1m:\u001b[0m\u001b[39m" + ], + [ + 0.17572, + "\b\u001b[1m:\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.209752, + "\b\u001b[1m \u001b[1m(\u001b[0m\u001b[39m" + ], + [ + 0.266264, + "\b\u001b[1m(\u001b[1mp\u001b[0m\u001b[39m" + ], + [ + 0.136174, + "\b\u001b[1mp\u001b[1mo\u001b[0m\u001b[39m" + ], + [ + 0.136549, + "\b\u001b[1mo\u001b[1ms\u001b[0m\u001b[39m" + ], + [ + 0.157321, + "\b\u001b[1ms\u001b[1ms\u001b[0m\u001b[39m" + ], + [ + 0.134812, + "\b\u001b[1ms\u001b[1mi\u001b[0m\u001b[39m" + ], + [ + 0.177707, + "\b\u001b[1mi\u001b[1mb\u001b[0m\u001b[39m" + ], + [ + 0.184458, + "\b\u001b[1mb\u001b[1ml\u001b[0m\u001b[39m" + ], + [ + 0.104718, + "\b\u001b[1ml\u001b[1my\u001b[0m\u001b[39m" + ], + [ + 0.132476, + "\b\u001b[1my\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.14269, + "\b\u001b[1m \u001b[1mn\u001b[0m\u001b[39m" + ], + [ + 0.109627, + "\b\u001b[1mn\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.150487, + "\b\u001b[1me\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.202663, + "\b\u001b[1me\u001b[1md\u001b[0m\u001b[39m" + ], + [ + 0.12975, + "\b\u001b[1md\u001b[1ms\u001b[0m\u001b[39m" + ], + [ + 0.095469, + "\b\u001b[1ms\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.160511, + "\b\u001b[1m \u001b[1ma\u001b[0m\u001b[39m" + ], + [ + 0.149495, + "\b\u001b[1ma\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.189727, + "\b\u001b[1m \u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.098768, + "\b\u001b[1mt\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.125099, + "\b\u001b[1me\u001b[1mr\u001b[0m\u001b[39m" + ], + [ + 0.077112, + "\b\u001b[1mr\u001b[1mm\u001b[0m\u001b[39m" + ], + [ + 0.147886, + "\b\u001b[1mm\u001b[1mi\u001b[0m\u001b[39m" + ], + [ + 0.124366, + "\b\u001b[1mi\u001b[1mn\u001b[0m\u001b[39m" + ], + [ + 0.088118, + "\b\u001b[1mn\u001b[1ma\u001b[0m\u001b[39m" + ], + [ + 0.116281, + "\b\u001b[1ma\u001b[1ml\u001b[0m\u001b[39m" + ], + [ + 0.146487, + "\b\u001b[1ml\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.156764, + "\b\u001b[1m \u001b[1mr\u001b[0m\u001b[39m" + ], + [ + 0.195688, + "\b\u001b[1mr\u001b[1me\u001b[0m\u001b[39m" + ], + [ + 0.40621, + "\b\u001b[1me\u001b[1ms\u001b[0m\u001b[39m" + ], + [ + 0.263813, + "\b\u001b[1ms\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.087475, + "\b\u001b[1mt\u001b[1ma\u001b[0m\u001b[39m" + ], + [ + 0.090176, + "\b\u001b[1ma\u001b[1mr\u001b[0m\u001b[39m" + ], + [ + 0.12059, + "\b\u001b[1mr\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.232423, + "\b\u001b[1mt\u001b[1m)\u001b[0m\u001b[39m" + ], + [ + 1.383975, + "\u001b[?1l\u001b>" + ], + [ + 0.000692, + "\u001b[?2004l\r\r\n" + ], + [ + 0.001339, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.000147, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 6.6e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000558, + "\u001b[?2004h" + ], + [ + 1.045406, + "\u001b[4mb\u001b[24m" + ], + [ + 0.163217, + "\b\u001b[4mb\u001b[4mo\u001b[24m" + ], + [ + 0.131464, + "\b\b\u001b[4mb\u001b[4mo\u001b[4mr\u001b[24m" + ], + [ + 0.103279, + "\b\b\b\u001b[24m\u001b[32mb\u001b[24m\u001b[32mo\u001b[24m\u001b[32mr\u001b[32mg\u001b[39m" + ], + [ + 0.181118, + " " + ], + [ + 0.440449, + "-" + ], + [ + 0.186299, + "V" + ], + [ + 0.522054, + "\u001b[?1l\u001b>" + ], + [ + 0.000643, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000967, + "\u001b]2;borg -V\u0007\u001b]1;borg\u0007" + ], + [ + 0.426128, + "borg 1.1.0b6\r\n" + ], + [ + 0.040916, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 0.000101, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 6.5e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000176, + "\u001b[?2004h" + ], + [ + 1.92655, + "\u001b[1m#\u001b[0m\u001b[39m" + ], + [ + 0.247681, + "\b\u001b[1m#\u001b[1m \u001b[0m\u001b[39m" + ], + [ + 0.233391, + "\b\b\u001b[1m#\u001b[1m \u001b[1mT\u001b[0m\u001b[39m" + ], + [ + 0.127191, + "\b\u001b[1mT\u001b[1mh\u001b[0m\u001b[39m" + ], + [ + 0.023053, + "\b\u001b[1mh\u001b[1ma\u001b[0m\u001b[39m" + ], + [ + 0.155649, + "\b\u001b[1ma\u001b[1mt\u001b[0m\u001b[39m" + ], + [ + 0.3483, + "\b\u001b[1mt\u001b[1m's it! Check out the other screencasts to see how to actually use borg\u001b[1mb\u001b[1mackup.\u001b[0m\u001b[39m\u001b[K" + ], + [ + 1.701253, + "\u001b[?1l\u001b>" + ], + [ + 0.000707, + "\u001b[?2004l\r\r\n" + ], + [ + 0.000682, + "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r" + ], + [ + 1.1e-05, + "\u001b]1;..lder/borgdemo\u0007" + ], + [ + 5.7e-05, + "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J$ \u001b[K" + ], + [ + 6.8e-05, + "\u001b[?1h\u001b=" + ], + [ + 0.000284, + "\u001b[?2004h" + ], + [ + 1.579085, + "\u001b[?2004l\r\r\n" + ] + ] +} diff --git a/docs/misc/asciinema/install.sh b/docs/misc/asciinema/install.sh new file mode 100644 index 0000000000..3ade47c889 --- /dev/null +++ b/docs/misc/asciinema/install.sh @@ -0,0 +1,21 @@ +# This asciinema will show you the installation of borg as a standalone binary. Usually you only need this if you want to have an up-to-date version of borg or no package is available for your distro/OS. + +# First, we need to download the version, we'd like to install… +wget -q --show-progress https://github.com/borgbackup/borg/releases/download/1.1.0b6/borg-linux64 +# and do not forget the GPG signature…! +wget -q --show-progress https://github.com/borgbackup/borg/releases/download/1.1.0b6/borg-linux64.asc + +# In this case, we have already imported the public key of a borg developer. So we only need to verify it: +gpg --verify borg-linux64.asc +# Okay, the binary is valid! + +# Now install it: +sudo cp borg-linux64 /usr/local/bin/borg +sudo chown root:root /usr/local/bin/borg +# and make it executable… +sudo chmod 755 /usr/local/bin/borg + +# Now check it: (possibly needs a terminal restart) +borg -V + +# That's it! Check out the other screencasts to see how to actually use borgbackup. diff --git a/docs/misc/asciinema/install_and_basics.json b/docs/misc/asciinema/install_and_basics.json deleted file mode 100644 index a50eb35f1a..0000000000 --- a/docs/misc/asciinema/install_and_basics.json +++ /dev/null @@ -1,5550 +0,0 @@ -{ - "version": 1, - "width": 80, - "height": 24, - "duration": 332.0, - "command": "/bin/bash", - "title": "borgbackup - installation and basic usage", - "env": { - "TERM": "xterm", - "SHELL": "/bin/bash" - }, - "stdout": [ - [ - 0.083341, - "\u001b]0;tw@tux: ~/borg/demo/download\u0007tw@tux:~/borg/demo/download$ " - ], - [ - 0.349103, - "#" - ], - [ - 0.338948, - " " - ], - [ - 0.185424, - "b" - ], - [ - 0.142971, - "o" - ], - [ - 0.091227, - "r" - ], - [ - 0.092867, - "g" - ], - [ - 0.222552, - "b" - ], - [ - 0.114706, - "a" - ], - [ - 0.125044, - "c" - ], - [ - 0.144755, - "k" - ], - [ - 0.241044, - "u" - ], - [ - 0.243681, - "p" - ], - [ - 0.265888, - " " - ], - [ - 0.345247, - "-" - ], - [ - 0.251918, - " " - ], - [ - 0.233420, - "i" - ], - [ - 0.078609, - "n" - ], - [ - 0.076809, - "s" - ], - [ - 0.070225, - "t" - ], - [ - 0.148413, - "a" - ], - [ - 0.077403, - "l" - ], - [ - 0.139884, - "l" - ], - [ - 0.084807, - "a" - ], - [ - 0.138823, - "t" - ], - [ - 0.068185, - "i" - ], - [ - 0.170422, - "o" - ], - [ - 0.161091, - "n" - ], - [ - 0.169247, - " " - ], - [ - 0.110722, - "a" - ], - [ - 0.113785, - "n" - ], - [ - 0.397895, - "d" - ], - [ - 0.305048, - " " - ], - [ - 0.211476, - "b" - ], - [ - 0.109865, - "a" - ], - [ - 0.230634, - "s" - ], - [ - 0.277915, - "i" - ], - [ - 0.206167, - "c" - ], - [ - 0.145265, - " " - ], - [ - 0.219619, - "u" - ], - [ - 0.139989, - "s" - ], - [ - 0.180240, - "a" - ], - [ - 0.200391, - "g" - ], - [ - 0.116961, - "e" - ], - [ - 0.172074, - "\r\n" - ], - [ - 0.000449, - "\u001b]0;tw@tux: ~/borg/demo/download\u0007tw@tux:~/borg/demo/download$ " - ], - [ - 0.620909, - "#" - ], - [ - 0.217833, - " " - ], - [ - 0.592920, - "I" - ], - [ - 0.166726, - " " - ], - [ - 0.161953, - "h" - ], - [ - 0.072501, - "a" - ], - [ - 0.170951, - "v" - ], - [ - 0.154067, - "e" - ], - [ - 0.110535, - " " - ], - [ - 0.155235, - "a" - ], - [ - 0.130825, - "l" - ], - [ - 0.111834, - "r" - ], - [ - 0.142378, - "e" - ], - [ - 0.165867, - "a" - ], - [ - 0.062556, - "d" - ], - [ - 0.091778, - "y" - ], - [ - 0.216280, - " " - ], - [ - 0.169501, - "d" - ], - [ - 0.198240, - "o" - ], - [ - 0.092373, - "w" - ], - [ - 0.143405, - "n" - ], - [ - 0.207324, - "l" - ], - [ - 0.164248, - "o" - ], - [ - 0.088481, - "a" - ], - [ - 0.129191, - "d" - ], - [ - 0.179234, - "e" - ], - [ - 0.189248, - "d" - ], - [ - 0.145203, - " " - ], - [ - 0.221625, - "t" - ], - [ - 0.100064, - "h" - ], - [ - 0.133349, - "e" - ], - [ - 0.066501, - " " - ], - [ - 0.187004, - "f" - ], - [ - 0.142461, - "i" - ], - [ - 0.204723, - "l" - ], - [ - 0.068716, - "e" - ], - [ - 0.237576, - "s" - ], - [ - 0.128085, - ":" - ], - [ - 0.242282, - "\r\n" - ], - [ - 0.000327, - "\u001b]0;tw@tux: ~/borg/demo/download\u0007tw@tux:~/borg/demo/download$ " - ], - [ - 1.796834, - "l" - ], - [ - 0.092545, - "s" - ], - [ - 0.210322, - " " - ], - [ - 0.189710, - "-" - ], - [ - 0.215532, - "l" - ], - [ - 0.852863, - "\r\n" - ], - [ - 0.002104, - "total 10620\r\n" - ], - [ - 0.000040, - "-rw-rw-r-- 1 tw tw 10869049 Oct 24 22:11 borg-linux64" - ], - [ - 0.000007, - "\r\n" - ], - [ - 0.000019, - "-rw-rw-r-- 1 tw tw 819 Oct 24 22:11 borg-linux64.asc\r\n" - ], - [ - 0.000431, - "\u001b]0;tw@tux: ~/borg/demo/download\u0007tw@tux:~/borg/demo/download$ " - ], - [ - 0.513172, - "#" - ], - [ - 0.284059, - " " - ], - [ - 0.330931, - "b" - ], - [ - 0.118806, - "i" - ], - [ - 0.100553, - "n" - ], - [ - 0.259930, - "a" - ], - [ - 0.106715, - "r" - ], - [ - 0.276545, - "y" - ], - [ - 0.126132, - " " - ], - [ - 0.379724, - "+" - ], - [ - 0.199249, - " " - ], - [ - 0.295913, - "G" - ], - [ - 0.108970, - "P" - ], - [ - 0.080480, - "G" - ], - [ - 0.349293, - " " - ], - [ - 0.236785, - "s" - ], - [ - 0.105197, - "i" - ], - [ - 0.289951, - "g" - ], - [ - 0.351385, - "n" - ], - [ - 0.282003, - "a" - ], - [ - 0.206591, - "t" - ], - [ - 0.163963, - "u" - ], - [ - 0.082416, - "r" - ], - [ - 0.125432, - "e" - ], - [ - 0.369988, - "\r\n" - ], - [ - 0.000341, - "\u001b]0;tw@tux: ~/borg/demo/download\u0007tw@tux:~/borg/demo/download$ " - ], - [ - 0.889617, - "#" - ], - [ - 0.226974, - " " - ], - [ - 0.218497, - "l" - ], - [ - 0.134545, - "e" - ], - [ - 0.103159, - "t" - ], - [ - 0.711682, - "'" - ], - [ - 0.185463, - "s" - ], - [ - 0.162130, - " " - ], - [ - 0.166049, - "v" - ], - [ - 0.183069, - "e" - ], - [ - 0.099764, - "r" - ], - [ - 0.234211, - "i" - ], - [ - 0.854328, - "f" - ], - [ - 0.203758, - "y" - ], - [ - 0.166681, - " " - ], - [ - 0.216715, - "t" - ], - [ - 0.560064, - "h" - ], - [ - 0.151837, - "a" - ], - [ - 0.194509, - "t" - ], - [ - 0.119665, - " " - ], - [ - 0.141089, - "t" - ], - [ - 0.096803, - "h" - ], - [ - 0.104718, - "e" - ], - [ - 0.106761, - " " - ], - [ - 0.229401, - "b" - ], - [ - 0.213802, - "i" - ], - [ - 0.075481, - "n" - ], - [ - 0.138720, - "a" - ], - [ - 0.062411, - "r" - ], - [ - 0.292719, - "y" - ], - [ - 0.482737, - " " - ], - [ - 0.211595, - "i" - ], - [ - 0.110964, - "s" - ], - [ - 0.102100, - " " - ], - [ - 0.143380, - "v" - ], - [ - 0.189214, - "a" - ], - [ - 0.099337, - "l" - ], - [ - 0.172757, - "i" - ], - [ - 0.082456, - "d" - ], - [ - 0.177514, - ":" - ], - [ - 0.622492, - "\r\n" - ], - [ - 0.000313, - "\u001b]0;tw@tux: ~/borg/demo/download\u0007tw@tux:~/borg/demo/download$ " - ], - [ - 2.000000, - "g" - ], - [ - 0.261924, - "p" - ], - [ - 0.108570, - "g" - ], - [ - 0.247315, - " " - ], - [ - 0.277162, - "-" - ], - [ - 0.141397, - "-" - ], - [ - 0.143255, - "v" - ], - [ - 0.162858, - "e" - ], - [ - 0.040051, - "r" - ], - [ - 0.105941, - "i" - ], - [ - 0.144872, - "f" - ], - [ - 0.306497, - "y" - ], - [ - 0.468271, - " " - ], - [ - 2.000000, - "b" - ], - [ - 0.119390, - "o" - ], - [ - 0.463137, - "\u0007" - ], - [ - 0.000095, - "rg-linux64" - ], - [ - 0.341519, - "." - ], - [ - 0.146977, - "asc " - ], - [ - 0.186292, - "\r\n" - ], - [ - 0.100648, - "gpg: Signature made Wed 07 Oct 2015 02:41:38 PM CEST\r\n" - ], - [ - 0.000011, - "gpg: using RSA key 243ACFA951F78E01\r\n" - ], - [ - 0.006906, - "gpg: Good signature from \"Thomas Waldmann \u003ctw@waldmann-edv.de\u003e\"" - ], - [ - 0.000033, - "\r\ngpg: aka \"Thomas Waldmann \u003ctw-public@gmx.de\u003e\"\r\ngpg: aka \"Thomas Waldmann \u003ctwaldmann@thinkmo.de\u003e\"\r\n" - ], - [ - 0.000018, - "gpg: aka \"Thomas Waldmann \u003cthomas.j.waldmann@gmail.com\u003e\"\r\n" - ], - [ - 0.003077, - "\u001b]0;tw@tux: ~/borg/demo/download\u0007tw@tux:~/borg/demo/download$ " - ], - [ - 2.000000, - "#" - ], - [ - 0.241501, - " " - ], - [ - 0.186571, - "e" - ], - [ - 0.214388, - "v" - ], - [ - 0.157101, - "e" - ], - [ - 0.042348, - "r" - ], - [ - 0.253261, - "y" - ], - [ - 0.254356, - "t" - ], - [ - 0.094622, - "h" - ], - [ - 0.213972, - "i" - ], - [ - 0.084853, - "n" - ], - [ - 0.084920, - "g" - ], - [ - 0.178519, - " " - ], - [ - 0.256151, - "o" - ], - [ - 0.217918, - "k" - ], - [ - 0.153899, - "!" - ], - [ - 0.246211, - "\r\n" - ], - [ - 0.000330, - "\u001b]0;tw@tux: ~/borg/demo/download\u0007tw@tux:~/borg/demo/download$ " - ], - [ - 2.000000, - "#" - ], - [ - 0.288008, - " " - ], - [ - 0.232836, - "i" - ], - [ - 0.055326, - "n" - ], - [ - 0.142978, - "s" - ], - [ - 0.080599, - "t" - ], - [ - 0.139018, - "a" - ], - [ - 0.111052, - "l" - ], - [ - 0.132419, - "l" - ], - [ - 0.169037, - " " - ], - [ - 0.117036, - "t" - ], - [ - 0.092749, - "h" - ], - [ - 0.124768, - "e" - ], - [ - 0.088888, - " " - ], - [ - 0.184118, - "b" - ], - [ - 0.182336, - "i" - ], - [ - 0.075466, - "n" - ], - [ - 0.085516, - "a" - ], - [ - 0.060363, - "r" - ], - [ - 0.843225, - "y" - ], - [ - 0.209758, - " " - ], - [ - 0.168892, - "a" - ], - [ - 0.151126, - "s" - ], - [ - 0.127487, - " " - ], - [ - 0.300923, - "\"" - ], - [ - 0.217060, - "b" - ], - [ - 0.221579, - "o" - ], - [ - 0.047775, - "r" - ], - [ - 0.107202, - "g" - ], - [ - 0.438939, - "\"" - ], - [ - 0.253153, - ":" - ], - [ - 0.617823, - "\r\n" - ], - [ - 0.000326, - "\u001b]0;tw@tux: ~/borg/demo/download\u0007tw@tux:~/borg/demo/download$ " - ], - [ - 0.816740, - "c" - ], - [ - 0.168734, - "p" - ], - [ - 0.230846, - " " - ], - [ - 0.299588, - "b" - ], - [ - 0.121082, - "o" - ], - [ - 0.214148, - "\u0007" - ], - [ - 0.000011, - "rg-linux64" - ], - [ - 0.331736, - " " - ], - [ - 0.812264, - "~" - ], - [ - 0.518926, - "/" - ], - [ - 0.233797, - "b" - ], - [ - 0.214141, - "i" - ], - [ - 0.098062, - "n" - ], - [ - 0.607725, - "/" - ], - [ - 0.566434, - "b" - ], - [ - 0.145886, - "o" - ], - [ - 0.113081, - "r" - ], - [ - 0.068870, - "g" - ], - [ - 0.851794, - "\r\n" - ], - [ - 0.012632, - "\u001b]0;tw@tux: ~/borg/demo/download\u0007tw@tux:~/borg/demo/download$ " - ], - [ - 2.000000, - "#" - ], - [ - 0.269926, - " " - ], - [ - 0.208575, - "m" - ], - [ - 0.135192, - "a" - ], - [ - 0.119543, - "k" - ], - [ - 0.080873, - "e" - ], - [ - 0.156871, - " " - ], - [ - 0.197124, - "i" - ], - [ - 0.078784, - "t" - ], - [ - 0.142373, - " " - ], - [ - 0.189080, - "e" - ], - [ - 0.232597, - "x" - ], - [ - 0.170105, - "e" - ], - [ - 0.132039, - "c" - ], - [ - 0.230568, - "u" - ], - [ - 0.086573, - "t" - ], - [ - 0.255047, - "a" - ], - [ - 0.231478, - "b" - ], - [ - 0.283723, - "l" - ], - [ - 0.112987, - "e" - ], - [ - 0.518611, - ":" - ], - [ - 0.459423, - "\r\n" - ], - [ - 0.000822, - "\u001b]0;tw@tux: ~/borg/demo/download\u0007tw@tux:~/borg/demo/download$ " - ], - [ - 0.353739, - "c" - ], - [ - 0.114161, - "h" - ], - [ - 0.268562, - "m" - ], - [ - 0.179085, - "o" - ], - [ - 0.145360, - "d" - ], - [ - 0.075599, - " " - ], - [ - 0.773964, - "+" - ], - [ - 0.113699, - "x" - ], - [ - 0.187579, - " " - ], - [ - 0.381391, - "~" - ], - [ - 0.512520, - "/" - ], - [ - 0.231090, - "b" - ], - [ - 0.197636, - "i" - ], - [ - 0.101238, - "n" - ], - [ - 0.341295, - "/" - ], - [ - 0.306047, - "b" - ], - [ - 0.106898, - "o" - ], - [ - 0.233773, - "rg " - ], - [ - 0.519336, - "\r\n" - ], - [ - 0.001408, - "\u001b]0;tw@tux: ~/borg/demo/download\u0007tw@tux:~/borg/demo/download$ " - ], - [ - 2.000000, - "#" - ], - [ - 0.247104, - " " - ], - [ - 0.218717, - "i" - ], - [ - 0.067769, - "n" - ], - [ - 0.139583, - "s" - ], - [ - 0.092034, - "t" - ], - [ - 0.152729, - "a" - ], - [ - 0.083844, - "l" - ], - [ - 0.145806, - "l" - ], - [ - 0.120879, - "a" - ], - [ - 0.164967, - "t" - ], - [ - 0.065308, - "i" - ], - [ - 0.816983, - "o" - ], - [ - 0.231669, - "n" - ], - [ - 0.185168, - " " - ], - [ - 0.125214, - "d" - ], - [ - 0.112630, - "o" - ], - [ - 0.068650, - "n" - ], - [ - 0.108386, - "e" - ], - [ - 0.563031, - "!" - ], - [ - 2.000000, - "\r\n" - ], - [ - 0.000365, - "\u001b]0;tw@tux: ~/borg/demo/download\u0007tw@tux:~/borg/demo/download$ " - ], - [ - 0.347093, - "#" - ], - [ - 0.262764, - " " - ], - [ - 0.191568, - "l" - ], - [ - 0.086614, - "e" - ], - [ - 0.110365, - "t" - ], - [ - 0.707057, - "'" - ], - [ - 0.220060, - "s" - ], - [ - 0.181690, - " " - ], - [ - 0.128039, - "c" - ], - [ - 0.176264, - "r" - ], - [ - 0.171208, - "e" - ], - [ - 0.199371, - "a" - ], - [ - 0.161622, - "t" - ], - [ - 0.145989, - "e" - ], - [ - 0.187920, - " " - ], - [ - 0.734653, - "a" - ], - [ - 0.185812, - " " - ], - [ - 0.270851, - "r" - ], - [ - 0.120000, - "e" - ], - [ - 0.161097, - "p" - ], - [ - 0.179813, - "o" - ], - [ - 0.170557, - "s" - ], - [ - 0.145457, - "i" - ], - [ - 0.165200, - "t" - ], - [ - 0.135578, - "o" - ], - [ - 0.130363, - "r" - ], - [ - 0.461631, - "y" - ], - [ - 0.303047, - ":" - ], - [ - 0.955198, - "\r\n" - ], - [ - 0.000300, - "\u001b]0;tw@tux: ~/borg/demo/download\u0007tw@tux:~/borg/demo/download$ " - ], - [ - 0.301237, - "c" - ], - [ - 0.084267, - "d" - ], - [ - 0.155241, - " " - ], - [ - 0.813751, - "." - ], - [ - 0.157147, - "." - ], - [ - 0.573720, - "\r\n" - ], - [ - 0.000508, - "\u001b]0;tw@tux: ~/borg/demo\u0007tw@tux:~/borg/demo$ " - ], - [ - 0.225463, - "b" - ], - [ - 0.274841, - "o" - ], - [ - 0.125292, - "r" - ], - [ - 0.083313, - "g" - ], - [ - 0.088596, - " " - ], - [ - 0.231502, - "i" - ], - [ - 0.062726, - "n" - ], - [ - 0.144877, - "i" - ], - [ - 0.112508, - "t" - ], - [ - 0.313489, - " " - ], - [ - 0.298944, - "r" - ], - [ - 0.216556, - "e" - ], - [ - 0.180484, - "p" - ], - [ - 0.204141, - "o" - ], - [ - 0.682782, - "\r\n" - ], - [ - 0.352828, - "Initializing repository at \"repo\"\r\n" - ], - [ - 0.001407, - "Encryption NOT enabled.\r\nUse the \"--encryption=repokey|keyfile|passphrase\" to enable encryption." - ], - [ - 0.000009, - "\r\n" - ], - [ - 0.008492, - "Synchronizing chunks cache..." - ], - [ - 0.000009, - "\r\n" - ], - [ - 0.000030, - "Archives: 0, w/ cached Idx: 0, w/ outdated Idx: 0, w/o cached Idx: 0." - ], - [ - 0.000004, - "\r\n" - ], - [ - 0.000024, - "Done." - ], - [ - 0.000004, - "\r\n" - ], - [ - 0.027827, - "\u001b]0;tw@tux: ~/borg/demo\u0007tw@tux:~/borg/demo$ " - ], - [ - 0.988184, - "#" - ], - [ - 0.248844, - " " - ], - [ - 0.199486, - "l" - ], - [ - 0.104455, - "e" - ], - [ - 0.127960, - "t" - ], - [ - 0.484976, - "'" - ], - [ - 0.186103, - "s" - ], - [ - 0.151763, - " " - ], - [ - 0.177456, - "c" - ], - [ - 0.178972, - "r" - ], - [ - 0.183533, - "e" - ], - [ - 0.192725, - "a" - ], - [ - 0.146352, - "t" - ], - [ - 0.156199, - "e" - ], - [ - 0.232699, - " " - ], - [ - 0.513490, - "o" - ], - [ - 0.229828, - "u" - ], - [ - 0.104744, - "r" - ], - [ - 0.115068, - " " - ], - [ - 0.201439, - "f" - ], - [ - 0.333315, - "i" - ], - [ - 0.209070, - "r" - ], - [ - 0.259194, - "s" - ], - [ - 0.076346, - "t" - ], - [ - 0.125673, - " " - ], - [ - 0.198575, - "b" - ], - [ - 0.089009, - "a" - ], - [ - 0.238307, - "c" - ], - [ - 0.105568, - "k" - ], - [ - 0.254971, - "u" - ], - [ - 0.318094, - "p" - ], - [ - 0.690770, - ":" - ], - [ - 0.580155, - "\r\n" - ], - [ - 0.000308, - "\u001b]0;tw@tux: ~/borg/demo\u0007tw@tux:~/borg/demo$ " - ], - [ - 0.603046, - "b" - ], - [ - 0.104492, - "o" - ], - [ - 0.148182, - "r" - ], - [ - 0.087024, - "g" - ], - [ - 0.176897, - " " - ], - [ - 0.183168, - "c" - ], - [ - 0.185325, - "r" - ], - [ - 0.183347, - "e" - ], - [ - 0.182868, - "a" - ], - [ - 0.170600, - "t" - ], - [ - 0.137005, - "e" - ], - [ - 0.164357, - " " - ], - [ - 0.427028, - "-" - ], - [ - 0.147791, - "-" - ], - [ - 0.440101, - "s" - ], - [ - 0.177193, - "t" - ], - [ - 0.203817, - "a" - ], - [ - 0.217150, - "t" - ], - [ - 0.229771, - "s" - ], - [ - 0.191220, - " " - ], - [ - 0.269939, - "-" - ], - [ - 0.145163, - "-" - ], - [ - 0.450053, - "p" - ], - [ - 0.165194, - "r" - ], - [ - 0.044264, - "o" - ], - [ - 0.204568, - "g" - ], - [ - 0.104759, - "r" - ], - [ - 0.213137, - "e" - ], - [ - 0.216596, - "s" - ], - [ - 0.163238, - "s" - ], - [ - 0.241084, - " " - ], - [ - 0.300727, - "-" - ], - [ - 0.149156, - "-" - ], - [ - 0.259608, - "c" - ], - [ - 0.120930, - "o" - ], - [ - 0.098838, - "m" - ], - [ - 0.234615, - "p" - ], - [ - 0.084600, - "r" - ], - [ - 0.166072, - "e" - ], - [ - 0.185576, - "s" - ], - [ - 0.159984, - "s" - ], - [ - 0.122793, - "i" - ], - [ - 0.180423, - "o" - ], - [ - 0.196311, - "n" - ], - [ - 0.181682, - " " - ], - [ - 0.242129, - "l" - ], - [ - 0.842020, - "z" - ], - [ - 0.707941, - "4" - ], - [ - 0.180354, - " " - ], - [ - 0.419080, - "r" - ], - [ - 0.189076, - "e" - ], - [ - 0.172527, - "p" - ], - [ - 0.154922, - "o" - ], - [ - 0.728059, - ":" - ], - [ - 0.147089, - ":" - ], - [ - 0.396117, - "b" - ], - [ - 0.090233, - "a" - ], - [ - 0.199537, - "c" - ], - [ - 0.084686, - "k" - ], - [ - 0.278049, - "u \r" - ], - [ - 0.268438, - "p" - ], - [ - 0.491592, - "1" - ], - [ - 0.508588, - " " - ], - [ - 0.174143, - "d" - ], - [ - 0.175430, - "a" - ], - [ - 0.166841, - "t" - ], - [ - 0.127029, - "a" - ], - [ - 0.380593, - "\r\n" - ], - [ - 0.557518, - " 2.68 MB O 1.25 MB C 1.25 MB D data/linux-4.1.8/Doc...ia/v4l/func-read.xml\r" - ], - [ - 0.200102, - " 5.37 MB O 2.46 MB C 2.46 MB D data/linux-4.1.8/Documentation/backlight \r" - ], - [ - 0.200342, - " 6.99 MB O 3.36 MB C 3.36 MB D data/linux-4.1.8/Doc...rm_big_little_dt.txt\r" - ], - [ - 0.200137, - " 7.83 MB O 3.87 MB C 3.87 MB D data/linux-4.1.8/Doc...s/mtd/atmel-nand.txt\r" - ], - [ - 0.200271, - " 8.77 MB O 4.41 MB C 4.41 MB D data/linux-4.1.8/Doc...ngs/soc/fsl/qman.txt\r" - ], - [ - 0.200577, - " 9.99 MB O 5.12 MB C 5.12 MB D data/linux-4.1.8/Doc...ching/cachefiles.txt\r" - ], - [ - 0.200033, - " 12.11 MB O 6.34 MB C 6.33 MB D data/linux-4.1.8/Doc...infiniband/ipoib.txt\r" - ], - [ - 0.200272, - " 15.27 MB O 8.08 MB C 8.08 MB D data/linux-4.1.8/Doc.../networking/team.txt\r" - ], - [ - 0.200072, - " 18.22 MB O 9.72 MB C 9.71 MB D data/linux-4.1.8/Doc...tation/sysctl/vm.txt\r" - ], - [ - 0.202107, - " 21.05 MB O 11.22 MB C 11.21 MB D data/linux-4.1.8/MAINTAINERS \r" - ], - [ - 0.200251, - " 23.04 MB O 12.20 MB C 12.20 MB D data/linux-4.1.8/arc...de/uapi/asm/unistd.h\r" - ], - [ - 0.200450, - " 25.45 MB O 13.17 MB C 13.17 MB D data/linux-4.1.8/arc.../boot/dts/imx23.dtsi\r" - ], - [ - 0.200093, - " 27.65 MB O 14.01 MB C 14.00 MB D data/linux-4.1.8/arc...omap3-overo-tobi.dts\r" - ], - [ - 0.200314, - " 30.26 MB O 14.89 MB C 14.89 MB D data/linux-4.1.8/arc...ot/dts/tps65910.dtsi\r" - ], - [ - 0.200003, - " 31.90 MB O 15.63 MB C 15.63 MB D data/linux-4.1.8/arc...include/asm/probes.h\r" - ], - [ - 0.200493, - " 34.66 MB O 16.84 MB C 16.83 MB D data/linux-4.1.8/arc...i/include/mach/psc.h\r" - ], - [ - 0.200675, - " 36.62 MB O 17.70 MB C 17.70 MB D data/linux-4.1.8/arc...mach-ixp4xx/common.c\r" - ], - [ - 0.200307, - " 38.40 MB O 18.54 MB C 18.53 MB D data/linux-4.1.8/arch/arm/mach-omap2/cm.h \r" - ], - [ - 0.200254, - " 41.29 MB O 19.63 MB C 19.63 MB D data/linux-4.1.8/arch/arm/mach-pxa/idp.c \r" - ], - [ - 0.219493, - " 43.57 MB O 20.67 MB C 20.66 MB D data/linux-4.1.8/arc...bile/clock-r8a7778.c\r" - ], - [ - 0.200451, - " 45.55 MB O 21.59 MB C 21.59 MB D data/linux-4.1.8/arc...m/plat-samsung/adc.c\r" - ], - [ - 0.200370, - " 47.50 MB O 22.51 MB C 22.51 MB D data/linux-4.1.8/arch/arm64/lib/memmove.S \r" - ], - [ - 0.200686, - " 49.21 MB O 23.33 MB C 23.32 MB D data/linux-4.1.8/arc...ckfin/kernel/trace.c\r" - ], - [ - 0.200393, - " 53.22 MB O 24.51 MB C 24.50 MB D data/linux-4.1.8/arch/c6x/include/asm/soc.h\r" - ], - [ - 0.200371, - " 56.19 MB O 25.50 MB C 25.49 MB D data/linux-4.1.8/arc...op/iop_sw_cpu_defs.h\r" - ], - [ - 0.200450, - " 57.84 MB O 26.17 MB C 26.14 MB D data/linux-4.1.8/arc...include/asm/vm_mmu.h\r" - ], - [ - 0.200573, - " 60.21 MB O 27.27 MB C 27.25 MB D data/linux-4.1.8/arch/ia64/kernel/time.c \r" - ], - [ - 0.200222, - " 62.31 MB O 28.18 MB C 28.15 MB D data/linux-4.1.8/arc.../coldfire/sltimers.c\r" - ], - [ - 0.200756, - " 67.09 MB O 29.98 MB C 29.90 MB D data/linux-4.1.8/arc...8k/include/asm/tlb.h\r" - ], - [ - 0.200716, - " 68.75 MB O 30.80 MB C 30.72 MB D data/linux-4.1.8/arc...ude/uapi/asm/fcntl.h\r" - ], - [ - 0.200734, - " 70.69 MB O 31.67 MB C 31.59 MB D data/linux-4.1.8/arc...figs/malta_defconfig\r" - ], - [ - 0.200198, - " 72.12 MB O 32.34 MB C 32.26 MB D data/linux-4.1.8/arc...de/asm/mc146818rtc.h\r" - ], - [ - 0.200446, - " 76.01 MB O 33.45 MB C 33.37 MB D data/linux-4.1.8/arch/mips/jazz/jazzdma.c \r" - ], - [ - 0.200111, - " 78.19 MB O 34.46 MB C 34.38 MB D data/linux-4.1.8/arc...tlogic/common/time.c\r" - ], - [ - 0.200191, - " 79.84 MB O 35.30 MB C 35.21 MB D data/linux-4.1.8/arc...de/uapi/asm/msgbuf.h\r" - ], - [ - 0.200421, - " 81.35 MB O 36.07 MB C 35.99 MB D data/linux-4.1.8/arc...sc/include/asm/rtc.h\r" - ], - [ - 0.200090, - " 83.49 MB O 37.03 MB C 36.95 MB D data/linux-4.1.8/arc...fsl/qoriq-dma-1.dtsi\r" - ], - [ - 0.200331, - " 85.13 MB O 37.80 MB C 37.72 MB D data/linux-4.1.8/arc...pc836x_rdk_defconfig\r" - ], - [ - 0.200114, - " 87.04 MB O 38.71 MB C 38.63 MB D data/linux-4.1.8/arc...ude/uapi/asm/nvram.h\r" - ], - [ - 0.200280, - " 90.24 MB O 40.19 MB C 40.11 MB D data/linux-4.1.8/arc...pc/math-emu/mtfsfi.c\r" - ], - [ - 0.216796, - " 92.68 MB O 41.41 MB C 41.33 MB D data/linux-4.1.8/arc...rms/powermac/nvram.c\r" - ], - [ - 0.200198, - " 95.32 MB O 42.60 MB C 42.52 MB D data/linux-4.1.8/arc...nclude/asm/pgtable.h\r" - ], - [ - 0.200304, - " 97.31 MB O 43.50 MB C 43.42 MB D data/linux-4.1.8/arc...mach-dreamcast/irq.c\r" - ], - [ - 0.200328, - " 99.46 MB O 44.41 MB C 44.33 MB D data/linux-4.1.8/arc...artner-jet-setup.txt\r" - ], - [ - 0.200102, - "101.28 MB O 45.25 MB C 45.16 MB D data/linux-4.1.8/arc...rc/include/asm/ecc.h\r" - ], - [ - 0.200253, - "103.53 MB O 46.27 MB C 46.19 MB D data/linux-4.1.8/arc.../kernel/una_asm_64.S\r" - ], - [ - 0.200503, - "105.76 MB O 47.32 MB C 47.23 MB D data/linux-4.1.8/arch/tile/kernel/reboot.c \r" - ], - [ - 0.201177, - "107.64 MB O 48.27 MB C 48.18 MB D data/linux-4.1.8/arc...t/compressed/eboot.c\r" - ], - [ - 0.200192, - "109.82 MB O 49.22 MB C 49.13 MB D data/linux-4.1.8/arc...clude/asm/spinlock.h\r" - ], - [ - 0.200851, - "112.71 MB O 50.56 MB C 50.48 MB D data/linux-4.1.8/arch/x86/kernel/ptrace.c \r" - ], - [ - 0.200195, - "115.71 MB O 51.96 MB C 51.87 MB D data/linux-4.1.8/arc...s/platform_emc1403.c\r" - ], - [ - 0.200306, - "117.28 MB O 52.79 MB C 52.70 MB D data/linux-4.1.8/arc...nclude/variant/tie.h\r" - ], - [ - 0.204475, - "122.68 MB O 55.35 MB C 55.26 MB D data/linux-4.1.8/fir...x-e1-6.2.9.0.fw.ihex\r" - ], - [ - 0.199974, - "127.39 MB O 58.15 MB C 57.97 MB D data/linux-4.1.8/fs/afs/fsclient.c \r" - ], - [ - 0.201254, - "132.58 MB O 60.42 MB C 60.24 MB D data/linux-4.1.8/fs/cifs/cifssmb.c \r" - ], - [ - 0.216710, - "136.76 MB O 62.28 MB C 62.10 MB D data/linux-4.1.8/fs/ext4/inline.c \r" - ], - [ - 0.200891, - "140.78 MB O 64.15 MB C 63.97 MB D data/linux-4.1.8/fs/jbd2/commit.c \r" - ], - [ - 0.199883, - "144.88 MB O 65.98 MB C 65.80 MB D data/linux-4.1.8/fs/nfs/objlayout \r" - ], - [ - 0.201488, - "150.31 MB O 67.96 MB C 67.78 MB D data/linux-4.1.8/fs/...fy/dnotify/dnotify.c\r" - ], - [ - 0.205472, - "154.72 MB O 69.97 MB C 69.79 MB D data/linux-4.1.8/fs/quota/dquot.c \r" - ], - [ - 0.200493, - "159.06 MB O 71.91 MB C 71.73 MB D data/linux-4.1.8/fs/...xfs/xfs_inode_fork.h\r" - ], - [ - 0.200000, - "161.54 MB O 73.09 MB C 72.91 MB D data/linux-4.1.8/inc.../crypto/public_key.h\r" - ], - [ - 0.205041, - "164.32 MB O 74.28 MB C 74.09 MB D data/linux-4.1.8/inc...inux/cgroup_subsys.h\r" - ], - [ - 0.200371, - "166.33 MB O 75.23 MB C 75.05 MB D data/linux-4.1.8/include/linux/if_team.h \r" - ], - [ - 0.200340, - "168.82 MB O 76.24 MB C 76.06 MB D data/linux-4.1.8/inc.../mfd/pcf50633/gpio.h\r" - ], - [ - 0.200162, - "171.65 MB O 77.36 MB C 77.17 MB D data/linux-4.1.8/include/linux/phy.h \r" - ], - [ - 0.200385, - "172.84 MB O 77.97 MB C 77.79 MB D data/linux-4.1.8/include/linux/scc.h \r" - ], - [ - 0.200918, - "174.87 MB O 78.94 MB C 78.76 MB D data/linux-4.1.8/include/linux/wait.h \r" - ], - [ - 0.200117, - "177.06 MB O 80.01 MB C 79.83 MB D data/linux-4.1.8/inc...er/nfnetlink_queue.h\r" - ], - [ - 0.200254, - "179.53 MB O 81.13 MB C 80.95 MB D data/linux-4.1.8/inc...e/events/intel-sst.h\r" - ], - [ - 0.200176, - "181.40 MB O 82.05 MB C 81.86 MB D data/linux-4.1.8/include/uapi/linux/mpls.h \r" - ], - [ - 0.200438, - "183.11 MB O 82.88 MB C 82.70 MB D data/linux-4.1.8/inc...api/scsi/fc/fc_els.h\r" - ], - [ - 0.200226, - "186.12 MB O 84.31 MB C 84.12 MB D data/linux-4.1.8/kernel/jump_label.c \r" - ], - [ - 0.200138, - "190.76 MB O 86.46 MB C 86.28 MB D data/linux-4.1.8/lib/Kconfig.debug \r" - ], - [ - 0.200958, - "194.21 MB O 87.82 MB C 87.64 MB D data/linux-4.1.8/mm/memblock.c \r" - ], - [ - 0.200544, - "198.19 MB O 89.69 MB C 89.51 MB D data/linux-4.1.8/net/bluetooth/ecc.c \r" - ], - [ - 0.200232, - "202.28 MB O 91.52 MB C 91.34 MB D data/linux-4.1.8/net/hsr/hsr_slave.c \r" - ], - [ - 0.200153, - "206.23 MB O 93.40 MB C 93.22 MB D data/linux-4.1.8/net/ipx/af_ipx.c \r" - ], - [ - 0.200526, - "210.30 MB O 95.08 MB C 94.89 MB D data/linux-4.1.8/net...ter/ipvs/ip_vs_ftp.c\r" - ], - [ - 0.200433, - "213.29 MB O 96.37 MB C 96.19 MB D data/linux-4.1.8/net/phonet/af_phonet.c \r" - ], - [ - 0.200669, - "217.21 MB O 98.21 MB C 98.03 MB D data/linux-4.1.8/net.../svc_rdma_recvfrom.c\r" - ], - [ - 0.200014, - "220.20 MB O 99.53 MB C 99.35 MB D data/linux-4.1.8/scr...e/free/iounmap.cocci\r" - ], - [ - 0.200446, - "222.94 MB O 100.82 MB C 100.64 MB D data/linux-4.1.8/security/selinux/Makefile \r" - ], - [ - 0.214711, - "226.41 MB O 102.23 MB C 102.05 MB D data/linux-4.1.8/sou...seq/seq_midi_event.c\r" - ], - [ - 0.202631, - "228.96 MB O 103.31 MB C 103.13 MB D data/linux-4.1.8/sound/mips/ad1843.c \r" - ], - [ - 0.200095, - "232.28 MB O 104.65 MB C 104.47 MB D data/linux-4.1.8/sound/pci/ctxfi/Makefile \r" - ], - [ - 0.200726, - "236.33 MB O 106.24 MB C 106.06 MB D data/linux-4.1.8/sound/pci/nm256/Makefile \r" - ], - [ - 0.199902, - "239.73 MB O 107.58 MB C 107.40 MB D data/linux-4.1.8/sou.../codecs/cs4271-spi.c\r" - ], - [ - 0.200592, - "244.29 MB O 109.08 MB C 108.90 MB D data/linux-4.1.8/sound/soc/codecs/wm8940.c \r" - ], - [ - 0.200357, - "247.98 MB O 110.35 MB C 110.17 MB D data/linux-4.1.8/sou...oc/omap/omap-mcpdm.c\r" - ], - [ - 0.200901, - "250.64 MB O 111.50 MB C 111.32 MB D data/linux-4.1.8/sound/usb/mixer_scarlett.c\r" - ], - [ - 0.200535, - "252.14 MB O 112.20 MB C 112.01 MB D data/linux-4.1.8/tools/perf/builtin-kvm.c \r" - ], - [ - 0.200239, - "254.11 MB O 113.07 MB C 112.88 MB D data/linux-4.1.8/tools/perf/util/record.c \r" - ], - [ - 0.200233, - "255.70 MB O 113.82 MB C 113.64 MB D data/linux-4.1.8/too...re/bin/configinit.sh\r" - ], - [ - 0.395702, - " \r------------------------------------------------------------------------------\r\nArchive name: backup1\r\nArchive fingerprint: b3104802be9faa610f281619c69e4d3e672df2ce97528a35d83f15080d02ed86\r\nStart time: Sat Oct 24 22:27:24 2015\r\nEnd time: Sat Oct 24 22:27:43 2015\r\nDuration: 19.32 seconds\r\nNumber of files: 31557\r\n\r\n Original size Compressed size Deduplicated size\r\nThis archive: 257.06 MB 114.44 MB 114.26 MB\r\nAll archives: 257.06 MB 114.44 MB 114.26 MB\r\n\r\n Unique chunks Total chunks\r\nChunk index: 33731 34030\r\n------------------------------------------------------------------------------\r\n" - ], - [ - 0.046138, - "\u001b]0;tw@tux: ~/borg/demo\u0007tw@tux:~/borg/demo$ " - ], - [ - 1.000000, - "#" - ], - [ - 0.564664, - " " - ], - [ - 0.313339, - "c" - ], - [ - 0.492152, - "h" - ], - [ - 0.479518, - "a" - ], - [ - 0.536708, - "n" - ], - [ - 0.134006, - "g" - ], - [ - 0.147326, - "e" - ], - [ - 0.068957, - " " - ], - [ - 0.179678, - "s" - ], - [ - 0.096249, - "o" - ], - [ - 0.081003, - "m" - ], - [ - 0.124342, - "e" - ], - [ - 0.117830, - " " - ], - [ - 0.138019, - "d" - ], - [ - 0.137898, - "a" - ], - [ - 0.199628, - "t" - ], - [ - 0.104935, - "a" - ], - [ - 0.150868, - " " - ], - [ - 0.144877, - "s" - ], - [ - 0.126816, - "l" - ], - [ - 0.178466, - "i" - ], - [ - 0.113395, - "g" - ], - [ - 0.101022, - "h" - ], - [ - 0.102395, - "t" - ], - [ - 0.311498, - "l" - ], - [ - 0.366608, - "y" - ], - [ - 0.657991, - ":" - ], - [ - 0.423140, - "\r\n" - ], - [ - 0.000708, - "\u001b]0;tw@tux: ~/borg/demo\u0007tw@tux:~/borg/demo$ " - ], - [ - 2.000000, - "e" - ], - [ - 0.000021, - "c" - ], - [ - 0.000024, - "h" - ], - [ - 0.000029, - "o" - ], - [ - 0.000018, - " " - ], - [ - 0.000025, - "\"" - ], - [ - 0.000025, - "s" - ], - [ - 0.000026, - "o" - ], - [ - 0.000070, - "me " - ], - [ - 0.000022, - "m" - ], - [ - 0.000028, - "o" - ], - [ - 0.000027, - "r" - ], - [ - 0.000026, - "e" - ], - [ - 0.000029, - " " - ], - [ - 0.000028, - "d" - ], - [ - 0.000028, - "a" - ], - [ - 0.000028, - "t" - ], - [ - 0.000026, - "a" - ], - [ - 0.000033, - "\"" - ], - [ - 0.000028, - " " - ], - [ - 0.000059, - "\u003e" - ], - [ - 0.000045, - " " - ], - [ - 0.000020, - "d" - ], - [ - 0.000040, - "a" - ], - [ - 0.000035, - "t" - ], - [ - 0.000039, - "a" - ], - [ - 0.000034, - "/" - ], - [ - 0.000034, - "o" - ], - [ - 0.000034, - "n" - ], - [ - 0.000037, - "e" - ], - [ - 0.000036, - "_" - ], - [ - 0.000037, - "f" - ], - [ - 0.000037, - "i" - ], - [ - 0.000717, - "le_more\r\n" - ], - [ - 0.000181, - "\u001b]0;tw@tux: ~/borg/demo\u0007tw@tux:~/borg/demo$ " - ], - [ - 2.000000, - "#" - ], - [ - 0.266289, - " " - ], - [ - 0.194686, - "n" - ], - [ - 0.157296, - "o" - ], - [ - 0.084026, - "w" - ], - [ - 0.092729, - " " - ], - [ - 0.148154, - "c" - ], - [ - 0.169136, - "r" - ], - [ - 0.214327, - "e" - ], - [ - 0.180678, - "a" - ], - [ - 0.161652, - "t" - ], - [ - 0.128260, - "e" - ], - [ - 0.158131, - " " - ], - [ - 0.118838, - "a" - ], - [ - 0.120885, - " " - ], - [ - 0.797511, - "s" - ], - [ - 0.200585, - "e" - ], - [ - 0.171811, - "c" - ], - [ - 0.106721, - "o" - ], - [ - 0.153298, - "n" - ], - [ - 0.052244, - "d" - ], - [ - 0.149675, - " " - ], - [ - 0.183517, - "b" - ], - [ - 0.076768, - "a" - ], - [ - 0.189428, - "c" - ], - [ - 0.088431, - "k" - ], - [ - 0.229617, - "u" - ], - [ - 0.272021, - "p" - ], - [ - 0.965855, - ":" - ], - [ - 0.674517, - "\r\n" - ], - [ - 0.000322, - "\u001b]0;tw@tux: ~/borg/demo\u0007tw@tux:~/borg/demo$ " - ], - [ - 0.946131, - "b" - ], - [ - 0.111159, - "o" - ], - [ - 0.094622, - "r" - ], - [ - 0.085288, - "g" - ], - [ - 0.165429, - " " - ], - [ - 0.936087, - "c" - ], - [ - 0.192608, - "r" - ], - [ - 0.187511, - "e" - ], - [ - 0.173135, - "a" - ], - [ - 0.179441, - "t" - ], - [ - 0.125923, - "e" - ], - [ - 0.164920, - " " - ], - [ - 0.737259, - "-" - ], - [ - 0.185417, - "-" - ], - [ - 0.233405, - "s" - ], - [ - 0.152945, - "t" - ], - [ - 0.181548, - "a" - ], - [ - 0.330237, - "t" - ], - [ - 0.735524, - "s" - ], - [ - 0.179019, - " " - ], - [ - 0.245324, - "-" - ], - [ - 0.142362, - "-" - ], - [ - 0.233989, - "p" - ], - [ - 0.153782, - "r" - ], - [ - 0.064431, - "o" - ], - [ - 0.104827, - "g" - ], - [ - 0.090533, - "r" - ], - [ - 0.168129, - "e" - ], - [ - 0.206325, - "s" - ], - [ - 0.157551, - "s" - ], - [ - 0.383630, - " " - ], - [ - 0.759364, - "r" - ], - [ - 0.199262, - "e" - ], - [ - 0.139781, - "p" - ], - [ - 0.151367, - "o" - ], - [ - 0.720350, - ":" - ], - [ - 0.144801, - ":" - ], - [ - 0.532566, - "b" - ], - [ - 0.226514, - "a" - ], - [ - 0.209449, - "c" - ], - [ - 0.142062, - "k" - ], - [ - 0.300090, - "u" - ], - [ - 0.262794, - "p" - ], - [ - 0.218785, - "2" - ], - [ - 0.249599, - " " - ], - [ - 0.187125, - "d" - ], - [ - 0.157741, - "a" - ], - [ - 0.175739, - "t" - ], - [ - 0.139896, - "a" - ], - [ - 0.795560, - "\r\n" - ], - [ - 0.571808, - " 6.50 MB O 3.09 MB C 0 B D data/linux-4.1.8/Doc...ngs/arm/armadeus.txt\r" - ], - [ - 0.200103, - " 11.82 MB O 6.17 MB C 0 B D data/linux-4.1.8/Documentation/hwmon/w83795\r" - ], - [ - 0.200121, - " 27.38 MB O 13.89 MB C 0 B D data/linux-4.1.8/arc...ot/dts/nspire-cx.dts\r" - ], - [ - 0.200110, - " 39.92 MB O 19.04 MB C 0 B D data/linux-4.1.8/arc...omap2/opp2430_data.c\r" - ], - [ - 0.200088, - " 52.28 MB O 24.23 MB C 0 B D data/linux-4.1.8/arc...fin/mach-bf561/smp.c\r" - ], - [ - 0.200078, - " 67.02 MB O 29.94 MB C 0 B D data/linux-4.1.8/arc...8k/include/asm/pci.h\r" - ], - [ - 0.200116, - " 78.29 MB O 34.52 MB C 0 B D data/linux-4.1.8/arc...etlogic/xlr/wakeup.c\r" - ], - [ - 0.200081, - " 90.07 MB O 40.11 MB C 0 B D data/linux-4.1.8/arc...eature-fixups-test.S\r" - ], - [ - 0.200092, - "101.15 MB O 45.19 MB C 0 B D data/linux-4.1.8/arc...rc/crypto/sha1_asm.S\r" - ], - [ - 0.200078, - "115.05 MB O 51.63 MB C 0 B D data/linux-4.1.8/arc...6/mm/kasan_init_64.c\r" - ], - [ - 0.200062, - "147.39 MB O 66.98 MB C 0 B D data/linux-4.1.8/fs/nls/nls_cp864.c \r" - ], - [ - 0.200117, - "169.16 MB O 76.38 MB C 0 B D data/linux-4.1.8/inc.../mfd/twl4030-audio.h\r" - ], - [ - 0.200074, - "181.43 MB O 82.06 MB C 0 B D data/linux-4.1.8/include/uapi/linux/mtio.h \r" - ], - [ - 0.200131, - "209.10 MB O 94.58 MB C 0 B D data/linux-4.1.8/net/mac80211/scan.c \r" - ], - [ - 0.200079, - "234.87 MB O 105.68 MB C 0 B D data/linux-4.1.8/sou...i/hda/patch_si3054.c\r" - ], - [ - 0.200110, - "255.66 MB O 113.80 MB C 0 B D data/linux-4.1.8/too...ves/asm/asm-compat.h\r" - ], - [ - 0.201350, - " \r------------------------------------------------------------------------------\r\nArchive name: backup2\r\nArchive fingerprint: 5737afe8ad2cda7667973b7f2e1d83f097ef3117b5753a38ba7664b616fbdc5a\r\nStart time: Sat Oct 24 22:28:24 2015\r\nEnd time: Sat Oct 24 22:28:27 2015\r\nDuration: 3.41 seconds\r\nNumber of files: 31557\r\n" - ], - [ - 0.001858, - "\r\n Original size Compressed size Deduplicated size\r\nThis archive: 257.06 MB 114.47 MB 45.19 kB\r\nAll archives: 514.12 MB 228.92 MB 114.31 MB\r\n\r\n Unique chunks Total chunks\r\nChunk index: 33733 68060\r\n------------------------------------------------------------------------------\r\n" - ], - [ - 0.033369, - "\u001b]0;tw@tux: ~/borg/demo\u0007tw@tux:~/borg/demo$ " - ], - [ - 1.013315, - "#" - ], - [ - 0.482095, - " " - ], - [ - 0.319571, - "w" - ], - [ - 0.140740, - "o" - ], - [ - 0.090380, - "w" - ], - [ - 0.304400, - "," - ], - [ - 0.137310, - " " - ], - [ - 0.662280, - "t" - ], - [ - 0.162678, - "h" - ], - [ - 0.114083, - "a" - ], - [ - 0.077660, - "t" - ], - [ - 0.120839, - " " - ], - [ - 0.207626, - "w" - ], - [ - 0.195480, - "a" - ], - [ - 0.060188, - "s" - ], - [ - 0.149129, - " " - ], - [ - 0.094522, - "a" - ], - [ - 0.098801, - " " - ], - [ - 0.266235, - "l" - ], - [ - 0.184774, - "o" - ], - [ - 0.255040, - "t" - ], - [ - 0.170498, - " " - ], - [ - 0.201599, - "f" - ], - [ - 0.189774, - "a" - ], - [ - 0.229140, - "s" - ], - [ - 0.275243, - "t" - ], - [ - 0.177347, - "e" - ], - [ - 0.090806, - "r" - ], - [ - 0.204494, - "!" - ], - [ - 0.479851, - "\r\n" - ], - [ - 0.000316, - "\u001b]0;tw@tux: ~/borg/demo\u0007tw@tux:~/borg/demo$ " - ], - [ - 0.734961, - "#" - ], - [ - 0.300000, - " " - ], - [ - 0.300000, - "n" - ], - [ - 0.199297, - "o" - ], - [ - 0.148047, - "t" - ], - [ - 0.071101, - "i" - ], - [ - 0.185554, - "c" - ], - [ - 0.395933, - "e" - ], - [ - 0.180285, - " " - ], - [ - 0.199321, - "t" - ], - [ - 0.094767, - "h" - ], - [ - 0.166966, - "a" - ], - [ - 0.102814, - "t" - ], - [ - 0.415016, - " " - ], - [ - 0.286089, - "\"" - ], - [ - 0.795323, - "D" - ], - [ - 0.180152, - "e" - ], - [ - 0.311214, - "d" - ], - [ - 0.214812, - "u" - ], - [ - 0.251616, - "p" - ], - [ - 0.203533, - "l" - ], - [ - 0.187084, - "i" - ], - [ - 0.124066, - "c" - ], - [ - 0.158062, - "a" - ], - [ - 0.260540, - "t" - ], - [ - 0.136405, - "e" - ], - [ - 0.278039, - "d" - ], - [ - 0.323148, - " " - ], - [ - 0.172337, - "s" - ], - [ - 0.074541, - "i" - ], - [ - 0.269245, - "z" - ], - [ - 0.123599, - "e" - ], - [ - 0.533647, - "\"" - ], - [ - 0.234738, - " " - ], - [ - 0.150720, - "f" - ], - [ - 0.144329, - "o" - ], - [ - 0.086533, - "r" - ], - [ - 0.159717, - " " - ], - [ - 0.274291, - "\"" - ], - [ - 0.471163, - "T" - ], - [ - 0.162135, - "h" - ], - [ - 0.233501, - "i" - ], - [ - 0.134923, - "s" - ], - [ - 0.190779, - " " - ], - [ - 0.307322, - "a" - ], - [ - 0.153882, - "r" - ], - [ - 0.246471, - "c" - ], - [ - 0.110018, - "h" - ], - [ - 0.259798, - "i" - ], - [ - 0.132853, - "v" - ], - [ - 0.171373, - "e" - ], - [ - 0.560405, - "\"" - ], - [ - 0.609162, - "!" - ], - [ - 0.559020, - "\r\n" - ], - [ - 0.000296, - "\u001b]0;tw@tux: ~/borg/demo\u0007tw@tux:~/borg/demo$ " - ], - [ - 0.941369, - "#" - ], - [ - 0.254237, - " " - ], - [ - 0.176998, - "b" - ], - [ - 0.124943, - "o" - ], - [ - 0.097140, - "r" - ], - [ - 0.057513, - "g" - ], - [ - 0.232990, - " " - ], - [ - 0.549539, - "r" - ], - [ - 0.112992, - "e" - ], - [ - 0.240733, - "c" - ], - [ - 0.164146, - "o" - ], - [ - 0.209755, - "g" - ], - [ - 0.145638, - "n" - ], - [ - 0.151826, - "i" - ], - [ - 0.471625, - "z" - ], - [ - 0.759625, - "e" - ], - [ - 0.229566, - "d" - ], - [ - 0.254596, - " " - ], - [ - 0.209452, - "t" - ], - [ - 0.088606, - "h" - ], - [ - 0.155981, - "a" - ], - [ - 0.086797, - "t" - ], - [ - 0.098574, - " " - ], - [ - 0.243290, - "m" - ], - [ - 0.120288, - "o" - ], - [ - 0.092890, - "s" - ], - [ - 0.058823, - "t" - ], - [ - 0.125344, - " " - ], - [ - 0.211464, - "f" - ], - [ - 0.086483, - "i" - ], - [ - 0.213685, - "l" - ], - [ - 0.096764, - "e" - ], - [ - 0.176075, - "s" - ], - [ - 0.122962, - " " - ], - [ - 0.174342, - "d" - ], - [ - 0.103474, - "i" - ], - [ - 0.089744, - "d" - ], - [ - 0.181539, - " " - ], - [ - 0.461771, - "n" - ], - [ - 0.219395, - "o" - ], - [ - 0.095042, - "t" - ], - [ - 0.119662, - " " - ], - [ - 0.156060, - "c" - ], - [ - 0.116988, - "h" - ], - [ - 0.118775, - "a" - ], - [ - 0.126173, - "n" - ], - [ - 0.118518, - "g" - ], - [ - 0.109977, - "e" - ], - [ - 0.167095, - " " - ], - [ - 0.208137, - "a" - ], - [ - 0.155464, - "n" - ], - [ - 0.074939, - "d" - ], - [ - 0.616534, - "\r\n" - ], - [ - 0.000405, - "\u001b]0;tw@tux: ~/borg/demo\u0007tw@tux:~/borg/demo$ " - ], - [ - 0.836535, - "#" - ], - [ - 0.248630, - " " - ], - [ - 0.211525, - "d" - ], - [ - 0.171252, - "e" - ], - [ - 0.244098, - "d" - ], - [ - 0.121718, - "u" - ], - [ - 0.219002, - "p" - ], - [ - 0.197839, - "l" - ], - [ - 0.161081, - "i" - ], - [ - 0.112763, - "c" - ], - [ - 0.154565, - "a" - ], - [ - 0.230427, - "t" - ], - [ - 0.180004, - "e" - ], - [ - 0.182279, - "d" - ], - [ - 0.201281, - " " - ], - [ - 0.202485, - "t" - ], - [ - 0.078397, - "h" - ], - [ - 0.178577, - "e" - ], - [ - 0.150264, - "m" - ], - [ - 0.482274, - "." - ], - [ - 0.300000, - "\r\n" - ], - [ - 0.000265, - "\u001b]0;tw@tux: ~/borg/demo\u0007tw@tux:~/borg/demo$ " - ], - [ - 0.287124, - "#" - ], - [ - 0.314208, - " " - ], - [ - 0.219731, - "n" - ], - [ - 0.176210, - "o" - ], - [ - 0.108529, - "w" - ], - [ - 0.224056, - "," - ], - [ - 0.210976, - " " - ], - [ - 0.190508, - "l" - ], - [ - 0.098452, - "e" - ], - [ - 0.101431, - "t" - ], - [ - 0.855722, - "'" - ], - [ - 0.220403, - "s" - ], - [ - 0.229447, - " " - ], - [ - 0.134839, - "e" - ], - [ - 0.241915, - "x" - ], - [ - 0.217004, - "t" - ], - [ - 0.183774, - "r" - ], - [ - 0.231721, - "a" - ], - [ - 0.221361, - "c" - ], - [ - 0.436221, - "t" - ], - [ - 0.097256, - " " - ], - [ - 0.163933, - "a" - ], - [ - 0.099964, - " " - ], - [ - 0.216806, - "b" - ], - [ - 0.086493, - "a" - ], - [ - 0.211732, - "c" - ], - [ - 0.139016, - "k" - ], - [ - 0.379423, - "u" - ], - [ - 0.250049, - "p" - ], - [ - 0.717916, - ":" - ], - [ - 0.307136, - "\r\n" - ], - [ - 0.000301, - "\u001b]0;tw@tux: ~/borg/demo\u0007tw@tux:~/borg/demo$ " - ], - [ - 0.967533, - "m" - ], - [ - 0.103707, - "v" - ], - [ - 0.407957, - " " - ], - [ - 0.177312, - "d" - ], - [ - 0.166158, - "a" - ], - [ - 0.242593, - "t" - ], - [ - 0.090471, - "a" - ], - [ - 0.699594, - " " - ], - [ - 0.273219, - "d" - ], - [ - 0.170371, - "a" - ], - [ - 0.169331, - "t" - ], - [ - 0.126739, - "a" - ], - [ - 0.288488, - "." - ], - [ - 0.305856, - "o" - ], - [ - 0.135252, - "r" - ], - [ - 0.152717, - "i" - ], - [ - 0.090343, - "g" - ], - [ - 0.312536, - "\r\n" - ], - [ - 0.002579, - "\u001b]0;tw@tux: ~/borg/demo\u0007tw@tux:~/borg/demo$ " - ], - [ - 0.944876, - "b" - ], - [ - 0.149049, - "o" - ], - [ - 0.114834, - "r" - ], - [ - 0.074682, - "g" - ], - [ - 0.129000, - " " - ], - [ - 0.129618, - "e" - ], - [ - 0.261479, - "x" - ], - [ - 0.203937, - "t" - ], - [ - 0.196213, - "r" - ], - [ - 0.193561, - "a" - ], - [ - 0.215314, - "c" - ], - [ - 0.236817, - "t" - ], - [ - 0.188232, - " " - ], - [ - 0.177286, - "r" - ], - [ - 0.200598, - "e" - ], - [ - 0.105866, - "p" - ], - [ - 0.173864, - "o" - ], - [ - 0.388954, - ":" - ], - [ - 0.144865, - ":" - ], - [ - 0.347420, - "b" - ], - [ - 0.105814, - "a" - ], - [ - 0.198728, - "c" - ], - [ - 0.096349, - "k" - ], - [ - 0.261559, - "u" - ], - [ - 0.241998, - "p" - ], - [ - 0.240033, - "2" - ], - [ - 0.981903, - "\r\n" - ], - [ - 2.000000, - "\u001b]0;tw@tux: ~/borg/demo\u0007tw@tux:~/borg/demo$ " - ], - [ - 0.937291, - "#" - ], - [ - 0.994897, - " " - ], - [ - 0.151752, - "c" - ], - [ - 0.096956, - "h" - ], - [ - 0.337975, - "e" - ], - [ - 0.207037, - "c" - ], - [ - 0.177028, - "k" - ], - [ - 0.740370, - " " - ], - [ - 0.330206, - "i" - ], - [ - 0.177976, - "f" - ], - [ - 0.218757, - " " - ], - [ - 0.329345, - "o" - ], - [ - 0.098735, - "r" - ], - [ - 0.098576, - "i" - ], - [ - 0.103157, - "g" - ], - [ - 0.107275, - "i" - ], - [ - 0.117332, - "n" - ], - [ - 0.194072, - "a" - ], - [ - 0.211456, - "l" - ], - [ - 0.197712, - " " - ], - [ - 0.189172, - "d" - ], - [ - 0.163930, - "a" - ], - [ - 0.188334, - "t" - ], - [ - 0.165129, - "a" - ], - [ - 0.220652, - " " - ], - [ - 0.224411, - "a" - ], - [ - 0.136137, - "n" - ], - [ - 0.155260, - "d" - ], - [ - 0.074238, - " " - ], - [ - 0.104154, - "r" - ], - [ - 0.690499, - "e" - ], - [ - 0.193678, - "s" - ], - [ - 0.165163, - "t" - ], - [ - 0.165594, - "o" - ], - [ - 0.111779, - "r" - ], - [ - 0.135625, - "e" - ], - [ - 0.202851, - "d" - ], - [ - 0.096040, - " " - ], - [ - 0.165090, - "d" - ], - [ - 0.155594, - "a" - ], - [ - 0.220606, - "t" - ], - [ - 0.163143, - "a" - ], - [ - 0.174099, - " " - ], - [ - 0.209780, - "d" - ], - [ - 0.166062, - "i" - ], - [ - 0.084688, - "f" - ], - [ - 0.140851, - "f" - ], - [ - 0.204458, - "e" - ], - [ - 0.088661, - "r" - ], - [ - 0.334162, - "s" - ], - [ - 0.904233, - ":" - ], - [ - 0.590489, - "\r\n" - ], - [ - 0.000283, - "\u001b]0;tw@tux: ~/borg/demo\u0007tw@tux:~/borg/demo$ " - ], - [ - 0.503183, - "d" - ], - [ - 0.082614, - "i" - ], - [ - 0.216272, - "f" - ], - [ - 0.123813, - "f" - ], - [ - 0.183603, - " " - ], - [ - 0.302144, - "-" - ], - [ - 0.150946, - "r" - ], - [ - 0.152436, - " " - ], - [ - 2.000000, - "d" - ], - [ - 0.196047, - "a" - ], - [ - 0.206372, - "t" - ], - [ - 0.146051, - "a" - ], - [ - 0.326306, - "." - ], - [ - 0.363408, - "o" - ], - [ - 0.269988, - "rig/" - ], - [ - 0.776581, - " " - ], - [ - 0.137720, - "d" - ], - [ - 0.156080, - "a" - ], - [ - 0.242275, - "\u0007" - ], - [ - 0.000020, - "ta" - ], - [ - 0.872887, - "/" - ], - [ - 0.273993, - "\r\n" - ], - [ - 2.000000, - "\u001b]0;tw@tux: ~/borg/demo\u0007tw@tux:~/borg/demo$ " - ], - [ - 0.488581, - "#" - ], - [ - 0.234021, - " " - ], - [ - 0.380938, - "n" - ], - [ - 0.240685, - "o" - ], - [ - 0.204436, - " " - ], - [ - 0.390794, - "o" - ], - [ - 0.225563, - "u" - ], - [ - 0.167295, - "t" - ], - [ - 0.140625, - "p" - ], - [ - 0.183668, - "u" - ], - [ - 0.106161, - "t" - ], - [ - 0.132063, - " " - ], - [ - 0.204757, - "m" - ], - [ - 0.082693, - "e" - ], - [ - 0.216428, - "a" - ], - [ - 0.121584, - "n" - ], - [ - 0.127398, - "s" - ], - [ - 0.264644, - " " - ], - [ - 0.201524, - "i" - ], - [ - 0.110738, - "t" - ], - [ - 0.120653, - " " - ], - [ - 0.311187, - "d" - ], - [ - 0.119826, - "o" - ], - [ - 0.082654, - "e" - ], - [ - 0.182518, - "s" - ], - [ - 0.096372, - " " - ], - [ - 0.192821, - "n" - ], - [ - 0.193829, - "o" - ], - [ - 0.065739, - "t" - ], - [ - 0.678808, - "." - ], - [ - 0.246797, - " " - ], - [ - 0.520369, - "f" - ], - [ - 0.058288, - "i" - ], - [ - 0.064783, - "n" - ], - [ - 0.104851, - "e" - ], - [ - 0.292910, - "." - ], - [ - 0.174086, - " " - ], - [ - 0.226556, - ":" - ], - [ - 0.249808, - ")" - ], - [ - 2.000000, - "\r\n" - ], - [ - 0.000261, - "\u001b]0;tw@tux: ~/borg/demo\u0007tw@tux:~/borg/demo$ " - ], - [ - 0.477759, - "#" - ], - [ - 0.223699, - " " - ], - [ - 0.237979, - "l" - ], - [ - 0.152769, - "i" - ], - [ - 0.135150, - "s" - ], - [ - 0.068576, - "t" - ], - [ - 0.100516, - "i" - ], - [ - 0.078648, - "n" - ], - [ - 0.099435, - "g" - ], - [ - 0.157388, - " " - ], - [ - 0.115327, - "t" - ], - [ - 0.133738, - "h" - ], - [ - 0.135662, - "e" - ], - [ - 0.100677, - " " - ], - [ - 0.180392, - "r" - ], - [ - 0.190922, - "e" - ], - [ - 0.093920, - "p" - ], - [ - 0.173588, - "o" - ], - [ - 0.193023, - " " - ], - [ - 0.206907, - "c" - ], - [ - 0.106376, - "o" - ], - [ - 0.175291, - "n" - ], - [ - 0.080726, - "t" - ], - [ - 0.179258, - "e" - ], - [ - 0.101491, - "n" - ], - [ - 0.096807, - "t" - ], - [ - 0.211455, - "s" - ], - [ - 0.508210, - ":" - ], - [ - 0.373837, - "\r\n" - ], - [ - 0.000249, - "\u001b]0;tw@tux: ~/borg/demo\u0007tw@tux:~/borg/demo$ " - ], - [ - 0.559782, - "b" - ], - [ - 0.116587, - "o" - ], - [ - 0.139513, - "r" - ], - [ - 0.072751, - "g" - ], - [ - 0.103968, - " " - ], - [ - 0.984928, - "l" - ], - [ - 0.173603, - "i" - ], - [ - 0.112444, - "s" - ], - [ - 0.066704, - "t" - ], - [ - 0.114771, - " " - ], - [ - 0.263745, - "r" - ], - [ - 0.113121, - "e" - ], - [ - 0.126283, - "p" - ], - [ - 0.187453, - "o" - ], - [ - 0.409044, - "\r\n" - ], - [ - 0.360675, - "backup1 Sat Oct 24 22:27:43 2015" - ], - [ - 0.000011, - "\r\n" - ], - [ - 0.000006, - "backup2 Sat Oct 24 22:28:27 2015" - ], - [ - 0.000005, - "\r\n" - ], - [ - 0.027766, - "\u001b]0;tw@tux: ~/borg/demo\u0007tw@tux:~/borg/demo$ " - ], - [ - 0.637813, - "#" - ], - [ - 0.257629, - " " - ], - [ - 0.231710, - "l" - ], - [ - 0.213387, - "i" - ], - [ - 0.132149, - "s" - ], - [ - 0.244957, - "t" - ], - [ - 0.180264, - "i" - ], - [ - 0.082882, - "n" - ], - [ - 0.142810, - "g" - ], - [ - 0.134815, - " " - ], - [ - 0.167455, - "s" - ], - [ - 0.114155, - "o" - ], - [ - 0.106847, - "m" - ], - [ - 0.070629, - "e" - ], - [ - 0.507340, - " " - ], - [ - 0.234237, - "b" - ], - [ - 0.070181, - "a" - ], - [ - 0.220534, - "c" - ], - [ - 0.092316, - "k" - ], - [ - 0.257003, - "u" - ], - [ - 0.233598, - "p" - ], - [ - 0.201484, - " " - ], - [ - 0.124810, - "a" - ], - [ - 0.084732, - "r" - ], - [ - 0.249719, - "c" - ], - [ - 0.119605, - "h" - ], - [ - 0.203875, - "i" - ], - [ - 0.076269, - "v" - ], - [ - 0.174299, - "e" - ], - [ - 0.109711, - " " - ], - [ - 0.238294, - "c" - ], - [ - 0.102351, - "o" - ], - [ - 0.155761, - "n" - ], - [ - 0.060278, - "t" - ], - [ - 0.179564, - "e" - ], - [ - 0.112342, - "n" - ], - [ - 0.078100, - "t" - ], - [ - 0.190203, - "s" - ], - [ - 0.865560, - " " - ], - [ - 0.297799, - "(" - ], - [ - 0.225741, - "s" - ], - [ - 0.080329, - "h" - ], - [ - 0.233668, - "o" - ], - [ - 0.127773, - "r" - ], - [ - 0.190065, - "t" - ], - [ - 0.187679, - "e" - ], - [ - 0.147219, - "n" - ], - [ - 0.064472, - "e" - ], - [ - 0.188512, - "d" - ], - [ - 0.459222, - ")" - ], - [ - 0.723165, - ":" - ], - [ - 0.645995, - "\r\n" - ], - [ - 0.000096, - "\u001b]0;tw@tux: ~/borg/demo\u0007tw@tux:~/borg/demo$ " - ], - [ - 0.446688, - "b" - ], - [ - 0.145841, - "o" - ], - [ - 0.105605, - "r" - ], - [ - 0.088953, - "g" - ], - [ - 0.120803, - " " - ], - [ - 0.227780, - "l" - ], - [ - 0.175052, - "i" - ], - [ - 0.106579, - "s" - ], - [ - 0.058441, - "t" - ], - [ - 0.093196, - " " - ], - [ - 0.172940, - "r" - ], - [ - 0.134731, - "e" - ], - [ - 0.119062, - "p" - ], - [ - 0.183075, - "o" - ], - [ - 0.388321, - ":" - ], - [ - 0.140589, - ":" - ], - [ - 0.324109, - "b" - ], - [ - 0.058606, - "a" - ], - [ - 0.205450, - "c" - ], - [ - 0.105362, - "k" - ], - [ - 0.235009, - "u" - ], - [ - 0.243485, - "p" - ], - [ - 0.485432, - "2" - ], - [ - 0.148177, - " " - ], - [ - 0.632383, - "|" - ], - [ - 0.389914, - " " - ], - [ - 0.174128, - "t" - ], - [ - 0.201473, - "a" - ], - [ - 0.116517, - "i" - ], - [ - 0.225072, - "l" - ], - [ - 0.699624, - "\r\n" - ], - [ - 2.000000, - "-rw-rw-r-- tw tw 5516 Jul 21 19:10 data/linux-4.1.8/virt/kvm/async_pf.c\r\n" - ], - [ - 0.000019, - "-rw-rw-r-- tw tw 1120 Jul 21 19:10 data/linux-4.1.8/virt/kvm/async_pf.h\r\n-rw-rw-r-- tw tw 4215 Jul 21 19:10 data/linux-4.1.8/virt/kvm/coalesced_mmio.c\r\n-rw-rw-r-- tw tw 915 Jul 21 19:10 data/linux-4.1.8/virt/kvm/coalesced_mmio.h\r\n-rw-rw-r-- tw tw 22879 Jul 21 19:10 data/linux-4.1.8/virt/kvm/eventfd.c\r\n-rw-rw-r-- tw tw 5563 Jul 21 19:10 data/linux-4.1.8/virt/kvm/irqchip.c\r\n-rw-rw-r-- tw tw 79385 Jul 21 19:10 data/linux-4.1.8/virt/kvm/kvm_main.c\r\n" - ], - [ - 0.000011, - "-rw-rw-r-- tw tw 6132 Jul 21 19:10 data/linux-4.1.8/virt/kvm/vfio.c\r\n-rw-rw-r-- tw tw 250 Jul 21 19:10 data/linux-4.1.8/virt/kvm/vfio.h\r\n" - ], - [ - 0.000009, - "-rw-rw-r-- tw tw 15 Oct 24 22:28 data/one_file_more\r\n" - ], - [ - 0.000389, - "\u001b]0;tw@tux: ~/borg/demo\u0007tw@tux:~/borg/demo$ " - ], - [ - 1.000000, - "#" - ], - [ - 0.337351, - " " - ], - [ - 0.147170, - "e" - ], - [ - 0.235736, - "a" - ], - [ - 0.251314, - "s" - ], - [ - 0.471185, - "y" - ], - [ - 0.277723, - "," - ], - [ - 0.204225, - " " - ], - [ - 0.182231, - "i" - ], - [ - 0.174424, - "s" - ], - [ - 0.074677, - "n" - ], - [ - 0.786274, - "'" - ], - [ - 0.264836, - "t" - ], - [ - 0.330352, - " " - ], - [ - 0.266876, - "i" - ], - [ - 0.112564, - "t" - ], - [ - 0.897299, - "?" - ], - [ - 0.623501, - " " - ], - [ - 0.656625, - "t" - ], - [ - 0.115934, - "h" - ], - [ - 0.625213, - "a" - ], - [ - 0.588409, - "t" - ], - [ - 0.160071, - " " - ], - [ - 0.830693, - "i" - ], - [ - 0.163118, - "s" - ], - [ - 0.075663, - " " - ], - [ - 0.186138, - "a" - ], - [ - 0.109916, - "l" - ], - [ - 0.137005, - "l" - ], - [ - 0.171009, - " " - ], - [ - 0.153348, - "y" - ], - [ - 0.132919, - "o" - ], - [ - 0.568100, - "u" - ], - [ - 0.211350, - " " - ], - [ - 0.195450, - "n" - ], - [ - 0.257974, - "e" - ], - [ - 0.185529, - "e" - ], - [ - 0.265130, - "d" - ], - [ - 0.129116, - " " - ], - [ - 0.169264, - "t" - ], - [ - 0.148964, - "o" - ], - [ - 0.437043, - " " - ], - [ - 0.431197, - "k" - ], - [ - 0.219557, - "n" - ], - [ - 0.257996, - "o" - ], - [ - 0.158826, - "w" - ], - [ - 0.406870, - " " - ], - [ - 0.659664, - "f" - ], - [ - 0.130963, - "o" - ], - [ - 0.125395, - "r" - ], - [ - 0.613713, - " " - ], - [ - 0.646957, - "b" - ], - [ - 0.154695, - "a" - ], - [ - 0.259741, - "s" - ], - [ - 0.156692, - "i" - ], - [ - 0.124345, - "c" - ], - [ - 0.513209, - "\r\n" - ], - [ - 0.000296, - "\u001b]0;tw@tux: ~/borg/demo\u0007tw@tux:~/borg/demo$ " - ], - [ - 0.965828, - "#" - ], - [ - 0.232285, - " " - ], - [ - 0.266818, - "u" - ], - [ - 0.132723, - "s" - ], - [ - 0.216208, - "a" - ], - [ - 0.206362, - "g" - ], - [ - 0.142608, - "e" - ], - [ - 2.000000, - "." - ], - [ - 0.238868, - " " - ], - [ - 0.302986, - "i" - ], - [ - 0.196338, - "f" - ], - [ - 0.092936, - " " - ], - [ - 0.197594, - "y" - ], - [ - 0.122297, - "o" - ], - [ - 0.175360, - "u" - ], - [ - 0.145063, - " " - ], - [ - 0.313719, - "l" - ], - [ - 0.169678, - "i" - ], - [ - 0.185628, - "k" - ], - [ - 0.120660, - "e" - ], - [ - 0.078389, - " " - ], - [ - 0.648628, - "#" - ], - [ - 0.337514, - "b" - ], - [ - 0.108598, - "o" - ], - [ - 0.123792, - "r" - ], - [ - 0.136099, - "g" - ], - [ - 0.235539, - "b" - ], - [ - 0.091671, - "a" - ], - [ - 0.208697, - "c" - ], - [ - 0.100567, - "k" - ], - [ - 0.227477, - "u" - ], - [ - 0.236900, - "p" - ], - [ - 0.302154, - "," - ], - [ - 0.207291, - " " - ], - [ - 0.205656, - "s" - ], - [ - 0.123737, - "p" - ], - [ - 0.142016, - "r" - ], - [ - 0.197260, - "e" - ], - [ - 0.197471, - "a" - ], - [ - 0.104498, - "d" - ], - [ - 0.163267, - " " - ], - [ - 0.178420, - "t" - ], - [ - 0.091669, - "h" - ], - [ - 0.107735, - "e" - ], - [ - 0.102742, - " " - ], - [ - 0.211413, - "w" - ], - [ - 0.124959, - "o" - ], - [ - 0.105787, - "r" - ], - [ - 0.231403, - "d" - ], - [ - 0.299061, - "!" - ], - [ - 2.000000, - "\r\n" - ], - [ - 0.000307, - "\u001b]0;tw@tux: ~/borg/demo\u0007tw@tux:~/borg/demo$ " - ], - [ - 0.304768, - "#" - ], - [ - 0.229433, - " " - ], - [ - 0.647220, - "t" - ], - [ - 0.070692, - "h" - ], - [ - 0.349432, - "a" - ], - [ - 0.112924, - "n" - ], - [ - 0.207031, - "k" - ], - [ - 0.567641, - "s" - ], - [ - 0.121708, - " " - ], - [ - 0.135723, - "f" - ], - [ - 0.139102, - "o" - ], - [ - 0.060453, - "r" - ], - [ - 0.152408, - " " - ], - [ - 0.116234, - "v" - ], - [ - 0.142885, - "i" - ], - [ - 0.106596, - "e" - ], - [ - 0.231115, - "w" - ], - [ - 0.416046, - "i" - ], - [ - 0.086563, - "n" - ], - [ - 0.144009, - "g" - ], - [ - 0.725139, - "!" - ], - [ - 0.299810, - "\r\n" - ], - [ - 0.000250, - "\u001b]0;tw@tux: ~/borg/demo\u0007tw@tux:~/borg/demo$ " - ], - [ - 0.710767, - "exit" - ], - [ - 0.000006, - "\r\n" - ] - ] -} diff --git a/docs/misc/asciinema/install_and_basics.txt b/docs/misc/asciinema/install_and_basics.txt deleted file mode 100644 index 135db1702a..0000000000 --- a/docs/misc/asciinema/install_and_basics.txt +++ /dev/null @@ -1,51 +0,0 @@ -# borgbackup - installation and basic usage - -# I have already downloaded the binary release from github: -ls -l -# binary file + GPG signature - -# verifying whether the binary is valid: -gpg --verify borg-linux64.asc borg-linux64 - -# install it as "borg": -cp borg-linux64 ~/bin/borg - -# making it executable: -chmod +x ~/bin/borg - -# yay, installation done! let's make backups! - -# creating a repository: -borg init repo - -# creating our first backup with stuff from "data" directory: -borg create --stats --progress --compression lz4 repo::backup1 data - -# changing the data slightly: -echo "some more data" > data/one_file_more - -# creating another backup: -borg create --stats --progress repo::backup2 data - -# that was much faster! it recognized/deduplicated unchanged files. -# see the "Deduplicated size" column for "This archive"! :) - -# extracting a backup archive: -mv data data.orig -borg extract repo::backup2 - -# checking if restored data differs from original data: -diff -r data.orig data - -# no, it doesn't! :) - -# listing the repo contents: -borg list repo - -# listing the backup2 archive contents (shortened): -borg list repo::backup2 | tail - -# easy, isn't it? - -# if you like #borgbackup, spread the word! - From 4eadb59c10d3d9afeb1ce569119423532060f431 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 16 Aug 2017 01:52:33 +0200 Subject: [PATCH 012/798] ignore corrupt files cache, fixes #2939 ignore the files cache when corrupt and emit a warning message so the users notices that there is a problem. (cherry picked from commit 5beaa5bd025e27e1981607ffece36106d6027418) --- src/borg/cache.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/borg/cache.py b/src/borg/cache.py index c1c7033261..b1b440ac08 100644 --- a/src/borg/cache.py +++ b/src/borg/cache.py @@ -503,10 +503,16 @@ def _read_files(self): if not data: break u.feed(data) - for path_hash, item in u: - entry = FileCacheEntry(*item) - # in the end, this takes about 240 Bytes per file - self.files[path_hash] = msgpack.packb(entry._replace(age=entry.age + 1)) + try: + for path_hash, item in u: + entry = FileCacheEntry(*item) + # in the end, this takes about 240 Bytes per file + self.files[path_hash] = msgpack.packb(entry._replace(age=entry.age + 1)) + except (TypeError, ValueError) as exc: + logger.warning('The files cache seems corrupt, ignoring it. ' + 'Expect lower performance. [%s]' % str(exc)) + self.files = {} + return def begin_txn(self): # Initialize transaction snapshot From dc6049cb738eae9952bcc2374c6b13fba1bcb5f6 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 20 Aug 2017 21:30:41 +0200 Subject: [PATCH 013/798] delete various nogil and threading related lines --- src/borg/compress.pyx | 6 ++---- src/borg/crypto/low_level.pyx | 6 ++---- src/borg/helpers.py | 10 ++++------ 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/borg/compress.pyx b/src/borg/compress.pyx index 16f7bcb548..6c1d4c31ff 100644 --- a/src/borg/compress.pyx +++ b/src/borg/compress.pyx @@ -126,8 +126,7 @@ class LZ4(CompressorBase): osize = LZ4_compressBound(isize) buf = buffer.get(osize) dest = buf - with nogil: - osize = LZ4_compress_limitedOutput(source, dest, isize, osize) + osize = LZ4_compress_limitedOutput(source, dest, isize, osize) if not osize: raise Exception('lz4 compress failed') return super().compress(dest[:osize]) @@ -150,8 +149,7 @@ class LZ4(CompressorBase): except MemoryError: raise DecompressionError('MemoryError') dest = buf - with nogil: - rsize = LZ4_decompress_safe(source, dest, isize, osize) + rsize = LZ4_decompress_safe(source, dest, isize, osize) if rsize >= 0: break if osize > 2 ** 27: # 128MiB (should be enough, considering max. repo obj size and very good compression) diff --git a/src/borg/crypto/low_level.pyx b/src/borg/crypto/low_level.pyx index a68cd820f6..db0ea8af6b 100644 --- a/src/borg/crypto/low_level.pyx +++ b/src/borg/crypto/low_level.pyx @@ -207,8 +207,7 @@ def hmac_sha256(key, data): cdef int key_len = len(key) cdef unsigned char md[32] try: - with nogil: - rc = HMAC(EVP_sha256(), key_ptr, key_len, data_buf.buf, data_buf.len, md, NULL) + rc = HMAC(EVP_sha256(), key_ptr, key_len, data_buf.buf, data_buf.len, md, NULL) if rc != md: raise Exception('HMAC(EVP_sha256) failed') finally: @@ -219,8 +218,7 @@ def hmac_sha256(key, data): cdef blake2b_update_from_buffer(blake2b_state *state, obj): cdef Py_buffer buf = ro_buffer(obj) try: - with nogil: - rc = blake2b_update(state, buf.buf, buf.len) + rc = blake2b_update(state, buf.buf, buf.len) if rc == -1: raise Exception('blake2b_update() failed') finally: diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 242b939651..86368a9d56 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -19,7 +19,6 @@ import subprocess import sys import textwrap -import threading import time import uuid from binascii import hexlify @@ -809,7 +808,7 @@ def format_archive(archive): class Buffer: """ - provide a thread-local buffer + managed buffer (like a resizable bytearray) """ class MemoryLimitExceeded(Error, OSError): @@ -822,13 +821,12 @@ def __init__(self, allocator, size=4096, limit=None): """ assert callable(allocator), 'must give alloc(size) function as first param' assert limit is None or size <= limit, 'initial size must be <= limit' - self._thread_local = threading.local() self.allocator = allocator self.limit = limit self.resize(size, init=True) def __len__(self): - return len(self._thread_local.buffer) + return len(self.buffer) def resize(self, size, init=False): """ @@ -840,7 +838,7 @@ def resize(self, size, init=False): if self.limit is not None and size > self.limit: raise Buffer.MemoryLimitExceeded(size, self.limit) if init or len(self) < size: - self._thread_local.buffer = self.allocator(size) + self.buffer = self.allocator(size) def get(self, size=None, init=False): """ @@ -849,7 +847,7 @@ def get(self, size=None, init=False): """ if size is not None: self.resize(size, init) - return self._thread_local.buffer + return self.buffer @lru_cache(maxsize=None) From ff93b6b972e66ead1b224cd36a97bf4f667d9707 Mon Sep 17 00:00:00 2001 From: Lukas Fleischer Date: Sun, 6 Aug 2017 16:46:08 +0200 Subject: [PATCH 014/798] Detect non-upgraded Attic repositories When opening a repository, always try to read the magic number of the latest segment and compare it to the Attic segment magic (unless the repository is opened for upgrading). If an Attic segment is detected, raise a dedicated exception, telling the user to upgrade the repository first. Fixes #1933. (cherry picked from commit 0943b322e3ee286e7672859fe995506c589bcb4e) --- docs/internals/frontends.rst | 2 ++ src/borg/remote.py | 5 +++++ src/borg/repository.py | 19 ++++++++++++++++++- src/borg/testsuite/archiver.py | 22 ++++++++++++++++++++++ src/borg/testsuite/upgrader.py | 8 ++++---- src/borg/upgrader.py | 1 + 6 files changed, 52 insertions(+), 5 deletions(-) diff --git a/docs/internals/frontends.rst b/docs/internals/frontends.rst index c41d427ebe..677db738df 100644 --- a/docs/internals/frontends.rst +++ b/docs/internals/frontends.rst @@ -499,6 +499,8 @@ Errors Insufficient free space to complete transaction (required: {}, available: {}). Repository.InvalidRepository {} is not a valid repository. Check repo config. + Repository.AtticRepository + Attic repository detected. Please run "borg upgrade {}". Repository.ObjectNotFound Object with key {} not found in repository {}. diff --git a/src/borg/remote.py b/src/borg/remote.py index d131a8266f..a508e04f97 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -733,6 +733,11 @@ def handle_error(unpacked): raise IntegrityError('(not available)') else: raise IntegrityError(args[0].decode()) + elif error == 'AtticRepository': + if old_server: + raise Repository.AtticRepository('(not available)') + else: + raise Repository.AtticRepository(args[0].decode()) elif error == 'PathNotAllowed': if old_server: raise PathNotAllowed('(unknown)') diff --git a/src/borg/repository.py b/src/borg/repository.py index a61e4e9491..942b862dc4 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -30,6 +30,8 @@ MAGIC = b'BORG_SEG' MAGIC_LEN = len(MAGIC) +ATTIC_MAGIC = b'ATTICSEG' +assert len(ATTIC_MAGIC) == MAGIC_LEN TAG_PUT = 0 TAG_DELETE = 1 TAG_COMMIT = 2 @@ -116,6 +118,9 @@ class AlreadyExists(Error): class InvalidRepository(Error): """{} is not a valid repository. Check repo config.""" + class AtticRepository(Error): + """Attic repository detected. Please run "borg upgrade {}".""" + class CheckNeeded(ErrorWithTraceback): """Inconsistency detected. Please run "borg check {}".""" @@ -134,7 +139,7 @@ class StorageQuotaExceeded(Error): """The storage quota ({}) has been exceeded ({}). Try deleting some archives.""" def __init__(self, path, create=False, exclusive=False, lock_wait=None, lock=True, - append_only=False, storage_quota=None): + append_only=False, storage_quota=None, check_segment_magic=True): self.path = os.path.abspath(path) self._location = Location('file://%s' % self.path) self.io = None # type: LoggedIO @@ -154,6 +159,7 @@ def __init__(self, path, create=False, exclusive=False, lock_wait=None, lock=Tru self.storage_quota = storage_quota self.storage_quota_use = 0 self.transaction_doomed = None + self.check_segment_magic = check_segment_magic def __del__(self): if self.lock: @@ -375,6 +381,12 @@ def open(self, path, exclusive, lock_wait=None, lock=True): self.storage_quota = self.config.getint('repository', 'storage_quota', fallback=0) self.id = unhexlify(self.config.get('repository', 'id').strip()) self.io = LoggedIO(self.path, self.max_segment_size, self.segments_per_dir) + if self.check_segment_magic: + # read a segment and check whether we are dealing with a non-upgraded Attic repository + segment = self.io.get_latest_segment() + if segment is not None and self.io.get_segment_magic(segment) == ATTIC_MAGIC: + self.close() + raise self.AtticRepository(path) def close(self): if self.lock: @@ -1250,6 +1262,11 @@ def segment_exists(self, segment): def segment_size(self, segment): return os.path.getsize(self.segment_filename(segment)) + def get_segment_magic(self, segment): + fd = self.get_fd(segment) + fd.seek(0) + return fd.read(MAGIC_LEN) + def iter_objects(self, segment, offset=0, include_data=False, read_data=True): """ Return object iterator for *segment*. diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index a26bc47cfd..5242d6dbad 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -55,6 +55,7 @@ from . import BaseTestCase, changedir, environment_variable, no_selinux from . import are_symlinks_supported, are_hardlinks_supported, are_fifos_supported, is_utime_fully_supported from .platform import fakeroot_detected +from .upgrader import attic_repo from . import key @@ -2731,6 +2732,27 @@ def test_extract_hardlinks(self): assert os.stat('input/dir1/aaaa').st_nlink == 2 assert os.stat('input/dir1/source2').st_nlink == 2 + def test_detect_attic_repo(self): + path = attic_repo(self.repository_path) + cmds = [ + ['create', path + '::test', self.tmpdir], + ['extract', path + '::test'], + ['check', path], + ['rename', path + '::test', 'newname'], + ['list', path], + ['delete', path], + ['prune', path], + ['info', path + '::test'], + ['mount', path, self.tmpdir], + ['key', 'export', path, 'exported'], + ['key', 'import', path, 'import'], + ['change-passphrase', path], + ['break-lock', path], + ] + for args in cmds: + output = self.cmd(*args, fork=True, exit_code=2) + assert 'Attic repository detected.' in output + @unittest.skipUnless('binary' in BORG_EXES, 'no borg.exe available') class ArchiverTestCaseBinary(ArchiverTestCase): diff --git a/src/borg/testsuite/upgrader.py b/src/borg/testsuite/upgrader.py index 3fd7500c3e..08c0693bcc 100644 --- a/src/borg/testsuite/upgrader.py +++ b/src/borg/testsuite/upgrader.py @@ -85,8 +85,8 @@ def test_convert_segments(attic_repo, inplace): :param attic_repo: a populated attic repository (fixture) """ repo_path = attic_repo - # check should fail because of magic number - assert not repo_valid(repo_path) + with pytest.raises(Repository.AtticRepository): + repo_valid(repo_path) repository = AtticRepositoryUpgrader(repo_path, create=False) with repository: segments = [filename for i, filename in repository.io.segment_iterator()] @@ -149,8 +149,8 @@ def test_convert_all(attic_repo, attic_key_file, inplace): """ repo_path = attic_repo - # check should fail because of magic number - assert not repo_valid(repo_path) + with pytest.raises(Repository.AtticRepository): + repo_valid(repo_path) def stat_segment(path): return os.stat(os.path.join(path, 'data', '0', '0')) diff --git a/src/borg/upgrader.py b/src/borg/upgrader.py index 0b92ce8e23..1044f649e9 100644 --- a/src/borg/upgrader.py +++ b/src/borg/upgrader.py @@ -19,6 +19,7 @@ class AtticRepositoryUpgrader(Repository): def __init__(self, *args, **kw): kw['lock'] = False # do not create borg lock files (now) in attic repo + kw['check_segment_magic'] = False # skip the Attic check when upgrading super().__init__(*args, **kw) def upgrade(self, dryrun=True, inplace=False, progress=False): From fa65c9b1434e4299639ee2e3c7e4439a5f8d20db Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Mon, 7 Aug 2017 13:01:33 +0200 Subject: [PATCH 015/798] list: fix weird mixup of mtime/isomtime (cherry picked from commit 2ff29891f197623c54d7f40147b7411ece67524c) --- docs/changes.rst | 13 +++++++++++++ src/borg/archiver.py | 4 ++-- src/borg/testsuite/archiver.py | 2 +- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index 25909cf30e..bed3b26d1c 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -131,6 +131,19 @@ The best check that everything is ok is to run a dry-run extraction:: Changelog ========= +Version 1.1.0rc2 (not released yet) +------------------------------------ + +Compatibility notes: + +- list: corrected mix-up of "isomtime" and "mtime" formats. Previously, + "isomtime" was the default but produced a verbose human format, + while "mtime" produced a ISO-8601-like format. + The behaviours have been swapped (so "mtime" is human, "isomtime" is ISO-like), + and the default is now "mtime". + "isomtime" is now a real ISO-8601 format ("T" between date and time, not a space). + + Version 1.1.0rc1 (2017-07-24) ----------------------------- diff --git a/src/borg/archiver.py b/src/borg/archiver.py index a3db174acf..0065cd8209 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -1334,7 +1334,7 @@ def _list_archive(self, args, repository, manifest, key, write): elif args.short: format = "{path}{NL}" else: - format = "{mode} {user:6} {group:6} {size:8} {isomtime} {path}{extra}{NL}" + format = "{mode} {user:6} {group:6} {size:8} {mtime} {path}{extra}{NL}" def _list_inner(cache): archive = Archive(repository, key, manifest, args.location.archive, cache=cache, @@ -3119,7 +3119,7 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): help='only print file/directory names, nothing else') subparser.add_argument('--format', '--list-format', metavar='FORMAT', dest='format', help='specify format for file listing ' - '(default: "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NL}")') + '(default: "{mode} {user:6} {group:6} {size:8d} {mtime} {path}{extra}{NL}")') subparser.add_argument('--json', action='store_true', help='Only valid for listing repository contents. Format output as JSON. ' 'The form of ``--format`` is ignored, ' diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index a26bc47cfd..6ae7d47100 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -1787,7 +1787,7 @@ def test_list_format(self): output_warn = self.cmd('list', '--list-format', '-', test_archive) self.assert_in('--list-format" has been deprecated.', output_warn) output_1 = self.cmd('list', test_archive) - output_2 = self.cmd('list', '--format', '{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}', test_archive) + output_2 = self.cmd('list', '--format', '{mode} {user:6} {group:6} {size:8d} {mtime} {path}{extra}{NEWLINE}', test_archive) output_3 = self.cmd('list', '--format', '{mtime:%s} {path}{NL}', test_archive) self.assertEqual(output_1, output_2) self.assertNotEqual(output_1, output_3) From 008571228f52d89a6235bf20b1e4a9644c479d4c Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Wed, 16 Aug 2017 17:57:08 +0200 Subject: [PATCH 016/798] one datetime formatter to rule them all (cherry picked from commit a836f451ab239da516fa9232c11005c62f7e04a3) --- docs/internals/frontends.rst | 21 +++++++------ src/borg/archive.py | 10 +++--- src/borg/helpers.py | 56 +++++++++++++++++++--------------- src/borg/testsuite/archiver.py | 2 ++ 4 files changed, 49 insertions(+), 40 deletions(-) diff --git a/docs/internals/frontends.rst b/docs/internals/frontends.rst index c41d427ebe..0441caa53c 100644 --- a/docs/internals/frontends.rst +++ b/docs/internals/frontends.rst @@ -209,8 +209,9 @@ Standard output *stdout* is different and more command-dependent than logging. Commands like :ref:`borg_info`, :ref:`borg_create` and :ref:`borg_list` implement a ``--json`` option which turns their regular output into a single JSON object. -Dates are formatted according to ISO-8601 with the strftime format string '%a, %Y-%m-%d %H:%M:%S', -e.g. *Sat, 2016-02-25 23:50:06*. +Dates are formatted according to ISO-8601 in local time. Neither an explicit time zone nor microseconds +are specified *at this time* (subject to change). The equivalent strftime format string is '%Y-%m-%dT%H:%M:%S', +e.g. 2017-08-07T12:27:20. The root object at least contains a *repository* key with an object containing: @@ -267,7 +268,7 @@ Example *borg info* output:: }, "repository": { "id": "0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23", - "last_modified": "Mon, 2017-02-27 21:21:58", + "last_modified": "2017-08-07T12:27:20", "location": "/home/user/testrepo" }, "security_dir": "/home/user/.config/borg/security/0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23", @@ -328,7 +329,7 @@ Example of a simple archive listing (``borg list --last 1 --json``):: { "id": "80cd07219ad725b3c5f665c1dcf119435c4dee1647a560ecac30f8d40221a46a", "name": "host-system-backup-2017-02-27", - "start": "Mon, 2017-02-27 21:21:52" + "start": "2017-08-07T12:27:20" } ], "encryption": { @@ -336,7 +337,7 @@ Example of a simple archive listing (``borg list --last 1 --json``):: }, "repository": { "id": "0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23", - "last_modified": "Mon, 2017-02-27 21:21:58", + "last_modified": "2017-08-07T12:27:20", "location": "/home/user/repository" } } @@ -354,14 +355,14 @@ The same archive with more information (``borg info --last 1 --json``):: ], "comment": "", "duration": 5.641542, - "end": "Mon, 2017-02-27 21:21:58", + "end": "2017-02-27T12:27:20", "hostname": "host", "id": "80cd07219ad725b3c5f665c1dcf119435c4dee1647a560ecac30f8d40221a46a", "limits": { "max_archive_size": 0.0001330855110409714 }, "name": "host-system-backup-2017-02-27", - "start": "Mon, 2017-02-27 21:21:52", + "start": "2017-02-27T12:27:20", "stats": { "compressed_size": 1880961894, "deduplicated_size": 2791, @@ -387,7 +388,7 @@ The same archive with more information (``borg info --last 1 --json``):: }, "repository": { "id": "0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23", - "last_modified": "Mon, 2017-02-27 21:21:58", + "last_modified": "2017-08-07T12:27:20", "location": "/home/user/repository" } } @@ -405,8 +406,8 @@ Refer to the *borg list* documentation for the available keys and their meaning. Example (excerpt) of ``borg list --json-lines``:: - {"type": "d", "mode": "drwxr-xr-x", "user": "user", "group": "user", "uid": 1000, "gid": 1000, "path": "linux", "healthy": true, "source": "", "linktarget": "", "flags": null, "isomtime": "Sat, 2016-05-07 19:46:01", "size": 0} - {"type": "d", "mode": "drwxr-xr-x", "user": "user", "group": "user", "uid": 1000, "gid": 1000, "path": "linux/baz", "healthy": true, "source": "", "linktarget": "", "flags": null, "isomtime": "Sat, 2016-05-07 19:46:01", "size": 0} + {"type": "d", "mode": "drwxr-xr-x", "user": "user", "group": "user", "uid": 1000, "gid": 1000, "path": "linux", "healthy": true, "source": "", "linktarget": "", "flags": null, "isomtime": "2017-02-27T12:27:20", "size": 0} + {"type": "d", "mode": "drwxr-xr-x", "user": "user", "group": "user", "uid": 1000, "gid": 1000, "path": "linux/baz", "healthy": true, "source": "", "linktarget": "", "flags": null, "isomtime": "2017-02-27T12:27:20", "size": 0} .. _msgid: diff --git a/src/borg/archive.py b/src/borg/archive.py index ff8f8729ef..f28c7f128f 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -32,7 +32,7 @@ from .helpers import Error, IntegrityError, set_ec from .helpers import uid2user, user2uid, gid2group, group2gid from .helpers import parse_timestamp, to_localtime -from .helpers import format_time, format_timedelta, format_file_size, file_status, FileSize +from .helpers import OutputTimestamp, format_timedelta, format_file_size, file_status, FileSize from .helpers import safe_encode, safe_decode, make_path_safe, remove_surrogates from .helpers import StableDict from .helpers import bin_to_hex @@ -381,8 +381,8 @@ def info(self): info = { 'name': self.name, 'id': self.fpr, - 'start': format_time(to_localtime(start)), - 'end': format_time(to_localtime(end)), + 'start': OutputTimestamp(start), + 'end': OutputTimestamp(end), 'duration': (end - start).total_seconds(), 'stats': stats.as_dict(), 'limits': { @@ -411,8 +411,8 @@ def __str__(self): Utilization of max. archive size: {csize_max:.0%} '''.format( self, - start=format_time(to_localtime(self.start.replace(tzinfo=timezone.utc))), - end=format_time(to_localtime(self.end.replace(tzinfo=timezone.utc))), + start=OutputTimestamp(self.start.replace(tzinfo=timezone.utc)), + end=OutputTimestamp(self.end.replace(tzinfo=timezone.utc)), csize_max=self.cache.chunks[self.id].csize / MAX_DATA_SIZE) def __repr__(self): diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 86368a9d56..4b68eabe3e 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -744,6 +744,22 @@ def format_timedelta(td): return txt +class OutputTimestamp: + def __init__(self, ts: datetime): + if ts.tzinfo == timezone.utc: + ts = to_localtime(ts) + self.ts = ts + + def __format__(self, format_spec): + return format_time(self.ts) + + def __str__(self): + return '{}'.format(self) + + def to_json(self): + return isoformat_time(self.ts) + + def format_file_size(v, precision=2, sign=False): """Format file size into a human friendly format """ @@ -1664,12 +1680,11 @@ def __init__(self, format, repository, manifest, key, *, json=False): if self.json: self.item_data = {} self.format_item = self.format_item_json - self.format_time = self.format_time_json else: self.item_data = static_keys def format_item_json(self, item): - return json.dumps(self.get_item_data(item)) + '\n' + return json.dumps(self.get_item_data(item), cls=BorgJsonEncoder) + '\n' def get_item_data(self, archive_info): self.name = archive_info.name @@ -1703,12 +1718,7 @@ def get_ts_end(self): return self.format_time(self.archive.ts_end) def format_time(self, ts): - t = to_localtime(ts) - return format_time(t) - - def format_time_json(self, ts): - t = to_localtime(ts) - return isoformat_time(t) + return OutputTimestamp(ts) class ItemFormatter(BaseFormatter): @@ -1784,7 +1794,6 @@ def __init__(self, archive, format, *, json_lines=False): if self.json_lines: self.item_data = {} self.format_item = self.format_item_json - self.format_time = self.format_time_json else: self.item_data = static_keys self.format = partial_format(format, static_keys) @@ -1796,19 +1805,19 @@ def __init__(self, archive, format, *, json_lines=False): 'dcsize': partial(self.sum_unique_chunks_metadata, lambda chunk: chunk.csize), 'num_chunks': self.calculate_num_chunks, 'unique_chunks': partial(self.sum_unique_chunks_metadata, lambda chunk: 1), - 'isomtime': partial(self.format_time, 'mtime'), - 'isoctime': partial(self.format_time, 'ctime'), - 'isoatime': partial(self.format_time, 'atime'), - 'mtime': partial(self.time, 'mtime'), - 'ctime': partial(self.time, 'ctime'), - 'atime': partial(self.time, 'atime'), + 'isomtime': partial(self.format_iso_time, 'mtime'), + 'isoctime': partial(self.format_iso_time, 'ctime'), + 'isoatime': partial(self.format_iso_time, 'atime'), + 'mtime': partial(self.format_time, 'mtime'), + 'ctime': partial(self.format_time, 'ctime'), + 'atime': partial(self.format_time, 'atime'), } for hash_function in hashlib.algorithms_guaranteed: self.add_key(hash_function, partial(self.hash_item, hash_function)) self.used_call_keys = set(self.call_keys) & self.format_keys def format_item_json(self, item): - return json.dumps(self.get_item_data(item)) + '\n' + return json.dumps(self.get_item_data(item), cls=BorgJsonEncoder) + '\n' def add_key(self, key, callable_with_item): self.call_keys[key] = callable_with_item @@ -1883,15 +1892,10 @@ def hash_item(self, hash_function, item): return hash.hexdigest() def format_time(self, key, item): - t = self.time(key, item) - return format_time(t) - - def format_time_json(self, key, item): - t = self.time(key, item) - return isoformat_time(t) + return OutputTimestamp(safe_timestamp(item.get(key) or item.mtime)) - def time(self, key, item): - return safe_timestamp(item.get(key) or item.mtime) + def format_iso_time(self, key, item): + return self.format_time(key, item).to_json() class ChunkIteratorFileWrapper: @@ -2204,6 +2208,8 @@ def default(self, o): return { 'stats': o.stats(), } + if callable(getattr(o, 'to_json', None)): + return o.to_json() return super().default(o) @@ -2216,7 +2222,7 @@ def basic_json_data(manifest, *, cache=None, extra=None): 'mode': key.ARG_NAME, }, }) - data['repository']['last_modified'] = isoformat_time(to_localtime(manifest.last_timestamp.replace(tzinfo=timezone.utc))) + data['repository']['last_modified'] = OutputTimestamp(manifest.last_timestamp.replace(tzinfo=timezone.utc)) if key.NAME.startswith('key file'): data['encryption']['keyfile'] = key.find_key() if cache: diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 6ae7d47100..a32cd6f8d1 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -1325,6 +1325,8 @@ def test_info_json(self): assert isinstance(archive['duration'], float) assert len(archive['id']) == 64 assert 'stats' in archive + assert datetime.strptime(archive['start'], ISO_FORMAT) + assert datetime.strptime(archive['end'], ISO_FORMAT) def test_comment(self): self.create_regular_file('file1', size=1024 * 80) From d5697fb4a8b9d3f21f76a11b224783ec36668f52 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Mon, 7 Aug 2017 13:08:25 +0200 Subject: [PATCH 017/798] always use microseconds for ISO 8601 output (cherry picked from commit ab4981eff65984f23dceb2adcfb29d2d74d70e35) --- docs/internals/frontends.rst | 22 +++++++++++----------- src/borg/helpers.py | 21 +++++++++++++-------- src/borg/testsuite/archiver.py | 4 ++-- 3 files changed, 26 insertions(+), 21 deletions(-) diff --git a/docs/internals/frontends.rst b/docs/internals/frontends.rst index 0441caa53c..b4ebf5eb14 100644 --- a/docs/internals/frontends.rst +++ b/docs/internals/frontends.rst @@ -209,9 +209,9 @@ Standard output *stdout* is different and more command-dependent than logging. Commands like :ref:`borg_info`, :ref:`borg_create` and :ref:`borg_list` implement a ``--json`` option which turns their regular output into a single JSON object. -Dates are formatted according to ISO-8601 in local time. Neither an explicit time zone nor microseconds -are specified *at this time* (subject to change). The equivalent strftime format string is '%Y-%m-%dT%H:%M:%S', -e.g. 2017-08-07T12:27:20. +Dates are formatted according to ISO 8601 in local time. No explicit time zone is specified *at this time* +(subject to change). The equivalent strftime format string is '%Y-%m-%dT%H:%M:%S.%f', +e.g. ``2017-08-07T12:27:20.123456``. The root object at least contains a *repository* key with an object containing: @@ -268,7 +268,7 @@ Example *borg info* output:: }, "repository": { "id": "0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23", - "last_modified": "2017-08-07T12:27:20", + "last_modified": "2017-08-07T12:27:20.789123", "location": "/home/user/testrepo" }, "security_dir": "/home/user/.config/borg/security/0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23", @@ -329,7 +329,7 @@ Example of a simple archive listing (``borg list --last 1 --json``):: { "id": "80cd07219ad725b3c5f665c1dcf119435c4dee1647a560ecac30f8d40221a46a", "name": "host-system-backup-2017-02-27", - "start": "2017-08-07T12:27:20" + "start": "2017-08-07T12:27:20.789123" } ], "encryption": { @@ -337,7 +337,7 @@ Example of a simple archive listing (``borg list --last 1 --json``):: }, "repository": { "id": "0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23", - "last_modified": "2017-08-07T12:27:20", + "last_modified": "2017-08-07T12:27:20.789123", "location": "/home/user/repository" } } @@ -355,14 +355,14 @@ The same archive with more information (``borg info --last 1 --json``):: ], "comment": "", "duration": 5.641542, - "end": "2017-02-27T12:27:20", + "end": "2017-02-27T12:27:20.789123", "hostname": "host", "id": "80cd07219ad725b3c5f665c1dcf119435c4dee1647a560ecac30f8d40221a46a", "limits": { "max_archive_size": 0.0001330855110409714 }, "name": "host-system-backup-2017-02-27", - "start": "2017-02-27T12:27:20", + "start": "2017-02-27T12:27:20.789123", "stats": { "compressed_size": 1880961894, "deduplicated_size": 2791, @@ -388,7 +388,7 @@ The same archive with more information (``borg info --last 1 --json``):: }, "repository": { "id": "0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23", - "last_modified": "2017-08-07T12:27:20", + "last_modified": "2017-08-07T12:27:20.789123", "location": "/home/user/repository" } } @@ -406,8 +406,8 @@ Refer to the *borg list* documentation for the available keys and their meaning. Example (excerpt) of ``borg list --json-lines``:: - {"type": "d", "mode": "drwxr-xr-x", "user": "user", "group": "user", "uid": 1000, "gid": 1000, "path": "linux", "healthy": true, "source": "", "linktarget": "", "flags": null, "isomtime": "2017-02-27T12:27:20", "size": 0} - {"type": "d", "mode": "drwxr-xr-x", "user": "user", "group": "user", "uid": 1000, "gid": 1000, "path": "linux/baz", "healthy": true, "source": "", "linktarget": "", "flags": null, "isomtime": "2017-02-27T12:27:20", "size": 0} + {"type": "d", "mode": "drwxr-xr-x", "user": "user", "group": "user", "uid": 1000, "gid": 1000, "path": "linux", "healthy": true, "source": "", "linktarget": "", "flags": null, "mtime": "2017-02-27T12:27:20.023407", "size": 0} + {"type": "d", "mode": "drwxr-xr-x", "user": "user", "group": "user", "uid": 1000, "gid": 1000, "path": "linux/baz", "healthy": true, "source": "", "linktarget": "", "flags": null, "mtime": "2017-02-27T12:27:20.585407", "size": 0} .. _msgid: diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 4b68eabe3e..64ee4c82a4 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -715,16 +715,19 @@ def safe_timestamp(item_timestamp_ns): return datetime.fromtimestamp(t_ns / 1e9) -def format_time(t): - """use ISO-8601-like date and time format (human readable, with wkday and blank date/time separator) +def format_time(ts: datetime): """ - return t.strftime('%a, %Y-%m-%d %H:%M:%S') + Convert *ts* to a human-friendly format with textual weekday. + """ + return ts.strftime('%a, %Y-%m-%d %H:%M:%S') -def isoformat_time(t): - """use ISO-8601 date and time format (machine readable, no wkday, no microseconds either) +def isoformat_time(ts: datetime): + """ + Format *ts* according to ISO 8601. """ - return t.strftime('%Y-%m-%dT%H:%M:%S') # note: first make all datetime objects tz aware before adding %z here. + # note: first make all datetime objects tz aware before adding %z here. + return ts.strftime('%Y-%m-%dT%H:%M:%S.%f') def format_timedelta(td): @@ -756,9 +759,11 @@ def __format__(self, format_spec): def __str__(self): return '{}'.format(self) - def to_json(self): + def isoformat(self): return isoformat_time(self.ts) + to_json = isoformat + def format_file_size(v, precision=2, sign=False): """Format file size into a human friendly format @@ -1895,7 +1900,7 @@ def format_time(self, key, item): return OutputTimestamp(safe_timestamp(item.get(key) or item.mtime)) def format_iso_time(self, key, item): - return self.format_time(key, item).to_json() + return self.format_time(key, item).isoformat() class ChunkIteratorFileWrapper: diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index a32cd6f8d1..a333eca170 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -60,7 +60,7 @@ src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) -ISO_FORMAT = '%Y-%m-%dT%H:%M:%S' +ISO_FORMAT = '%Y-%m-%dT%H:%M:%S.%f' def exec_cmd(*args, archiver=None, fork=False, exe=None, input=b'', binary_output=False, **kw): @@ -1863,7 +1863,7 @@ def test_list_json(self): file1 = items[1] assert file1['path'] == 'input/file1' assert file1['size'] == 81920 - assert datetime.strptime(file1['isomtime'], ISO_FORMAT) # must not raise + assert datetime.strptime(file1['mtime'], ISO_FORMAT) # must not raise list_archive = self.cmd('list', '--json-lines', '--format={sha256}', self.repository_location + '::test') items = [json.loads(s) for s in list_archive.splitlines()] From 987a99dffe96840eab81a3569eade8a650a88584 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 20 Aug 2017 21:37:17 +0200 Subject: [PATCH 018/798] create: document exclusion through nodump (cherry picked from commit 495f838d88dea02ba3ec08f5985a4827c37a76d1) --- src/borg/archiver.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index a3db174acf..e1a5ee83e5 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -2807,6 +2807,9 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): and not include any other contents of the containing folder, this can be enabled through using the ``--keep-exclude-tags`` option. + Borg respects the nodump flag. Files flagged nodump will be marked as excluded (x) + in ``--list`` output. + Item flags ++++++++++ From d2f9e2868304729bbd63ca4900485ac5d49b5410 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Thu, 24 Aug 2017 16:24:44 +0200 Subject: [PATCH 019/798] travis: only short-circuit docs-only changes for pull requests if a branch build is stopped, then codecov will complain about missing base branch coverage. (cherry picked from commit 920d974b2504dd55ea12e2e018eb308246c4e21e) --- .travis.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index e12266b9dc..f1b01b14bd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,10 +44,12 @@ matrix: before_install: - | - echo Checking whether $TRAVIS_COMMIT_RANGE changed only docs - git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(AUTHORS|README\.rst|^(docs)/)' || { - echo "Only docs were updated, stopping build process." - exit + test $TRAVIS_EVENT_TYPE != "pull_request" || { + echo Checking whether $TRAVIS_COMMIT_RANGE changed only docs + git diff --name-only $TRAVIS_COMMIT_RANGE | grep --quiet --invert-match --extended-regexp '(AUTHORS|README\.rst|^(docs)/)' || { + echo "Only docs were updated, stopping build process." + exit + } } install: From ac404e3a7442323875489c24921aa85c3f21d5ea Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 24 Aug 2017 04:07:37 +0200 Subject: [PATCH 020/798] borg create --timestamp: set start time, fixes #2957 (cherry picked from commit 8a299ae24ced26dbb53517e594634ea676779bc3) --- src/borg/archive.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index ff8f8729ef..88167ab122 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -446,15 +446,13 @@ def save(self, name=None, comment=None, timestamp=None, additional_metadata=None self.items_buffer.flush(flush=True) duration = timedelta(seconds=time.monotonic() - self.start_monotonic) if timestamp is None: - self.end = datetime.utcnow() - self.start = self.end - duration - start = self.start - end = self.end + end = datetime.utcnow() + start = end - duration else: - self.end = timestamp - self.start = timestamp - duration - end = timestamp - start = self.start + end = timestamp + duration + start = timestamp + self.start = start + self.end = end metadata = { 'version': 1, 'name': name, From 24400fcecd3900e417f6a270965186ec4210b710 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 27 Aug 2017 21:48:29 +0200 Subject: [PATCH 021/798] update CHANGES (1.1-maint) --- docs/changes.rst | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/docs/changes.rst b/docs/changes.rst index bed3b26d1c..04d301bfbf 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -132,7 +132,7 @@ Changelog ========= Version 1.1.0rc2 (not released yet) ------------------------------------- +----------------------------------- Compatibility notes: @@ -143,6 +143,42 @@ Compatibility notes: and the default is now "mtime". "isomtime" is now a real ISO-8601 format ("T" between date and time, not a space). +New features: + +- None. + +Fixes: + +- list: fix weird mixup of mtime/isomtime +- create --timestamp: set start time, #2957 +- ignore corrupt files cache, #2939 +- migrate locks to child PID when daemonize is used +- fix exitcode of borg serve, #2910 +- only compare contents when chunker params match, #2899 +- umount: try fusermount, then try umount, #2863 + +Other changes: + +- JSON: use a more standard ISO 8601 datetime format, #2376 +- cache: write_archive_index: truncate_and_unlink on error, #2628 +- detect non-upgraded Attic repositories, #1933 +- delete various nogil and threading related lines +- coala / pylint related improvements +- docs: + + - renew asciinema/screencasts, #669 + - create: document exclusion through nodump, #2949 + - minor formatting fixes + - tar: tarpipe example + - improve "with-lock" and "info" docs, #2869 + - detail how to use macOS/GNOME/KDE keyrings for repo passwords, #392 +- travis: only short-circuit docs-only changes for pull requests +- vagrant: + + - netbsd: bash is already installed + - fix netbsd version in PKG_PATH + - add exe location to PATH when we build an exe + Version 1.1.0rc1 (2017-07-24) ----------------------------- From d07a6e14e7e3a72008b5e998174d041e2694d89c Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 27 Aug 2017 22:08:14 +0200 Subject: [PATCH 022/798] add release date to changelog --- docs/changes.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index 04d301bfbf..bbdb09fc19 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -131,8 +131,8 @@ The best check that everything is ok is to run a dry-run extraction:: Changelog ========= -Version 1.1.0rc2 (not released yet) ------------------------------------ +Version 1.1.0rc2 (2017-08-28) +----------------------------- Compatibility notes: From 9f61747dae65d96f7eeaabd5c2a0b8ad40b40b13 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 27 Aug 2017 22:17:55 +0200 Subject: [PATCH 023/798] python setup.py build_usage --- docs/usage/create.rst.inc | 3 +++ docs/usage/info.rst.inc | 9 +++++++-- docs/usage/list.rst.inc | 4 ++-- docs/usage/with-lock.rst.inc | 5 +++-- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/docs/usage/create.rst.inc b/docs/usage/create.rst.inc index c70e2a8f16..3fc59dff4d 100644 --- a/docs/usage/create.rst.inc +++ b/docs/usage/create.rst.inc @@ -193,6 +193,9 @@ only include the objects specified by ``--exclude-if-present`` in your backup, and not include any other contents of the containing folder, this can be enabled through using the ``--keep-exclude-tags`` option. +Borg respects the nodump flag. Files flagged nodump will be marked as excluded (x) +in ``--list`` output. + Item flags ++++++++++ diff --git a/docs/usage/info.rst.inc b/docs/usage/info.rst.inc index ec5d15098d..5ba601f4ff 100644 --- a/docs/usage/info.rst.inc +++ b/docs/usage/info.rst.inc @@ -77,6 +77,11 @@ up to the deduplicated size of the repository ("all archives"), because the two are meaning different things: This archive / deduplicated size = amount of data stored ONLY for this archive - = unique chunks of this archive. += unique chunks of this archive. All archives / deduplicated size = amount of data stored in the repo - = all chunks in the repository. \ No newline at end of file += all chunks in the repository. + +Borg archives can only contain a limited amount of file metadata. +The size of an archive relative to this limit depends on a number of factors, +mainly the number of files, the lengths of paths and other metadata stored for files. +This is shown as *utilization of maximum supported archive size*. \ No newline at end of file diff --git a/docs/usage/list.rst.inc b/docs/usage/list.rst.inc index 3358a74bf1..e6b8988840 100644 --- a/docs/usage/list.rst.inc +++ b/docs/usage/list.rst.inc @@ -23,7 +23,7 @@ borg list +-----------------------------------------------------------------------------+-----------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``--short`` | only print file/directory names, nothing else | +-----------------------------------------------------------------------------+-----------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ - | | ``--format FORMAT``, ``--list-format FORMAT`` | specify format for file listing (default: "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NL}") | + | | ``--format FORMAT``, ``--list-format FORMAT`` | specify format for file listing (default: "{mode} {user:6} {group:6} {size:8d} {mtime} {path}{extra}{NL}") | +-----------------------------------------------------------------------------+-----------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``--json`` | Only valid for listing repository contents. Format output as JSON. The form of ``--format`` is ignored, but keys used in it are added to the JSON output. Some keys are always present. Note: JSON can only represent text. A "barchive" key is therefore not available. | +-----------------------------------------------------------------------------+-----------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -80,7 +80,7 @@ borg list optional arguments --short only print file/directory names, nothing else - --format FORMAT, --list-format FORMAT specify format for file listing (default: "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NL}") + --format FORMAT, --list-format FORMAT specify format for file listing (default: "{mode} {user:6} {group:6} {size:8d} {mtime} {path}{extra}{NL}") --json Only valid for listing repository contents. Format output as JSON. The form of ``--format`` is ignored, but keys used in it are added to the JSON output. Some keys are always present. Note: JSON can only represent text. A "barchive" key is therefore not available. --json-lines Only valid for listing archive contents. Format output as JSON Lines. The form of ``--format`` is ignored, but keys used in it are added to the JSON output. Some keys are always present. Note: JSON can only represent text. A "bpath" key is therefore not available. diff --git a/docs/usage/with-lock.rst.inc b/docs/usage/with-lock.rst.inc index fc45411641..6f4fc741af 100644 --- a/docs/usage/with-lock.rst.inc +++ b/docs/usage/with-lock.rst.inc @@ -60,5 +60,6 @@ code as borg's return code. .. note:: If you copy a repository with the lock held, the lock will be present in - the copy, obviously. Thus, before using borg on the copy, you need to - use "borg break-lock" on it. \ No newline at end of file + the copy. Thus, before using borg on the copy from a different host, + you need to use "borg break-lock" on the copied repository, because + Borg is cautious and does not automatically remove stale locks made by a different host. \ No newline at end of file From 168293ea807329d1a0cfa7213b2ae1a309be8326 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 27 Aug 2017 22:18:45 +0200 Subject: [PATCH 024/798] python setup.py build_man --- docs/man/borg-benchmark-crud.1 | 4 +- docs/man/borg-benchmark.1 | 2 +- docs/man/borg-break-lock.1 | 4 +- docs/man/borg-change-passphrase.1 | 4 +- docs/man/borg-check.1 | 12 +- docs/man/borg-common.1 | 8 +- docs/man/borg-compression.1 | 2 +- docs/man/borg-create.1 | 41 +++--- docs/man/borg-delete.1 | 16 +-- docs/man/borg-diff.1 | 18 +-- docs/man/borg-export-tar.1 | 12 +- docs/man/borg-extract.1 | 43 +++--- docs/man/borg-info.1 | 82 ++++++++---- docs/man/borg-init.1 | 8 +- docs/man/borg-key-change-passphrase.1 | 4 +- docs/man/borg-key-export.1 | 4 +- docs/man/borg-key-import.1 | 6 +- docs/man/borg-key-migrate-to-repokey.1 | 4 +- docs/man/borg-key.1 | 2 +- docs/man/borg-list.1 | 45 ++++--- docs/man/borg-mount.1 | 12 +- docs/man/borg-patterns.1 | 4 +- docs/man/borg-placeholders.1 | 2 +- docs/man/borg-prune.1 | 29 ++-- docs/man/borg-recreate.1 | 33 ++--- docs/man/borg-rename.1 | 4 +- docs/man/borg-serve.1 | 25 +++- docs/man/borg-umount.1 | 40 +++--- docs/man/borg-upgrade.1 | 15 +-- docs/man/borg-with-lock.1 | 16 ++- docs/man/borg.1 | 176 +++++++++++++++++++++++++ 31 files changed, 478 insertions(+), 199 deletions(-) diff --git a/docs/man/borg-benchmark-crud.1 b/docs/man/borg-benchmark-crud.1 index c763aae31f..31b059f61d 100644 --- a/docs/man/borg-benchmark-crud.1 +++ b/docs/man/borg-benchmark-crud.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-BENCHMARK-CRUD 1 "2017-06-18" "" "borg backup tool" +.TH BORG-BENCHMARK-CRUD 1 "2017-08-27" "" "borg backup tool" .SH NAME borg-benchmark-crud \- Benchmark Create, Read, Update, Delete for archives. . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -borg [common options] benchmark crud REPO PATH +borg [common options] benchmark crud [options] REPO PATH .SH DESCRIPTION .sp This command benchmarks borg CRUD (create, read, update, delete) operations. diff --git a/docs/man/borg-benchmark.1 b/docs/man/borg-benchmark.1 index 79e356ac14..a56a0bc8c5 100644 --- a/docs/man/borg-benchmark.1 +++ b/docs/man/borg-benchmark.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-BENCHMARK 1 "2017-06-18" "" "borg backup tool" +.TH BORG-BENCHMARK 1 "2017-08-27" "" "borg backup tool" .SH NAME borg-benchmark \- benchmark command . diff --git a/docs/man/borg-break-lock.1 b/docs/man/borg-break-lock.1 index 7b4291cd52..7a3b9c2534 100644 --- a/docs/man/borg-break-lock.1 +++ b/docs/man/borg-break-lock.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-BREAK-LOCK 1 "2017-06-18" "" "borg backup tool" +.TH BORG-BREAK-LOCK 1 "2017-08-27" "" "borg backup tool" .SH NAME borg-break-lock \- Break the repository lock (e.g. in case it was left by a dead borg. . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -borg [common options] break\-lock REPOSITORY +borg [common options] break\-lock [options] [REPOSITORY] .SH DESCRIPTION .sp This command breaks the repository and cache locks. diff --git a/docs/man/borg-change-passphrase.1 b/docs/man/borg-change-passphrase.1 index d5b3edbfa6..fc85a7da5e 100644 --- a/docs/man/borg-change-passphrase.1 +++ b/docs/man/borg-change-passphrase.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-CHANGE-PASSPHRASE 1 "2017-06-18" "" "borg backup tool" +.TH BORG-CHANGE-PASSPHRASE 1 "2017-08-27" "" "borg backup tool" .SH NAME borg-change-passphrase \- Change repository key file passphrase . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -borg [common options] change\-passphrase REPOSITORY +borg [common options] change\-passphrase [options] [REPOSITORY] .SH DESCRIPTION .sp The key files used for repository encryption are optionally passphrase diff --git a/docs/man/borg-check.1 b/docs/man/borg-check.1 index cf2996a2ac..a05dcf23d3 100644 --- a/docs/man/borg-check.1 +++ b/docs/man/borg-check.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-CHECK 1 "2017-06-18" "" "borg backup tool" +.TH BORG-CHECK 1 "2017-08-27" "" "borg backup tool" .SH NAME borg-check \- Check repository consistency . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -borg [common options] check REPOSITORY_OR_ARCHIVE +borg [common options] check [options] [REPOSITORY_OR_ARCHIVE] .SH DESCRIPTION .sp The check command verifies the consistency of a repository and the corresponding archives. @@ -121,16 +121,16 @@ attempt to repair any inconsistencies found .B \-\-save\-space work slower, but using less space .UNINDENT -.SS filters +.SS Archive filters .INDENT 0.0 .TP -.B \-P\fP,\fB \-\-prefix +.BI \-P \ PREFIX\fP,\fB \ \-\-prefix \ PREFIX only consider archive names starting with this prefix. .TP -.B \-a\fP,\fB \-\-glob\-archives +.BI \-a \ GLOB\fP,\fB \ \-\-glob\-archives \ GLOB only consider archive names matching the glob. sh: rules apply, see "borg help patterns". \fB\-\-prefix\fP and \fB\-\-glob\-archives\fP are mutually exclusive. .TP -.B \-\-sort\-by +.BI \-\-sort\-by \ KEYS Comma\-separated list of sorting keys; valid keys are: timestamp, name, id; default is: timestamp .TP .BI \-\-first \ N diff --git a/docs/man/borg-common.1 b/docs/man/borg-common.1 index f48ccb6c15..cd6cfc7572 100644 --- a/docs/man/borg-common.1 +++ b/docs/man/borg-common.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-COMMON 1 "2017-06-18" "" "borg backup tool" +.TH BORG-COMMON 1 "2017-08-27" "" "borg backup tool" .SH NAME borg-common \- Common options of Borg commands . @@ -60,8 +60,8 @@ show progress information .B \-\-log\-json Output one JSON object per log line instead of formatted text. .TP -.BI \-\-lock\-wait \ N -wait for the lock, but max. N seconds (default: 1). +.BI \-\-lock\-wait \ SECONDS +wait at most SECONDS for acquiring a repository/cache lock (default: 1). .TP .B \-\-show\-version show/log the borg version @@ -78,7 +78,7 @@ set umask to M (local and remote, default: 0077) .BI \-\-remote\-path \ PATH use PATH as borg executable on the remote (default: "borg") .TP -.BI \-\-remote\-ratelimit \ rate +.BI \-\-remote\-ratelimit \ RATE set remote network upload rate limit in kiByte/s (default: 0=unlimited) .TP .B \-\-consider\-part\-files diff --git a/docs/man/borg-compression.1 b/docs/man/borg-compression.1 index 3347a65820..3c6c53468f 100644 --- a/docs/man/borg-compression.1 +++ b/docs/man/borg-compression.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-COMPRESSION 1 "2017-06-18" "" "borg backup tool" +.TH BORG-COMPRESSION 1 "2017-08-27" "" "borg backup tool" .SH NAME borg-compression \- Details regarding compression . diff --git a/docs/man/borg-create.1 b/docs/man/borg-create.1 index ec8d7b525b..f41536784b 100644 --- a/docs/man/borg-create.1 +++ b/docs/man/borg-create.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-CREATE 1 "2017-06-18" "" "borg backup tool" +.TH BORG-CREATE 1 "2017-08-27" "" "borg backup tool" .SH NAME borg-create \- Create new archive . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -borg [common options] create ARCHIVE PATH +borg [common options] create [options] ARCHIVE [PATH...] .SH DESCRIPTION .sp This command creates a backup archive containing all files found while recursively @@ -53,10 +53,14 @@ checkpoints and treated in special ways. In the archive name, you may use the following placeholders: {now}, {utcnow}, {fqdn}, {hostname}, {user} and some others. .sp -To speed up pulling backups over sshfs and similar network file systems which do -not provide correct inode information the \fB\-\-ignore\-inode\fP flag can be used. This -potentially decreases reliability of change detection, while avoiding always reading -all files on these file systems. +Backup speed is increased by not reprocessing files that are already part of +existing archives and weren\(aqt modified. Normally, detecting file modifications +will take inode information into consideration. This is problematic for files +located on sshfs and similar network file systems which do not provide stable +inode numbers, such files will always be considered modified. The +\fB\-\-ignore\-inode\fP flag can be used to prevent this and improve performance. +This flag will reduce reliability of change detection however, with files +considered unmodified as long as their size and modification time are unchanged. .sp The mount points of filesystems or filesystem snapshots should be the same for every creation of a new archive to ensure fast operation. This is because the file cache that @@ -94,13 +98,13 @@ print statistics for the created archive output verbose list of items (files, dirs, ...) .TP .BI \-\-filter \ STATUSCHARS -only display items with the given status characters +only display items with the given status characters (see description) .TP .B \-\-json -output stats as JSON (implies \-\-stats) +output stats as JSON. Implies \fB\-\-stats\fP\&. .TP .B \-\-no\-cache\-sync -experimental: do not synchronize the cache. Implies \-\-no\-files\-cache. +experimental: do not synchronize the cache. Implies \fB\-\-no\-files\-cache\fP\&. .UNINDENT .SS Exclusion options .INDENT 0.0 @@ -111,6 +115,12 @@ exclude paths matching PATTERN .BI \-\-exclude\-from \ EXCLUDEFILE read exclude patterns from EXCLUDEFILE, one per line .TP +.BI \-\-pattern \ PATTERN +experimental: include/exclude paths matching PATTERN +.TP +.BI \-\-patterns\-from \ PATTERNFILE +experimental: read include/exclude patterns from PATTERNFILE, one per line +.TP .B \-\-exclude\-caches exclude directories that contain a CACHEDIR.TAG file (\fI\%http://www.brynosaurus.com/cachedir/spec.html\fP) .TP @@ -118,13 +128,7 @@ exclude directories that contain a CACHEDIR.TAG file (\fI\%http://www.brynosauru exclude directories that are tagged by containing a filesystem object with the given NAME .TP .B \-\-keep\-exclude\-tags\fP,\fB \-\-keep\-tag\-files -if tag objects are specified with \-\-exclude\-if\-present, don\(aqt omit the tag objects themselves from the backup archive -.TP -.BI \-\-pattern \ PATTERN -experimental: include/exclude paths matching PATTERN -.TP -.BI \-\-patterns\-from \ PATTERNFILE -experimental: read include/exclude patterns from PATTERNFILE, one per line +if tag objects are specified with \fB\-\-exclude\-if\-present\fP, don\(aqt omit the tag objects themselves from the backup archive .UNINDENT .SS Filesystem options .INDENT 0.0 @@ -154,7 +158,7 @@ open and read block and char device files as well as FIFOs as if they were regul add a comment text to the archive .TP .BI \-\-timestamp \ TIMESTAMP -manually specify the archive creation date/time (UTC, yyyy\-mm\-ddThh:mm:ss format). alternatively, give a reference file/directory. +manually specify the archive creation date/time (UTC, yyyy\-mm\-ddThh:mm:ss format). Alternatively, give a reference file/directory. .TP .BI \-c \ SECONDS\fP,\fB \ \-\-checkpoint\-interval \ SECONDS write checkpoint every SECONDS seconds (Default: 1800) @@ -250,6 +254,9 @@ all of its contents will be omitted from the backup. If, however, you wish to only include the objects specified by \fB\-\-exclude\-if\-present\fP in your backup, and not include any other contents of the containing folder, this can be enabled through using the \fB\-\-keep\-exclude\-tags\fP option. +.sp +Borg respects the nodump flag. Files flagged nodump will be marked as excluded (x) +in \fB\-\-list\fP output. .SS Item flags .sp \fB\-\-list\fP outputs a list of all files, directories and other diff --git a/docs/man/borg-delete.1 b/docs/man/borg-delete.1 index 2e8891532b..318d8b16d0 100644 --- a/docs/man/borg-delete.1 +++ b/docs/man/borg-delete.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-DELETE 1 "2017-06-18" "" "borg backup tool" +.TH BORG-DELETE 1 "2017-08-27" "" "borg backup tool" .SH NAME borg-delete \- Delete an existing repository or archives . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -borg [common options] delete TARGET +borg [common options] delete [options] [TARGET] .SH DESCRIPTION .sp This command deletes an archive from the repository or the complete repository. @@ -53,25 +53,25 @@ archive or repository to delete .B \-s\fP,\fB \-\-stats print statistics for the deleted archive .TP -.B \-c\fP,\fB \-\-cache\-only +.B \-\-cache\-only delete only the local cache for the given repository .TP .B \-\-force -force deletion of corrupted archives, use \-\-force \-\-force in case \-\-force does not work. +force deletion of corrupted archives, use \fB\-\-force \-\-force\fP in case \fB\-\-force\fP does not work. .TP .B \-\-save\-space work slower, but using less space .UNINDENT -.SS filters +.SS Archive filters .INDENT 0.0 .TP -.B \-P\fP,\fB \-\-prefix +.BI \-P \ PREFIX\fP,\fB \ \-\-prefix \ PREFIX only consider archive names starting with this prefix. .TP -.B \-a\fP,\fB \-\-glob\-archives +.BI \-a \ GLOB\fP,\fB \ \-\-glob\-archives \ GLOB only consider archive names matching the glob. sh: rules apply, see "borg help patterns". \fB\-\-prefix\fP and \fB\-\-glob\-archives\fP are mutually exclusive. .TP -.B \-\-sort\-by +.BI \-\-sort\-by \ KEYS Comma\-separated list of sorting keys; valid keys are: timestamp, name, id; default is: timestamp .TP .BI \-\-first \ N diff --git a/docs/man/borg-diff.1 b/docs/man/borg-diff.1 index ad030a6728..555e6cc7a7 100644 --- a/docs/man/borg-diff.1 +++ b/docs/man/borg-diff.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-DIFF 1 "2017-06-18" "" "borg backup tool" +.TH BORG-DIFF 1 "2017-08-27" "" "borg backup tool" .SH NAME borg-diff \- Diff contents of two archives . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -borg [common options] diff REPO_ARCHIVE1 ARCHIVE2 PATH +borg [common options] diff [options] REPO_ARCHIVE1 ARCHIVE2 [PATH...] .SH DESCRIPTION .sp This command finds differences (file contents, user/group/mode) between archives. @@ -87,6 +87,12 @@ exclude paths matching PATTERN .BI \-\-exclude\-from \ EXCLUDEFILE read exclude patterns from EXCLUDEFILE, one per line .TP +.BI \-\-pattern \ PATTERN +experimental: include/exclude paths matching PATTERN +.TP +.BI \-\-patterns\-from \ PATTERNFILE +experimental: read include/exclude patterns from PATTERNFILE, one per line +.TP .B \-\-exclude\-caches exclude directories that contain a CACHEDIR.TAG file (\fI\%http://www.brynosaurus.com/cachedir/spec.html\fP) .TP @@ -94,13 +100,7 @@ exclude directories that contain a CACHEDIR.TAG file (\fI\%http://www.brynosauru exclude directories that are tagged by containing a filesystem object with the given NAME .TP .B \-\-keep\-exclude\-tags\fP,\fB \-\-keep\-tag\-files -if tag objects are specified with \-\-exclude\-if\-present, don\(aqt omit the tag objects themselves from the backup archive -.TP -.BI \-\-pattern \ PATTERN -experimental: include/exclude paths matching PATTERN -.TP -.BI \-\-patterns\-from \ PATTERNFILE -experimental: read include/exclude patterns from PATTERNFILE, one per line +if tag objects are specified with \fB\-\-exclude\-if\-present\fP, don\(aqt omit the tag objects themselves from the backup archive .UNINDENT .SH EXAMPLES .INDENT 0.0 diff --git a/docs/man/borg-export-tar.1 b/docs/man/borg-export-tar.1 index 515b4d849c..ba509ac4dd 100644 --- a/docs/man/borg-export-tar.1 +++ b/docs/man/borg-export-tar.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-EXPORT-TAR 1 "2017-06-18" "" "borg backup tool" +.TH BORG-EXPORT-TAR 1 "2017-08-27" "" "borg backup tool" .SH NAME borg-export-tar \- Export archive contents as a tarball . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -borg [common options] export\-tar ARCHIVE FILE PATH +borg [common options] export\-tar [options] ARCHIVE FILE [PATH...] .SH DESCRIPTION .sp This command creates a tarball from an archive. @@ -95,6 +95,9 @@ filter program to pipe data through .TP .B \-\-list output verbose list of items (files, dirs, ...) +.UNINDENT +.SS Exclusion options +.INDENT 0.0 .TP .BI \-e \ PATTERN\fP,\fB \ \-\-exclude \ PATTERN exclude paths matching PATTERN @@ -109,7 +112,7 @@ experimental: include/exclude paths matching PATTERN experimental: read include/exclude patterns from PATTERNFILE, one per line .TP .BI \-\-strip\-components \ NUMBER -Remove the specified number of leading path elements. Pathnames with fewer elements will be silently skipped. +Remove the specified number of leading path elements. Paths with fewer elements will be silently skipped. .UNINDENT .SH EXAMPLES .INDENT 0.0 @@ -129,6 +132,9 @@ $ borg export\-tar testrepo::linux \-\-tar\-filter="gzip \-9" Monday.tar.gz # export a gzipped tar, but instead of storing it on disk, # upload it to a remote site using curl. $ borg export\-tar ... \-\-tar\-filter="gzip" \- | curl \-\-data\-binary @\- https://somewhere/to/POST + +# remote extraction via "tarpipe" +$ borg export\-tar /path/to/repo::Monday \- | ssh somewhere "cd extracted; tar x" .ft P .fi .UNINDENT diff --git a/docs/man/borg-extract.1 b/docs/man/borg-extract.1 index 13a71ab721..ee261b497e 100644 --- a/docs/man/borg-extract.1 +++ b/docs/man/borg-extract.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-EXTRACT 1 "2017-06-18" "" "borg backup tool" +.TH BORG-EXTRACT 1 "2017-08-27" "" "borg backup tool" .SH NAME borg-extract \- Extract archive contents . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -borg [common options] extract ARCHIVE PATH +borg [common options] extract [options] ARCHIVE [PATH...] .SH DESCRIPTION .sp This command extracts the contents of an archive. By default the entire @@ -48,6 +48,14 @@ decrypting, decompressing. .sp \fB\-\-progress\fP can be slower than no progress display, since it makes one additional pass over the archive metadata. +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +Currently, extract always writes into the current working directory ("."), +so make sure you \fBcd\fP to the right place before calling \fBborg extract\fP\&. +.UNINDENT +.UNINDENT .SH OPTIONS .sp See \fIborg\-common(1)\fP for common options of Borg commands. @@ -69,6 +77,18 @@ output verbose list of items (files, dirs, ...) .B \-n\fP,\fB \-\-dry\-run do not actually change any files .TP +.B \-\-numeric\-owner +only obey numeric user and group identifiers +.TP +.B \-\-stdout +write all extracted data to stdout +.TP +.B \-\-sparse +create holes in output sparse file from all\-zero chunks +.UNINDENT +.SS Exclusion options +.INDENT 0.0 +.TP .BI \-e \ PATTERN\fP,\fB \ \-\-exclude \ PATTERN exclude paths matching PATTERN .TP @@ -81,17 +101,8 @@ experimental: include/exclude paths matching PATTERN .BI \-\-patterns\-from \ PATTERNFILE experimental: read include/exclude patterns from PATTERNFILE, one per line .TP -.B \-\-numeric\-owner -only obey numeric user and group identifiers -.TP .BI \-\-strip\-components \ NUMBER -Remove the specified number of leading path elements. Pathnames with fewer elements will be silently skipped. -.TP -.B \-\-stdout -write all extracted data to stdout -.TP -.B \-\-sparse -create holes in output sparse file from all\-zero chunks +Remove the specified number of leading path elements. Paths with fewer elements will be silently skipped. .UNINDENT .SH EXAMPLES .INDENT 0.0 @@ -120,14 +131,6 @@ $ borg extract \-\-stdout /path/to/repo::my\-sdx | dd of=/dev/sdx bs=10M .fi .UNINDENT .UNINDENT -.sp -\fBNOTE:\fP -.INDENT 0.0 -.INDENT 3.5 -Currently, extract always writes into the current working directory ("."), -so make sure you \fBcd\fP to the right place before calling \fBborg extract\fP\&. -.UNINDENT -.UNINDENT .SH SEE ALSO .sp \fIborg\-common(1)\fP, \fIborg\-mount(1)\fP diff --git a/docs/man/borg-info.1 b/docs/man/borg-info.1 index 338d3ca603..7016d2cf44 100644 --- a/docs/man/borg-info.1 +++ b/docs/man/borg-info.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-INFO 1 "2017-06-18" "" "borg backup tool" +.TH BORG-INFO 1 "2017-08-27" "" "borg backup tool" .SH NAME borg-info \- Show archive details such as disk space used . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -borg [common options] info REPOSITORY_OR_ARCHIVE +borg [common options] info [options] [REPOSITORY_OR_ARCHIVE] .SH DESCRIPTION .sp This command displays detailed information about the specified archive or repository. @@ -40,14 +40,16 @@ This command displays detailed information about the specified archive or reposi Please note that the deduplicated sizes of the individual archives do not add up to the deduplicated size of the repository ("all archives"), because the two are meaning different things: -.INDENT 0.0 -.TP -.B This archive / deduplicated size = amount of data stored ONLY for this archive +.sp +This archive / deduplicated size = amount of data stored ONLY for this archive = unique chunks of this archive. -.TP -.B All archives / deduplicated size = amount of data stored in the repo +All archives / deduplicated size = amount of data stored in the repo = all chunks in the repository. -.UNINDENT +.sp +Borg archives can only contain a limited amount of file metadata. +The size of an archive relative to this limit depends on a number of factors, +mainly the number of files, the lengths of paths and other metadata stored for files. +This is shown as \fIutilization of maximum supported archive size\fP\&. .SH OPTIONS .sp See \fIborg\-common(1)\fP for common options of Borg commands. @@ -63,16 +65,16 @@ archive or repository to display information about .B \-\-json format output as JSON .UNINDENT -.SS filters +.SS Archive filters .INDENT 0.0 .TP -.B \-P\fP,\fB \-\-prefix +.BI \-P \ PREFIX\fP,\fB \ \-\-prefix \ PREFIX only consider archive names starting with this prefix. .TP -.B \-a\fP,\fB \-\-glob\-archives +.BI \-a \ GLOB\fP,\fB \ \-\-glob\-archives \ GLOB only consider archive names matching the glob. sh: rules apply, see "borg help patterns". \fB\-\-prefix\fP and \fB\-\-glob\-archives\fP are mutually exclusive. .TP -.B \-\-sort\-by +.BI \-\-sort\-by \ KEYS Comma\-separated list of sorting keys; valid keys are: timestamp, name, id; default is: timestamp .TP .BI \-\-first \ N @@ -87,22 +89,58 @@ consider last N archives after other filters were applied .sp .nf .ft C -$ borg info /path/to/repo::root\-2016\-02\-15 -Name: root\-2016\-02\-15 -Fingerprint: 57c827621f21b000a8d363c1e163cc55983822b3afff3a96df595077a660be50 +$ borg info /path/to/repo::2017\-06\-29T11:00\-srv +Archive name: 2017\-06\-29T11:00\-srv +Archive fingerprint: b2f1beac2bd553b34e06358afa45a3c1689320d39163890c5bbbd49125f00fe5 +Comment: Hostname: myhostname Username: root -Time (start): Mon, 2016\-02\-15 19:36:29 -Time (end): Mon, 2016\-02\-15 19:39:26 -Command line: /usr/local/bin/borg create \-\-list \-C zlib,6 /path/to/repo::root\-2016\-02\-15 / \-\-one\-file\-system -Number of files: 38100 +Time (start): Thu, 2017\-06\-29 11:03:07 +Time (end): Thu, 2017\-06\-29 11:03:13 +Duration: 5.66 seconds +Number of files: 17037 +Command line: /usr/sbin/borg create /path/to/repo::2017\-06\-29T11:00\-srv /srv +Utilization of max. archive size: 0% +\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\- + Original size Compressed size Deduplicated size +This archive: 12.53 GB 12.49 GB 1.62 kB +All archives: 121.82 TB 112.41 TB 215.42 GB + + Unique chunks Total chunks +Chunk index: 1015213 626934122 + +$ borg info /path/to/repo \-\-last 1 +Archive name: 2017\-06\-29T11:00\-srv +Archive fingerprint: b2f1beac2bd553b34e06358afa45a3c1689320d39163890c5bbbd49125f00fe5 +Comment: +Hostname: myhostname +Username: root +Time (start): Thu, 2017\-06\-29 11:03:07 +Time (end): Thu, 2017\-06\-29 11:03:13 +Duration: 5.66 seconds +Number of files: 17037 +Command line: /usr/sbin/borg create /path/to/repo::2017\-06\-29T11:00\-srv /srv +Utilization of max. archive size: 0% +\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\- + Original size Compressed size Deduplicated size +This archive: 12.53 GB 12.49 GB 1.62 kB +All archives: 121.82 TB 112.41 TB 215.42 GB + + Unique chunks Total chunks +Chunk index: 1015213 626934122 +$ borg info /path/to/repo +Repository ID: d857ce5788c51272c61535062e89eac4e8ef5a884ffbe976e0af9d8765dedfa5 +Location: /path/to/repo +Encrypted: Yes (repokey) +Cache: /root/.cache/borg/d857ce5788c51272c61535062e89eac4e8ef5a884ffbe976e0af9d8765dedfa5 +Security dir: /root/.config/borg/security/d857ce5788c51272c61535062e89eac4e8ef5a884ffbe976e0af9d8765dedfa5 +\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\- Original size Compressed size Deduplicated size -This archive: 1.33 GB 613.25 MB 571.64 MB -All archives: 1.63 GB 853.66 MB 584.12 MB +All archives: 121.82 TB 112.41 TB 215.42 GB Unique chunks Total chunks -Chunk index: 36858 48844 +Chunk index: 1015213 626934122 .ft P .fi .UNINDENT diff --git a/docs/man/borg-init.1 b/docs/man/borg-init.1 index 9576afa8d7..dc7a930235 100644 --- a/docs/man/borg-init.1 +++ b/docs/man/borg-init.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-INIT 1 "2017-06-18" "" "borg backup tool" +.TH BORG-INIT 1 "2017-08-27" "" "borg backup tool" .SH NAME borg-init \- Initialize an empty repository . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -borg [common options] init REPOSITORY +borg [common options] init [options] [REPOSITORY] .SH DESCRIPTION .sp This command initializes an empty repository. A repository is a filesystem @@ -178,13 +178,13 @@ repository to create .SS optional arguments .INDENT 0.0 .TP -.B \-e\fP,\fB \-\-encryption +.BI \-e \ MODE\fP,\fB \ \-\-encryption \ MODE select encryption key mode \fB(required)\fP .TP .B \-\-append\-only create an append\-only mode repository .TP -.B \-\-storage\-quota +.BI \-\-storage\-quota \ QUOTA Set storage quota of the new repository (e.g. 5G, 1.5T). Default: no quota. .UNINDENT .SH EXAMPLES diff --git a/docs/man/borg-key-change-passphrase.1 b/docs/man/borg-key-change-passphrase.1 index 88d1b2c1a0..f6da306e3b 100644 --- a/docs/man/borg-key-change-passphrase.1 +++ b/docs/man/borg-key-change-passphrase.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-CHANGE-PASSPHRASE 1 "2017-06-18" "" "borg backup tool" +.TH BORG-KEY-CHANGE-PASSPHRASE 1 "2017-08-27" "" "borg backup tool" .SH NAME borg-key-change-passphrase \- Change repository key file passphrase . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -borg [common options] key change\-passphrase REPOSITORY +borg [common options] key change\-passphrase [options] [REPOSITORY] .SH DESCRIPTION .sp The key files used for repository encryption are optionally passphrase diff --git a/docs/man/borg-key-export.1 b/docs/man/borg-key-export.1 index 236886831f..f4e52608c1 100644 --- a/docs/man/borg-key-export.1 +++ b/docs/man/borg-key-export.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-EXPORT 1 "2017-06-18" "" "borg backup tool" +.TH BORG-KEY-EXPORT 1 "2017-08-27" "" "borg backup tool" .SH NAME borg-key-export \- Export the repository key for backup . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -borg [common options] key export REPOSITORY PATH +borg [common options] key export [options] [REPOSITORY] [PATH] .SH DESCRIPTION .sp If repository encryption is used, the repository is inaccessible diff --git a/docs/man/borg-key-import.1 b/docs/man/borg-key-import.1 index 92a1754d89..0bd5aa482c 100644 --- a/docs/man/borg-key-import.1 +++ b/docs/man/borg-key-import.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-IMPORT 1 "2017-06-18" "" "borg backup tool" +.TH BORG-KEY-IMPORT 1 "2017-08-27" "" "borg backup tool" .SH NAME borg-key-import \- Import the repository key from backup . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -borg [common options] key import REPOSITORY PATH +borg [common options] key import [options] [REPOSITORY] [PATH] .SH DESCRIPTION .sp This command allows to restore a key previously backed up with the @@ -50,7 +50,7 @@ REPOSITORY .INDENT 0.0 .TP .B PATH -path to the backup +path to the backup (\(aq\-\(aq to read from stdin) .UNINDENT .SS optional arguments .INDENT 0.0 diff --git a/docs/man/borg-key-migrate-to-repokey.1 b/docs/man/borg-key-migrate-to-repokey.1 index 0d408612e1..b6f5efac24 100644 --- a/docs/man/borg-key-migrate-to-repokey.1 +++ b/docs/man/borg-key-migrate-to-repokey.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-MIGRATE-TO-REPOKEY 1 "2017-06-18" "" "borg backup tool" +.TH BORG-KEY-MIGRATE-TO-REPOKEY 1 "2017-08-27" "" "borg backup tool" .SH NAME borg-key-migrate-to-repokey \- Migrate passphrase -> repokey . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -borg [common options] key migrate\-to\-repokey REPOSITORY +borg [common options] key migrate\-to\-repokey [options] [REPOSITORY] .SH DESCRIPTION .sp This command migrates a repository from passphrase mode (removed in Borg 1.0) diff --git a/docs/man/borg-key.1 b/docs/man/borg-key.1 index 0915aa5fda..f24211c6e2 100644 --- a/docs/man/borg-key.1 +++ b/docs/man/borg-key.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY 1 "2017-06-18" "" "borg backup tool" +.TH BORG-KEY 1 "2017-08-27" "" "borg backup tool" .SH NAME borg-key \- Manage a keyfile or repokey of a repository . diff --git a/docs/man/borg-list.1 b/docs/man/borg-list.1 index 3bceff410b..d26bcaba86 100644 --- a/docs/man/borg-list.1 +++ b/docs/man/borg-list.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-LIST 1 "2017-06-18" "" "borg backup tool" +.TH BORG-LIST 1 "2017-08-27" "" "borg backup tool" .SH NAME borg-list \- List archive or repository contents . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -borg [common options] list REPOSITORY_OR_ARCHIVE PATH +borg [common options] list [options] [REPOSITORY_OR_ARCHIVE] [PATH...] .SH DESCRIPTION .sp This command lists the contents of a repository or an archive. @@ -56,9 +56,8 @@ paths to list; patterns are supported .B \-\-short only print file/directory names, nothing else .TP -.B \-\-format\fP,\fB \-\-list\-format -specify format for file listing -(default: "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NL}") +.BI \-\-format \ FORMAT\fP,\fB \ \-\-list\-format \ FORMAT +specify format for file listing (default: "{mode} {user:6} {group:6} {size:8d} {mtime} {path}{extra}{NL}") .TP .B \-\-json Only valid for listing repository contents. Format output as JSON. The form of \fB\-\-format\fP is ignored, but keys used in it are added to the JSON output. Some keys are always present. Note: JSON can only represent text. A "barchive" key is therefore not available. @@ -66,16 +65,16 @@ Only valid for listing repository contents. Format output as JSON. The form of \ .B \-\-json\-lines Only valid for listing archive contents. Format output as JSON Lines. The form of \fB\-\-format\fP is ignored, but keys used in it are added to the JSON output. Some keys are always present. Note: JSON can only represent text. A "bpath" key is therefore not available. .UNINDENT -.SS filters +.SS Archive filters .INDENT 0.0 .TP -.B \-P\fP,\fB \-\-prefix +.BI \-P \ PREFIX\fP,\fB \ \-\-prefix \ PREFIX only consider archive names starting with this prefix. .TP -.B \-a\fP,\fB \-\-glob\-archives +.BI \-a \ GLOB\fP,\fB \ \-\-glob\-archives \ GLOB only consider archive names matching the glob. sh: rules apply, see "borg help patterns". \fB\-\-prefix\fP and \fB\-\-glob\-archives\fP are mutually exclusive. .TP -.B \-\-sort\-by +.BI \-\-sort\-by \ KEYS Comma\-separated list of sorting keys; valid keys are: timestamp, name, id; default is: timestamp .TP .BI \-\-first \ N @@ -93,6 +92,12 @@ exclude paths matching PATTERN .BI \-\-exclude\-from \ EXCLUDEFILE read exclude patterns from EXCLUDEFILE, one per line .TP +.BI \-\-pattern \ PATTERN +experimental: include/exclude paths matching PATTERN +.TP +.BI \-\-patterns\-from \ PATTERNFILE +experimental: read include/exclude patterns from PATTERNFILE, one per line +.TP .B \-\-exclude\-caches exclude directories that contain a CACHEDIR.TAG file (\fI\%http://www.brynosaurus.com/cachedir/spec.html\fP) .TP @@ -100,13 +105,7 @@ exclude directories that contain a CACHEDIR.TAG file (\fI\%http://www.brynosauru exclude directories that are tagged by containing a filesystem object with the given NAME .TP .B \-\-keep\-exclude\-tags\fP,\fB \-\-keep\-tag\-files -if tag objects are specified with \-\-exclude\-if\-present, don\(aqt omit the tag objects themselves from the backup archive -.TP -.BI \-\-pattern \ PATTERN -experimental: include/exclude paths matching PATTERN -.TP -.BI \-\-patterns\-from \ PATTERNFILE -experimental: read include/exclude patterns from PATTERNFILE, one per line +if tag objects are specified with \fB\-\-exclude\-if\-present\fP, don\(aqt omit the tag objects themselves from the backup archive .UNINDENT .SH EXAMPLES .INDENT 0.0 @@ -162,13 +161,23 @@ LF Keys for listing repository archives: .INDENT 0.0 .IP \(bu 2 -archive, name: archive name interpreted as text (might be missing non\-text characters, see barchive) +name: archive name interpreted as text (might be missing non\-text characters, see barchive) +.IP \(bu 2 +archive: archive name interpreted as text (might be missing non\-text characters, see barchive) .IP \(bu 2 barchive: verbatim archive name, can contain any character except NUL .IP \(bu 2 -time: time of creation of the archive +comment: archive comment interpreted as text (might be missing non\-text characters, see bcomment) +.IP \(bu 2 +bcomment: verbatim archive comment, can contain any character except NUL .IP \(bu 2 id: internal ID of the archive +.IP \(bu 2 +time: time (start) of creation of the archive +.IP \(bu 2 +start: time (start) of creation of the archive +.IP \(bu 2 +end: time (end) of creation of the archive .UNINDENT .sp Keys for listing archive files: diff --git a/docs/man/borg-mount.1 b/docs/man/borg-mount.1 index d1892ac129..0e5d36ee90 100644 --- a/docs/man/borg-mount.1 +++ b/docs/man/borg-mount.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-MOUNT 1 "2017-06-18" "" "borg backup tool" +.TH BORG-MOUNT 1 "2017-08-27" "" "borg backup tool" .SH NAME borg-mount \- Mount archive or an entire repository as a FUSE filesystem . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -borg [common options] mount REPOSITORY_OR_ARCHIVE MOUNTPOINT +borg [common options] mount [options] REPOSITORY_OR_ARCHIVE MOUNTPOINT .SH DESCRIPTION .sp This command mounts an archive as a FUSE filesystem. This can be useful for @@ -91,16 +91,16 @@ stay in foreground, do not daemonize .B \-o Extra mount options .UNINDENT -.SS filters +.SS Archive filters .INDENT 0.0 .TP -.B \-P\fP,\fB \-\-prefix +.BI \-P \ PREFIX\fP,\fB \ \-\-prefix \ PREFIX only consider archive names starting with this prefix. .TP -.B \-a\fP,\fB \-\-glob\-archives +.BI \-a \ GLOB\fP,\fB \ \-\-glob\-archives \ GLOB only consider archive names matching the glob. sh: rules apply, see "borg help patterns". \fB\-\-prefix\fP and \fB\-\-glob\-archives\fP are mutually exclusive. .TP -.B \-\-sort\-by +.BI \-\-sort\-by \ KEYS Comma\-separated list of sorting keys; valid keys are: timestamp, name, id; default is: timestamp .TP .BI \-\-first \ N diff --git a/docs/man/borg-patterns.1 b/docs/man/borg-patterns.1 index 239a441f0e..acbfdb4641 100644 --- a/docs/man/borg-patterns.1 +++ b/docs/man/borg-patterns.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-PATTERNS 1 "2017-06-18" "" "borg backup tool" +.TH BORG-PATTERNS 1 "2017-08-27" "" "borg backup tool" .SH NAME borg-patterns \- Details regarding patterns . @@ -59,7 +59,7 @@ matching is attempted. Thus, if a given pattern ends in a path separator, a \(aq*\(aq is appended before matching is attempted. .TP .B Shell\-style patterns, selector \fIsh:\fP -This is the default style for \-\-pattern and \-\-patterns\-from. +This is the default style for \fB\-\-pattern\fP and \fB\-\-patterns\-from\fP\&. Like fnmatch patterns these are similar to shell patterns. The difference is that the pattern may include \fI**/\fP for matching zero or more directory levels, \fI*\fP for matching zero or more arbitrary characters with the diff --git a/docs/man/borg-placeholders.1 b/docs/man/borg-placeholders.1 index 3c3efbf8bb..23ec79353c 100644 --- a/docs/man/borg-placeholders.1 +++ b/docs/man/borg-placeholders.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-PLACEHOLDERS 1 "2017-06-18" "" "borg backup tool" +.TH BORG-PLACEHOLDERS 1 "2017-08-27" "" "borg backup tool" .SH NAME borg-placeholders \- Details regarding placeholders . diff --git a/docs/man/borg-prune.1 b/docs/man/borg-prune.1 index 941a398d92..92b2cd35fa 100644 --- a/docs/man/borg-prune.1 +++ b/docs/man/borg-prune.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-PRUNE 1 "2017-06-18" "" "borg backup tool" +.TH BORG-PRUNE 1 "2017-08-27" "" "borg backup tool" .SH NAME borg-prune \- Prune repository archives according to specified rules . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -borg [common options] prune REPOSITORY +borg [common options] prune [options] [REPOSITORY] .SH DESCRIPTION .sp The prune command prunes a repository by deleting all archives not matching @@ -97,7 +97,7 @@ print statistics for the deleted archive .B \-\-list output verbose list of archives it keeps/prunes .TP -.BI \-\-keep\-within \ WITHIN +.BI \-\-keep\-within \ INTERVAL keep all archives within this time interval .TP .B \-\-keep\-last\fP,\fB \-\-keep\-secondly @@ -124,13 +124,13 @@ number of yearly archives to keep .B \-\-save\-space work slower, but using less space .UNINDENT -.SS filters +.SS Archive filters .INDENT 0.0 .TP -.B \-P\fP,\fB \-\-prefix +.BI \-P \ PREFIX\fP,\fB \ \-\-prefix \ PREFIX only consider archive names starting with this prefix. .TP -.B \-a\fP,\fB \-\-glob\-archives +.BI \-a \ GLOB\fP,\fB \ \-\-glob\-archives \ GLOB only consider archive names matching the glob. sh: rules apply, see "borg help patterns". \fB\-\-prefix\fP and \fB\-\-glob\-archives\fP are mutually exclusive. .UNINDENT .SH EXAMPLES @@ -145,8 +145,6 @@ prefix "foo" if you do not also want to match "foobar". .sp It is strongly recommended to always run \fBprune \-v \-\-list \-\-dry\-run ...\fP first so you will see what it would do without it actually doing anything. -.sp -There is also a visualized prune example in \fBdocs/misc/prune\-example.txt\fP\&. .INDENT 0.0 .INDENT 3.5 .sp @@ -167,6 +165,21 @@ $ borg prune \-v \-\-list \-\-keep\-daily=7 \-\-keep\-weekly=4 \-\-keep\-monthly # Keep all backups in the last 10 days, 4 additional end of week archives, # and an end of month archive for every month: $ borg prune \-v \-\-list \-\-keep\-within=10d \-\-keep\-weekly=4 \-\-keep\-monthly=\-1 /path/to/repo +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +There is also a visualized prune example in \fBdocs/misc/prune\-example.txt\fP: +.IP "System Message: ERROR/3 (docs/virtmanpage.rst:, line 140)" +Unknown directive type "highlight". +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +\&.. highlight:: none + .ft P .fi .UNINDENT diff --git a/docs/man/borg-recreate.1 b/docs/man/borg-recreate.1 index 9124d11017..226e79f1bf 100644 --- a/docs/man/borg-recreate.1 +++ b/docs/man/borg-recreate.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-RECREATE 1 "2017-06-18" "" "borg backup tool" +.TH BORG-RECREATE 1 "2017-08-27" "" "borg backup tool" .SH NAME borg-recreate \- Re-create archives . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -borg [common options] recreate REPOSITORY_OR_ARCHIVE PATH +borg [common options] recreate [options] [REPOSITORY_OR_ARCHIVE] [PATH...] .SH DESCRIPTION .sp Recreate the contents of existing archives. @@ -90,7 +90,7 @@ paths to recreate; patterns are supported output verbose list of items (files, dirs, ...) .TP .BI \-\-filter \ STATUSCHARS -only display items with the given status characters +only display items with the given status characters (listed in borg create \-\-help) .TP .B \-n\fP,\fB \-\-dry\-run do not change anything @@ -107,6 +107,12 @@ exclude paths matching PATTERN .BI \-\-exclude\-from \ EXCLUDEFILE read exclude patterns from EXCLUDEFILE, one per line .TP +.BI \-\-pattern \ PATTERN +experimental: include/exclude paths matching PATTERN +.TP +.BI \-\-patterns\-from \ PATTERNFILE +experimental: read include/exclude patterns from PATTERNFILE, one per line +.TP .B \-\-exclude\-caches exclude directories that contain a CACHEDIR.TAG file (\fI\%http://www.brynosaurus.com/cachedir/spec.html\fP) .TP @@ -115,12 +121,6 @@ exclude directories that are tagged by containing a filesystem object with the g .TP .B \-\-keep\-exclude\-tags\fP,\fB \-\-keep\-tag\-files if tag objects are specified with \fB\-\-exclude\-if\-present\fP, don\(aqt omit the tag objects themselves from the backup archive -.TP -.BI \-\-pattern \ PATTERN -experimental: include/exclude paths matching PATTERN -.TP -.BI \-\-patterns\-from \ PATTERNFILE -experimental: read include/exclude patterns from PATTERNFILE, one per line .UNINDENT .SS Archive options .INDENT 0.0 @@ -152,19 +152,20 @@ specify the chunker parameters (CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, HA .sp .nf .ft C -# Make old (Attic / Borg 0.xx) archives deduplicate with Borg 1.x archives -# Archives created with Borg 1.1+ and the default chunker params are skipped (archive ID stays the same) +# Make old (Attic / Borg 0.xx) archives deduplicate with Borg 1.x archives. +# Archives created with Borg 1.1+ and the default chunker params are skipped +# (archive ID stays the same). $ borg recreate /mnt/backup \-\-chunker\-params default \-\-progress # Create a backup with little but fast compression $ borg create /mnt/backup::archive /some/files \-\-compression lz4 -# Then compress it \- this might take longer, but the backup has already completed, so no inconsistencies -# from a long\-running backup job. +# Then compress it \- this might take longer, but the backup has already completed, +# so no inconsistencies from a long\-running backup job. $ borg recreate /mnt/backup::archive \-\-recompress \-\-compression zlib,9 -# Remove unwanted files from all archives in a repository -$ borg recreate /mnt/backup \-e /home/icke/Pictures/drunk_photos - +# Remove unwanted files from all archives in a repository. +# Note the relative path for the \-\-exclude option \- archives only contain relative paths. +$ borg recreate /mnt/backup \-\-exclude home/icke/Pictures/drunk_photos # Change archive comment $ borg create \-\-comment "This is a comment" /mnt/backup::archivename ~ diff --git a/docs/man/borg-rename.1 b/docs/man/borg-rename.1 index 17e65f4395..a118cd1668 100644 --- a/docs/man/borg-rename.1 +++ b/docs/man/borg-rename.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-RENAME 1 "2017-06-18" "" "borg backup tool" +.TH BORG-RENAME 1 "2017-08-27" "" "borg backup tool" .SH NAME borg-rename \- Rename an existing archive . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -borg [common options] rename ARCHIVE NEWNAME +borg [common options] rename [options] ARCHIVE NEWNAME .SH DESCRIPTION .sp This command renames an archive in the repository. diff --git a/docs/man/borg-serve.1 b/docs/man/borg-serve.1 index 5052c724de..8ad1216df6 100644 --- a/docs/man/borg-serve.1 +++ b/docs/man/borg-serve.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-SERVE 1 "2017-06-18" "" "borg backup tool" +.TH BORG-SERVE 1 "2017-08-27" "" "borg backup tool" .SH NAME borg-serve \- Start in server mode. This command is usually not used manually. . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -borg [common options] serve +borg [common options] serve [options] .SH DESCRIPTION .sp This command starts a repository server process. This command is usually not used manually. @@ -46,12 +46,12 @@ See \fIborg\-common(1)\fP for common options of Borg commands. restrict repository access to PATH. Can be specified multiple times to allow the client access to several directories. Access to all sub\-directories is granted implicitly; PATH doesn\(aqt need to directly point to a repository. .TP .BI \-\-restrict\-to\-repository \ PATH -restrict repository access. Only the repository located at PATH (no sub\-directories are considered) is accessible. Can be specified multiple times to allow the client access to several repositories. Unlike \-\-restrict\-to\-path sub\-directories are not accessible; PATH needs to directly point at a repository location. PATH may be an empty directory or the last element of PATH may not exist, in which case the client may initialize a repository there. +restrict repository access. Only the repository located at PATH (no sub\-directories are considered) is accessible. Can be specified multiple times to allow the client access to several repositories. Unlike \fB\-\-restrict\-to\-path\fP sub\-directories are not accessible; PATH needs to directly point at a repository location. PATH may be an empty directory or the last element of PATH may not exist, in which case the client may initialize a repository there. .TP .B \-\-append\-only only allow appending to repository segment files .TP -.B \-\-storage\-quota +.BI \-\-storage\-quota \ QUOTA Override storage quota of the repository (e.g. 5G, 1.5T). When a new repository is initialized, sets the storage quota on the new repository as well. Default: no quota. .UNINDENT .SH EXAMPLES @@ -78,7 +78,7 @@ locations like \fB/etc/environment\fP or in the forced command itself (example b # Use key options to disable unneeded and potentially dangerous SSH functionality. # This will help to secure an automated remote backup system. $ cat ~/.ssh/authorized_keys -command="borg serve \-\-restrict\-to\-path /path/to/repo",no\-pty,no\-agent\-forwarding,no\-port\-forwarding,no\-X11\-forwarding,no\-user\-rc ssh\-rsa AAAAB3[...] +command="borg serve \-\-restrict\-to\-path /path/to/repo",restrict ssh\-rsa AAAAB3[...] # Set a BORG_XXX environment variable on the "borg serve" side $ cat ~/.ssh/authorized_keys @@ -87,6 +87,21 @@ command="export BORG_XXX=value; borg serve [...]",restrict ssh\-rsa [...] .fi .UNINDENT .UNINDENT +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +The examples above use the \fBrestrict\fP directive. This does automatically +block potential dangerous ssh features, even when they are added in a future +update. Thus, this option should be preferred. +.sp +If you\(aqre using openssh\-server < 7.2, however, you have to explicitly specify +the ssh features to restrict and cannot simply use the restrict option as it +has been introduced in v7.2. We recommend to use +\fBno\-port\-forwarding,no\-X11\-forwarding,no\-pty,no\-agent\-forwarding,no\-user\-rc\fP +in this case. +.UNINDENT +.UNINDENT .SH SEE ALSO .sp \fIborg\-common(1)\fP diff --git a/docs/man/borg-umount.1 b/docs/man/borg-umount.1 index 21b3c2eec7..6f7471ead9 100644 --- a/docs/man/borg-umount.1 +++ b/docs/man/borg-umount.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-UMOUNT 1 "2017-06-18" "" "borg backup tool" +.TH BORG-UMOUNT 1 "2017-08-27" "" "borg backup tool" .SH NAME borg-umount \- un-mount the FUSE filesystem . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -borg [common options] umount MOUNTPOINT +borg [common options] umount [options] MOUNTPOINT .SH DESCRIPTION .sp This command un\-mounts a FUSE filesystem that was mounted with \fBborg mount\fP\&. @@ -49,31 +49,39 @@ See \fIborg\-common(1)\fP for common options of Borg commands. mountpoint of the filesystem to umount .UNINDENT .SH EXAMPLES -.SS borg mount .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C +# Mounting the repository shows all archives. +# Archives are loaded lazily, expect some delay when navigating to an archive +# for the first time. +$ borg mount /path/to/repo /tmp/mymountpoint +$ ls /tmp/mymountpoint +root\-2016\-02\-14 root\-2016\-02\-15 +$ borg umount /tmp/mymountpoint + +# Mounting a specific archive is possible as well. $ borg mount /path/to/repo::root\-2016\-02\-15 /tmp/mymountpoint $ ls /tmp/mymountpoint -bin boot etc home lib lib64 lost+found media mnt opt root sbin srv tmp usr var +bin boot etc home lib lib64 lost+found media mnt opt +root sbin srv tmp usr var $ borg umount /tmp/mymountpoint -.ft P -.fi -.UNINDENT -.UNINDENT -.INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C + +# The experimental "versions view" merges all archives in the repository +# and provides a versioned view on files. $ borg mount \-o versions /path/to/repo /tmp/mymountpoint $ ls \-l /tmp/mymountpoint/home/user/doc.txt/ total 24 -\-rw\-rw\-r\-\- 1 user group 12357 Aug 26 21:19 doc.txt.cda00bc9 -\-rw\-rw\-r\-\- 1 user group 12204 Aug 26 21:04 doc.txt.fa760f28 -$ fusermount \-u /tmp/mymountpoint +\-rw\-rw\-r\-\- 1 user group 12357 Aug 26 21:19 doc.cda00bc9.txt +\-rw\-rw\-r\-\- 1 user group 12204 Aug 26 21:04 doc.fa760f28.txt +$ borg umount /tmp/mymountpoint + +# Archive filters are supported. +# These are especially handy for the "versions view", +# which does not support lazy processing of archives. +$ borg mount \-o versions \-\-glob\-archives \(aq*\-my\-home\(aq \-\-last 10 /path/to/repo /tmp/mymountpoint .ft P .fi .UNINDENT diff --git a/docs/man/borg-upgrade.1 b/docs/man/borg-upgrade.1 index 0d4cef89c9..b716f23eea 100644 --- a/docs/man/borg-upgrade.1 +++ b/docs/man/borg-upgrade.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-UPGRADE 1 "2017-06-18" "" "borg backup tool" +.TH BORG-UPGRADE 1 "2017-08-27" "" "borg backup tool" .SH NAME borg-upgrade \- upgrade a repository from a previous version . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -borg [common options] upgrade REPOSITORY +borg [common options] upgrade [options] [REPOSITORY] .SH DESCRIPTION .sp Upgrade an existing, local Borg repository. @@ -106,7 +106,7 @@ borg delete borg .sp Unless \fB\-\-inplace\fP is specified, the upgrade process first creates a backup copy of the repository, in -REPOSITORY.upgrade\-DATETIME, using hardlinks. This takes +REPOSITORY.before\-upgrade\-DATETIME, using hardlinks. This takes longer than in place upgrades, but is much safer and gives progress information (as opposed to \fBcp \-al\fP). Once you are satisfied with the conversion, you can safely destroy the @@ -133,17 +133,16 @@ path to the repository to be upgraded do not change repository .TP .B \-\-inplace -rewrite repository in place, with no chance of going back to older -versions of the repository. +rewrite repository in place, with no chance of going back to older versions of the repository. .TP .B \-\-force Force upgrade .TP .B \-\-tam -Enable manifest authentication (in key and cache) (Borg 1.0.9 and later) +Enable manifest authentication (in key and cache) (Borg 1.0.9 and later). .TP .B \-\-disable\-tam -Disable manifest authentication (in key and cache) +Disable manifest authentication (in key and cache). .UNINDENT .SH EXAMPLES .INDENT 0.0 @@ -153,7 +152,7 @@ Disable manifest authentication (in key and cache) .ft C # Upgrade the borg repository to the most recent version. $ borg upgrade \-v /path/to/repo -making a hardlink copy in /path/to/repo.upgrade\-2016\-02\-15\-20:51:55 +making a hardlink copy in /path/to/repo.before\-upgrade\-2016\-02\-15\-20:51:55 opening attic repository with borg and converting no key file found for repository converting repo index /path/to/repo/index.0 diff --git a/docs/man/borg-with-lock.1 b/docs/man/borg-with-lock.1 index c71bd28a24..1ad8ae6b8a 100644 --- a/docs/man/borg-with-lock.1 +++ b/docs/man/borg-with-lock.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-WITH-LOCK 1 "2017-06-18" "" "borg backup tool" +.TH BORG-WITH-LOCK 1 "2017-08-27" "" "borg backup tool" .SH NAME borg-with-lock \- run a user specified command with the repository lock held . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -borg [common options] with\-lock REPOSITORY COMMAND ARGS +borg [common options] with\-lock [options] REPOSITORY COMMAND [ARGS...] .SH DESCRIPTION .sp This command runs a user\-specified command while the repository lock is held. @@ -41,11 +41,15 @@ It will first try to acquire the lock (make sure that no other operation is running in the repo), then execute the given command as a subprocess and wait for its termination, release the lock and return the user command\(aqs return code as borg\(aqs return code. +.sp +\fBNOTE:\fP .INDENT 0.0 -.TP -.B Note: if you copy a repository with the lock held, the lock will be present in -the copy, obviously. Thus, before using borg on the copy, you need to -use "borg break\-lock" on it. +.INDENT 3.5 +If you copy a repository with the lock held, the lock will be present in +the copy. Thus, before using borg on the copy from a different host, +you need to use "borg break\-lock" on the copied repository, because +Borg is cautious and does not automatically remove stale locks made by a different host. +.UNINDENT .UNINDENT .SH OPTIONS .sp diff --git a/docs/man/borg.1 b/docs/man/borg.1 index 99493444ea..797788a809 100644 --- a/docs/man/borg.1 +++ b/docs/man/borg.1 @@ -472,6 +472,21 @@ know a list of affected hardware. If you are suspicious whether your Borg repository is still consistent and readable after one of the failures mentioned above occurred, run \fBborg check \-\-verify\-data\fP to make sure it is consistent. +Requirements for Borg repository file systems.INDENT 0.0 +.IP \(bu 2 +Long file names +.IP \(bu 2 +At least three directory levels with short names +.IP \(bu 2 +Typically, file sizes up to a few hundred MB. +Large repositories may require large files (>2 GB). +.IP \(bu 2 +Up to 1000 files per directory (10000 for repositories initialized with Borg 1.0) +.IP \(bu 2 +mkdir(2) should be atomic, since it is used for locking +.IP \(bu 2 +Hardlinks are needed for \fIborg_upgrade\fP \fB\-\-inplace\fP +.UNINDENT .SS Units .sp To display quantities, Borg takes care of respecting the @@ -574,6 +589,167 @@ operations used for transaction support also go over the connection. If you backup multiple sources to one target repository, additional traffic happens for cache resynchronization. .UNINDENT +.SS Support for file metadata +.sp +Besides regular file and directory structures, Borg can preserve +.INDENT 0.0 +.IP \(bu 2 +symlinks (stored as symlink, the symlink is not followed) +.IP \(bu 2 +special files: +.INDENT 2.0 +.IP \(bu 2 +character and block device files (restored via mknod) +.IP \(bu 2 +FIFOs ("named pipes") +.IP \(bu 2 +special file \fIcontents\fP can be backed up in \fB\-\-read\-special\fP mode. +By default the metadata to create them with mknod(2), mkfifo(2) etc. is stored. +.UNINDENT +.IP \(bu 2 +hardlinked regular files, devices, FIFOs (considering all items in the same archive) +.IP \(bu 2 +timestamps in nanosecond precision: mtime, atime, ctime +.IP \(bu 2 +permissions: +.INDENT 2.0 +.IP \(bu 2 +IDs of owning user and owning group +.IP \(bu 2 +names of owning user and owning group (if the IDs can be resolved) +.IP \(bu 2 +Unix Mode/Permissions (u/g/o permissions, suid, sgid, sticky) +.UNINDENT +.UNINDENT +.sp +On some platforms additional features are supported: +.\" Yes/No's are grouped by reason/mechanism/reference. +. +.TS +center; +|l|l|l|l|. +_ +T{ +Platform +T} T{ +ACLs +[5] +T} T{ +xattr +[6] +T} T{ +Flags +[7] +T} +_ +T{ +Linux +T} T{ +Yes +T} T{ +Yes +T} T{ +Yes [1] +T} +_ +T{ +Mac OS X +T} T{ +Yes +T} T{ +Yes +T} T{ +Yes (all) +T} +_ +T{ +FreeBSD +T} T{ +Yes +T} T{ +Yes +T} +_ +T{ +OpenBSD +T} T{ +n/a +T} T{ +n/a +T} +_ +T{ +NetBSD +T} T{ +n/a +T} T{ +No [2] +T} +_ +T{ +Solaris 11 +T} T{ +No [3] +T} T{ +n/a +T} +_ +T{ +OpenIndiana +T} +_ +T{ +Windows (cygwin) +T} T{ +No [4] +T} T{ +No +T} T{ +No +T} +_ +.TE +.sp +Other Unix\-like operating systems may work as well, but have not been tested at all. +.sp +Note that most of the platform\-dependent features also depend on the file system. +For example, ntfs\-3g on Linux isn\(aqt able to convey NTFS ACLs. +.IP [1] 5 +Only "nodump", "immutable", "compressed" and "append" are supported. +Feature request +.nf +:issue:\(ga618\(ga +.fi + for more flags. +.IP "System Message: ERROR/3 (docs/usage_general.rst.inc:, line 410)" +Unknown interpreted text role "issue". +.IP [2] 5 +Feature request +.nf +:issue:\(ga1332\(ga +.fi + +.IP "System Message: ERROR/3 (docs/usage_general.rst.inc:, line 412)" +Unknown interpreted text role "issue". +.IP [3] 5 +Feature request +.nf +:issue:\(ga1337\(ga +.fi + +.IP "System Message: ERROR/3 (docs/usage_general.rst.inc:, line 413)" +Unknown interpreted text role "issue". +.IP [4] 5 +Cygwin tries to map NTFS ACLs to permissions with varying degress of success. +.IP [5] 5 +The native access control list mechanism of the OS. This normally limits access to +non\-native ACLs. For example, NTFS ACLs aren\(aqt completely accessible on Linux with ntfs\-3g. +.IP [6] 5 +extended attributes; key\-value pairs attached to a file, mainly used by the OS. +This includes resource forks on Mac OS X. +.IP [7] 5 +aka \fIBSD flags\fP\&. The Linux set of flags [1] is portable across platforms. +The BSDs define additional flags. .SH SEE ALSO .sp \fIborg\-common(1)\fP for common command line options From 35c8975a5c6ea7d11be7018caf59f55dcf66a8d5 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Mon, 28 Aug 2017 10:15:38 +0200 Subject: [PATCH 025/798] keymanager: don't depend on optional readline module (cherry picked from commit b8793d9577da588d8167b7f15d4bc8209e771569) --- src/borg/crypto/keymanager.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/borg/crypto/keymanager.py b/src/borg/crypto/keymanager.py index a997c0db65..aa657af957 100644 --- a/src/borg/crypto/keymanager.py +++ b/src/borg/crypto/keymanager.py @@ -142,8 +142,11 @@ def import_keyfile(self, args): self.store_keyblob(args) def import_paperkey(self, args): - # imported here because it has global side effects - import readline + try: + # imported here because it has global side effects + import readline + except ImportError: + print('Note: No line editing available due to missing readline support') repoid = bin_to_hex(self.repository.id)[:18] try: @@ -151,8 +154,8 @@ def import_paperkey(self, args): # id line input while True: idline = input('id: ').replace(' ', '') - if idline == "": - if yes("Abort import? [yN]:"): + if idline == '': + if yes('Abort import? [yN]:'): raise EOFError() try: @@ -184,8 +187,8 @@ def import_paperkey(self, args): while True: inline = input('{0:2d}: '.format(idx)) inline = inline.replace(' ', '') - if inline == "": - if yes("Abort import? [yN]:"): + if inline == '': + if yes('Abort import? [yN]:'): raise EOFError() try: (data, checksum) = inline.split('-') From 9a362aecc4cc848871a71d09e0f6bc86cc9f8d8a Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 1 Sep 2017 05:26:27 +0200 Subject: [PATCH 026/798] recover_segment: use mmap(), fixes #2982 (cherry picked from commit 9fc4d00bf6b8cae85246bff508e490498279e737) --- src/borg/repository.py | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/src/borg/repository.py b/src/borg/repository.py index 942b862dc4..8de6e52050 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -1,4 +1,5 @@ import errno +import mmap import os import shutil import struct @@ -1305,25 +1306,28 @@ def iter_objects(self, segment, offset=0, include_data=False, read_data=True): header = fd.read(self.header_fmt.size) def recover_segment(self, segment, filename): + logger.info('attempting to recover ' + filename) if segment in self.fds: del self.fds[segment] - with open(filename, 'rb') as fd: - # XXX: Rather use mmap, this loads the entire segment (up to 500 MB by default) into memory. - data = memoryview(fd.read()) - os.rename(filename, filename + '.beforerecover') - logger.info('attempting to recover ' + filename) - with open(filename, 'wb') as fd: - fd.write(MAGIC) - while len(data) >= self.header_fmt.size: - crc, size, tag = self.header_fmt.unpack(data[:self.header_fmt.size]) - if size < self.header_fmt.size or size > len(data): - data = data[1:] - continue - if crc32(data[4:size]) & 0xffffffff != crc: - data = data[1:] - continue - fd.write(data[:size]) - data = data[size:] + backup_filename = filename + '.beforerecover' + os.rename(filename, backup_filename) + with open(backup_filename, 'rb') as backup_fd: + # note: file must not be 0 size (windows can't create 0 size mapping) + with mmap.mmap(backup_fd.fileno(), 0, access=mmap.ACCESS_READ) as mm: + data = memoryview(mm) + with open(filename, 'wb') as fd: + fd.write(MAGIC) + while len(data) >= self.header_fmt.size: + crc, size, tag = self.header_fmt.unpack(data[:self.header_fmt.size]) + if size < self.header_fmt.size or size > len(data): + data = data[1:] + continue + if crc32(data[4:size]) & 0xffffffff != crc: + data = data[1:] + continue + fd.write(data[:size]) + data = data[size:] + data.release() def read(self, segment, offset, id, read_data=True): """ From 9ca490ad5e99c606861ce2777b651c3d388bbad4 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 31 Aug 2017 22:49:30 +0200 Subject: [PATCH 027/798] add debug logging for repository cleanup so we can know whether it did a cleanup and if so, which and how many segments were cleaned up. (cherry picked from commit 57f808e4bb751b841b2614b9d5488d43783a93ba) --- src/borg/repository.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/borg/repository.py b/src/borg/repository.py index 942b862dc4..806a993000 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -1172,11 +1172,15 @@ def cleanup(self, transaction_id): """Delete segment files left by aborted transactions """ self.segment = transaction_id + 1 + count = 0 for segment, filename in self.segment_iterator(reverse=True): if segment > transaction_id: truncate_and_unlink(filename) + count += 1 else: break + logger.debug('Cleaned up %d uncommitted segment files (== everything after segment %d).', + count, transaction_id) def is_committed_segment(self, segment): """Check if segment ends with a COMMIT_TAG tag From f85494d574d99cfc2b02b502e989b69e7dc2f441 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Mon, 28 Aug 2017 10:16:49 +0200 Subject: [PATCH 028/798] test_detect_attic_repo: don't test mount since mount is not always available and if it works for all the other commands, then it is likely it works for mount as well. (cherry picked from commit a6be34f8f7e803dcc0ec8934c2b70d90e5ea03a7) --- src/borg/testsuite/archiver.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index b74de74578..741f6b9672 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -2745,7 +2745,6 @@ def test_detect_attic_repo(self): ['delete', path], ['prune', path], ['info', path + '::test'], - ['mount', path, self.tmpdir], ['key', 'export', path, 'exported'], ['key', 'import', path, 'import'], ['change-passphrase', path], From 732eab841fbf1cdae2a49b1a97602b86b8d171e0 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 3 Sep 2017 08:19:32 +0200 Subject: [PATCH 029/798] vagrant: clean up shell profile init, user name - deduplicated .bash_profile creation - now we always have XDISTN=... and LANG=... (not just for pyenv) - username is just given where needed ("ubuntu" vs "vagrant") - override for cygwin not needed as init is not global any more --- Vagrantfile | 65 +++++++++++++++++++++++------------------------------ 1 file changed, 28 insertions(+), 37 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 2df83ff658..9b516a9320 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -14,21 +14,14 @@ def packages_prepare_wheezy EOF end -def packages_debianoid +def packages_debianoid(user) return <<-EOF - if id "vagrant" >/dev/null 2>&1; then - username='vagrant' - home_dir=/home/vagrant - else - username='ubuntu' - home_dir=/home/ubuntu - fi apt-get update # install all the (security and other) updates apt-get dist-upgrade -y # for building borgbackup and dependencies: apt-get install -y libssl-dev libacl1-dev liblz4-dev libfuse-dev fuse pkg-config - usermod -a -G fuse $username + usermod -a -G fuse #{user} chgrp fuse /dev/fuse chmod 666 /dev/fuse apt-get install -y fakeroot build-essential git @@ -40,7 +33,6 @@ def packages_debianoid # newer versions are not compatible with py 3.2 any more. easy_install3 'pip<8.0' pip3 install 'virtualenv<14.0' - touch $home_dir/.bash_profile ; chown $username $home_dir/.bash_profile EOF end @@ -60,7 +52,6 @@ def packages_redhatted yum install -y zlib-devel bzip2-devel ncurses-devel readline-devel xz xz-devel sqlite-devel #yum install -y python-pip #pip install virtualenv - touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile EOF end @@ -83,7 +74,6 @@ def packages_darwin brew install fakeroot brew install git brew install pkg-config - touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile EOF end @@ -110,7 +100,6 @@ def packages_freebsd pw groupmod operator -M vagrant # /dev/fuse has group operator chmod 666 /dev/fuse - touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile # install all the (security and other) updates, packages pkg update yes | pkg upgrade @@ -132,7 +121,6 @@ def packages_openbsd ln -sf /usr/local/bin/python3.4 /usr/local/bin/python easy_install-3.4 pip pip3 install virtualenv - touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile EOF end @@ -157,7 +145,6 @@ def packages_netbsd ln -s /usr/pkg/bin/python3.4 /usr/pkg/bin/python3 easy_install-3.4 pip pip install virtualenv - touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile EOF end @@ -167,7 +154,6 @@ def packages_openindiana pkg install python-34 clang-3.4 lz4 git python3 -m ensurepip pip3 install -U setuptools pip wheel virtualenv - touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile EOF end @@ -231,9 +217,7 @@ def install_pyenv(boxname) echo 'eval "$(pyenv init -)"' >> ~/.bash_profile echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bash_profile echo 'export PYTHON_CONFIGURE_OPTS="--enable-shared"' >> ~/.bash_profile - echo 'export LANG=en_US.UTF-8' >> ~/.bash_profile EOF - script += "echo 'export XDISTN=%d' >> ~/.bash_profile\n" % [$xdistn] return script end @@ -345,14 +329,13 @@ def run_tests(boxname) EOF end -def fix_perms +def fs_init(user) return <<-EOF - # . ~/.profile - if id "vagrant" >/dev/null 2>&1; then - chown -R vagrant /vagrant/borg - else - chown -R ubuntu /vagrant/borg - fi + chown -R #{user} /vagrant/borg + touch ~#{user}/.bash_profile ; chown #{user} ~#{user}/.bash_profile + echo 'export LANG=en_US.UTF-8' >> ~#{user}/.bash_profile + echo 'export LC_CTYPE=en_US.UTF-8' >> ~#{user}/.bash_profile + echo 'export XDISTN=#{$xdistn}' >> ~#{user}/.bash_profile EOF end @@ -362,9 +345,6 @@ Vagrant.configure(2) do |config| # do not let the VM access . on the host machine via the default shared folder! config.vm.synced_folder ".", "/vagrant", disabled: true - # fix permissions on synced folder - config.vm.provision "fix perms", :type => :shell, :inline => fix_perms - config.vm.provider :virtualbox do |v| #v.gui = true v.cpus = $cpus @@ -376,6 +356,7 @@ Vagrant.configure(2) do |config| b.vm.provider :virtualbox do |v| v.memory = 1024 + $wmem end + b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant") b.vm.provision "install system packages", :type => :shell, :inline => packages_redhatted b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("centos7_64") b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("centos7_64") @@ -389,6 +370,7 @@ Vagrant.configure(2) do |config| b.vm.provider :virtualbox do |v| v.memory = 768 + $wmem end + b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant") b.vm.provision "install system packages", :type => :shell, :inline => packages_redhatted b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("centos6_32") b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("centos6_32") @@ -402,6 +384,7 @@ Vagrant.configure(2) do |config| b.vm.provider :virtualbox do |v| v.memory = 1024 + $wmem end + b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant") b.vm.provision "install system packages", :type => :shell, :inline => packages_redhatted b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("centos6_64") b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("centos6_64") @@ -415,7 +398,8 @@ Vagrant.configure(2) do |config| b.vm.provider :virtualbox do |v| v.memory = 1024 + $wmem end - b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid + b.vm.provision "fs init", :type => :shell, :inline => fs_init("ubuntu") + b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("ubuntu") b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("xenial64") b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true) b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("xenial64") @@ -426,7 +410,8 @@ Vagrant.configure(2) do |config| b.vm.provider :virtualbox do |v| v.memory = 1024 + $wmem end - b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid + b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant") + b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant") b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("trusty64") b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true) b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("trusty64") @@ -437,7 +422,8 @@ Vagrant.configure(2) do |config| b.vm.provider :virtualbox do |v| v.memory = 1024 + $wmem end - b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid + b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant") + b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant") b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("stretch64") b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true) b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("stretch64") @@ -448,7 +434,8 @@ Vagrant.configure(2) do |config| b.vm.provider :virtualbox do |v| v.memory = 1024 + $wmem end - b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid + b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant") + b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant") b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("jessie64") b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true) b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("jessie64") @@ -459,8 +446,9 @@ Vagrant.configure(2) do |config| b.vm.provider :virtualbox do |v| v.memory = 768 + $wmem end + b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant") b.vm.provision "packages prepare wheezy", :type => :shell, :inline => packages_prepare_wheezy - b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid + b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant") b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("wheezy32") b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("wheezy32") b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("wheezy32") @@ -475,8 +463,9 @@ Vagrant.configure(2) do |config| b.vm.provider :virtualbox do |v| v.memory = 1024 + $wmem end + b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant") b.vm.provision "packages prepare wheezy", :type => :shell, :inline => packages_prepare_wheezy - b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid + b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant") b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("wheezy64") b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("wheezy64") b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("wheezy64") @@ -500,6 +489,7 @@ Vagrant.configure(2) do |config| # Disable USB variant requiring Virtualbox proprietary extension pack v.customize ["modifyvm", :id, '--usbehci', 'off', '--usbxhci', 'off'] end + b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant") b.vm.provision "packages darwin", :type => :shell, :privileged => false, :inline => packages_darwin b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("darwin64") b.vm.provision "fix pyenv", :type => :shell, :privileged => false, :inline => fix_pyenv_darwin("darwin64") @@ -519,6 +509,7 @@ Vagrant.configure(2) do |config| v.memory = 1024 + $wmem end b.ssh.shell = "sh" + b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant") b.vm.provision "install system packages", :type => :shell, :inline => packages_freebsd b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("freebsd") b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("freebsd") @@ -535,6 +526,7 @@ Vagrant.configure(2) do |config| v.memory = 1024 + $wmem end b.ssh.shell = "sh" + b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant") b.vm.provision "packages openbsd", :type => :shell, :inline => packages_openbsd b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("openbsd64") b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(false) @@ -546,6 +538,7 @@ Vagrant.configure(2) do |config| b.vm.provider :virtualbox do |v| v.memory = 1024 + $wmem end + b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant") b.vm.provision "packages netbsd", :type => :shell, :inline => packages_netbsd b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("netbsd64") b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(false) @@ -559,6 +552,7 @@ Vagrant.configure(2) do |config| b.vm.provider :virtualbox do |v| v.memory = 1536 + $wmem end + b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant") b.vm.provision "packages openindiana", :type => :shell, :inline => packages_openindiana b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("openindiana64") b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(false) @@ -581,9 +575,6 @@ Vagrant.configure(2) do |config| #v.gui = true end - # fix permissions placeholder - b.vm.provision "fix perms", :type => :shell, :privileged => false, :inline => "echo 'fix permission placeholder'" - b.vm.provision "packages cygwin", :type => :shell, :privileged => false, :inline => packages_cygwin("x86_64") b.vm.provision :reload b.vm.provision "cygwin install pip", :type => :shell, :privileged => false, :inline => install_cygwin_venv From 6dab333616e6630110c5a8d75b6f3e55c0818747 Mon Sep 17 00:00:00 2001 From: Markus Engelbrecht Date: Sun, 3 Sep 2017 20:03:11 +0200 Subject: [PATCH 030/798] Fix macOS keychain integration command (cherry picked from commit aadb9cd2eeb78704410efbbb645cc6fb393c7987) --- docs/faq.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/faq.rst b/docs/faq.rst index fe5700cfa8..2363b4525e 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -342,7 +342,7 @@ Using ``BORG_PASSCOMMAND`` with MacOS Keychain In your backup script retrieve it in the ``BORG_PASSCOMMAND``:: - export BORG_PASSCOMMAND="security find-generic-password -a $USER -s borg-passphrase" + export BORG_PASSCOMMAND="security find-generic-password -a $USER -s borg-passphrase -w" Using ``BORG_PASSCOMMAND`` with GNOME Keyring GNOME also has a keyring daemon that can be used to store a Borg passphrase. From ff1c4d1f4c204ab4cef991902ca5845388a071ac Mon Sep 17 00:00:00 2001 From: Markus Engelbrecht Date: Sun, 3 Sep 2017 20:04:28 +0200 Subject: [PATCH 031/798] Use correct casing for macOS (cherry picked from commit e15732822845b549e83266e4d13065121712c1fc) --- docs/faq.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 2363b4525e..0cc0444354 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -328,8 +328,8 @@ Using keyfile-based encryption with a blank passphrase ``repokey`` mode and use a blank passphrase for the key file. See :ref:`encrypted_repos` for more details. -Using ``BORG_PASSCOMMAND`` with MacOS Keychain - MacOS has a native manager for secrets (such as passphrases) which is safer +Using ``BORG_PASSCOMMAND`` with macOS Keychain + macOS has a native manager for secrets (such as passphrases) which is safer than just using a file as it is encrypted at rest and unlocked manually (fortunately, the login keyring automatically unlocks when you login). With the built-in ``security`` command, you can access it from the command line, @@ -759,7 +759,7 @@ Here's a (incomplete) list of some major changes: * uses fadvise to not spoil / blow up the fs cache * better error messages / exception handling * better logging, screen output, progress indication -* tested on misc. Linux systems, 32 and 64bit, FreeBSD, OpenBSD, NetBSD, MacOS +* tested on misc. Linux systems, 32 and 64bit, FreeBSD, OpenBSD, NetBSD, macOS Please read the :ref:`changelog` (or ``docs/changes.rst`` in the source distribution) for more information. From eeaa43b66845d58669e7e7c363c6cca978008433 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 6 Sep 2017 03:49:45 +0200 Subject: [PATCH 032/798] with-lock: close segment file before invoking subprocess (cherry picked from commit b9dce0ebdcdd709818de2ca61ca02065af69f01c) --- src/borg/archiver.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 0a684c19b2..0e567b9111 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -1664,6 +1664,11 @@ def do_with_lock(self, args, repository): # the encryption key (and can operate just with encrypted data). data = repository.get(Manifest.MANIFEST_ID) repository.put(Manifest.MANIFEST_ID, data) + # usually, a 0 byte (open for writing) segment file would be visible in the filesystem here. + # we write and close this file, to rather have a valid segment file on disk, before invoking the subprocess. + # we can only do this for local repositories (with .io), though: + if hasattr(repository, 'io'): + repository.io.close_segment() try: # we exit with the return code we get from the subprocess return subprocess.call([args.command] + args.args) From 2a58fe42668f5375ee6d2a811ef25a2ddf4a97f4 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 6 Sep 2017 05:56:26 +0200 Subject: [PATCH 033/798] repo: add test case for uncommitted garbage segment files (cherry picked from commit 4a4c8884ee50de5a9f02e542f6813ca5c25e3181) --- src/borg/testsuite/repository.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/borg/testsuite/repository.py b/src/borg/testsuite/repository.py index e29c7d532a..1bec140069 100644 --- a/src/borg/testsuite/repository.py +++ b/src/borg/testsuite/repository.py @@ -210,6 +210,23 @@ def test_sparse_delete(self): self.repository.commit() assert 0 not in [segment for segment, _ in self.repository.io.segment_iterator()] + def test_uncommitted_garbage(self): + # uncommitted garbage should be no problem, it is cleaned up automatically. + # we just have to be careful with invalidation of cached FDs in LoggedIO. + self.repository.put(H(0), b'foo') + self.repository.commit() + # write some crap to a uncommitted segment file + last_segment = self.repository.io.get_latest_segment() + with open(self.repository.io.segment_filename(last_segment + 1), 'wb') as f: + f.write(MAGIC + b'crapcrapcrap') + self.repository.close() + # usually, opening the repo and starting a transaction should trigger a cleanup. + self.repository = self.open() + with self.repository: + self.repository.put(H(0), b'bar') # this may trigger compact_segments() + self.repository.commit() + # the point here is that nothing blows up with an exception. + class RepositoryCommitTestCase(RepositoryTestCaseBase): From 88e134044572445b643f8b6ac7185704baa84a37 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 6 Sep 2017 05:42:22 +0200 Subject: [PATCH 034/798] repo cleanup/write: invalidate cached FDs (cherry picked from commit 71229138255e9c3541efd83b89bbec960944b9d6) --- src/borg/repository.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/borg/repository.py b/src/borg/repository.py index ca55c69880..dada3c14e7 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -1176,6 +1176,8 @@ def cleanup(self, transaction_id): count = 0 for segment, filename in self.segment_iterator(reverse=True): if segment > transaction_id: + if segment in self.fds: + del self.fds[segment] truncate_and_unlink(filename) count += 1 else: @@ -1232,6 +1234,12 @@ def get_write_fd(self, no_new=False, raise_full=False): self._write_fd = SyncFile(self.segment_filename(self.segment), binary=True) self._write_fd.write(MAGIC) self.offset = MAGIC_LEN + if self.segment in self.fds: + # we may have a cached fd for a segment file we already deleted and + # we are writing now a new segment file to same file name. get rid of + # of the cached fd that still refers to the old file, so it will later + # get repopulated (on demand) with a fd that refers to the new file. + del self.fds[self.segment] return self._write_fd def get_fd(self, segment): From 764324304d5abe512ea2d4df679700e1511df65b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 5 Sep 2017 04:44:38 +0200 Subject: [PATCH 035/798] move ISO_FORMAT to constants module also: add ISO_FORMAT_NO_USECS (cherry picked from commit 1cb158a4b582f0543f92e97ed5a074012a090ea2) --- src/borg/constants.py | 6 ++++++ src/borg/testsuite/archiver.py | 2 -- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/borg/constants.py b/src/borg/constants.py index 6c13a9c98a..36d6ee45bc 100644 --- a/src/borg/constants.py +++ b/src/borg/constants.py @@ -66,6 +66,12 @@ EXIT_WARNING = 1 # reached normal end of operation, but there were issues EXIT_ERROR = 2 # terminated abruptly, did not reach end of operation +# never use datetime.isoformat(), it is evil. always use one of these: +# datetime.strftime(ISO_FORMAT) # output always includes .microseconds +# datetime.strftime(ISO_FORMAT_NO_USECS) # output never includes microseconds +ISO_FORMAT_NO_USECS = '%Y-%m-%dT%H:%M:%S' +ISO_FORMAT = ISO_FORMAT_NO_USECS + '.%f' + DASHES = '-' * 78 PBKDF2_ITERATIONS = 100000 diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 741f6b9672..3583562086 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -61,8 +61,6 @@ src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) -ISO_FORMAT = '%Y-%m-%dT%H:%M:%S.%f' - def exec_cmd(*args, archiver=None, fork=False, exe=None, input=b'', binary_output=False, **kw): if fork: From 457f5ceb30c1817556c1423ef42be4a63dee9ff1 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 5 Sep 2017 05:02:44 +0200 Subject: [PATCH 036/798] use ISO_FORMAT* constants (cherry picked from commit eebb117349dff243ea8fd61169678c99b79865dd) --- src/borg/helpers.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 64ee4c82a4..b7bf885253 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -318,7 +318,7 @@ def id_str(self): @property def last_timestamp(self): - return datetime.strptime(self.timestamp, "%Y-%m-%dT%H:%M:%S.%f") + return datetime.strptime(self.timestamp, ISO_FORMAT) @classmethod def load(cls, repository, operations, key=None, force_tam_not_required=False): @@ -525,10 +525,8 @@ def to_localtime(ts): def parse_timestamp(timestamp): """Parse a ISO 8601 timestamp string""" - if '.' in timestamp: # microseconds might not be present - return datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%f').replace(tzinfo=timezone.utc) - else: - return datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S').replace(tzinfo=timezone.utc) + fmt = ISO_FORMAT if '.' in timestamp else ISO_FORMAT_NO_USECS + return datetime.strptime(timestamp, fmt).replace(tzinfo=timezone.utc) def timestamp(s): @@ -616,7 +614,7 @@ def __init__(self, dt): def __format__(self, format_spec): if format_spec == '': - format_spec = '%Y-%m-%dT%H:%M:%S' + format_spec = ISO_FORMAT_NO_USECS return self.dt.__format__(format_spec) @@ -727,7 +725,7 @@ def isoformat_time(ts: datetime): Format *ts* according to ISO 8601. """ # note: first make all datetime objects tz aware before adding %z here. - return ts.strftime('%Y-%m-%dT%H:%M:%S.%f') + return ts.strftime(ISO_FORMAT) def format_timedelta(td): From a9aa3c5f340eb3d8baca1871aa93579a81ef5f9f Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 5 Sep 2017 05:51:00 +0200 Subject: [PATCH 037/798] use safe parse_timestamp to parse timestamps, fixes #2994 also: refactor so it is possible to get tz-unaware datetime objects from parse_timestamp. (cherry picked from commit 7996a87357e4e67a833bcee5a54ef8b41c491518) --- src/borg/helpers.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/borg/helpers.py b/src/borg/helpers.py index b7bf885253..4815d54caa 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -318,7 +318,7 @@ def id_str(self): @property def last_timestamp(self): - return datetime.strptime(self.timestamp, ISO_FORMAT) + return parse_timestamp(self.timestamp, tzinfo=None) @classmethod def load(cls, repository, operations, key=None, force_tam_not_required=False): @@ -523,10 +523,13 @@ def to_localtime(ts): return datetime(*time.localtime((ts - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds())[:6]) -def parse_timestamp(timestamp): +def parse_timestamp(timestamp, tzinfo=timezone.utc): """Parse a ISO 8601 timestamp string""" fmt = ISO_FORMAT if '.' in timestamp else ISO_FORMAT_NO_USECS - return datetime.strptime(timestamp, fmt).replace(tzinfo=timezone.utc) + dt = datetime.strptime(timestamp, fmt) + if tzinfo is not None: + dt = dt.replace(tzinfo=tzinfo) + return dt def timestamp(s): From da2f8dbe81cd88aff973c58b5dd69193625335bc Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 5 Sep 2017 06:13:47 +0200 Subject: [PATCH 038/798] get rid of datetime.isoformat to avoid bugs like #2994 (cherry picked from commit 928bde8676a1b8cd45c271291af0f15da0308d93) --- src/borg/archive.py | 4 ++-- src/borg/helpers.py | 8 ++++---- src/borg/repository.py | 3 ++- src/borg/testsuite/archiver.py | 4 ++-- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index eb9ee053b9..6d78aff413 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -461,8 +461,8 @@ def save(self, name=None, comment=None, timestamp=None, additional_metadata=None 'cmdline': sys.argv, 'hostname': socket.gethostname(), 'username': getuser(), - 'time': start.isoformat(), - 'time_end': end.isoformat(), + 'time': start.strftime(ISO_FORMAT), + 'time_end': end.strftime(ISO_FORMAT), 'chunker_params': self.chunker_params, } metadata.update(additional_metadata or {}) diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 4815d54caa..0432db2f0e 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -216,7 +216,7 @@ def __setitem__(self, name, info): id, ts = info assert isinstance(id, bytes) if isinstance(ts, datetime): - ts = ts.replace(tzinfo=None).isoformat() + ts = ts.replace(tzinfo=None).strftime(ISO_FORMAT) assert isinstance(ts, str) ts = ts.encode() self._archives[name] = {b'id': id, b'time': ts} @@ -388,11 +388,11 @@ def write(self): self.config[b'tam_required'] = True # self.timestamp needs to be strictly monotonically increasing. Clocks often are not set correctly if self.timestamp is None: - self.timestamp = datetime.utcnow().isoformat() + self.timestamp = datetime.utcnow().strftime(ISO_FORMAT) else: prev_ts = self.last_timestamp - incremented = (prev_ts + timedelta(microseconds=1)).isoformat() - self.timestamp = max(incremented, datetime.utcnow().isoformat()) + incremented = (prev_ts + timedelta(microseconds=1)).strftime(ISO_FORMAT) + self.timestamp = max(incremented, datetime.utcnow().strftime(ISO_FORMAT)) # include checks for limits as enforced by limited unpacker (used by load()) assert len(self.archives) <= MAX_ARCHIVES assert all(len(name) <= 255 for name in self.archives) diff --git a/src/borg/repository.py b/src/borg/repository.py index ca55c69880..86b4ae9b75 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -540,7 +540,8 @@ def rename_tmp(file): # Log transaction in append-only mode if self.append_only: with open(os.path.join(self.path, 'transactions'), 'a') as log: - print('transaction %d, UTC time %s' % (transaction_id, datetime.utcnow().isoformat()), file=log) + print('transaction %d, UTC time %s' % ( + transaction_id, datetime.utcnow().strftime(ISO_FORMAT)), file=log) # Write hints file hints_name = 'hints.%d' % transaction_id diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 3583562086..b3575ddb29 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -3035,7 +3035,7 @@ def spoof_manifest(self, repository): 'version': 1, 'archives': {}, 'config': {}, - 'timestamp': (datetime.utcnow() + timedelta(days=1)).isoformat(), + 'timestamp': (datetime.utcnow() + timedelta(days=1)).strftime(ISO_FORMAT), }))) repository.commit() @@ -3047,7 +3047,7 @@ def test_fresh_init_tam_required(self): repository.put(Manifest.MANIFEST_ID, key.encrypt(msgpack.packb({ 'version': 1, 'archives': {}, - 'timestamp': (datetime.utcnow() + timedelta(days=1)).isoformat(), + 'timestamp': (datetime.utcnow() + timedelta(days=1)).strftime(ISO_FORMAT), }))) repository.commit() From b398f0dcf706998a020b1c758db1ea1546f90efd Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 7 Sep 2017 19:15:53 +0200 Subject: [PATCH 039/798] show/link new screencasts in README, fixes #2936 (cherry picked from commit 7c9561afa2260383b47dc03e2645ae661c495736) --- README.rst | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index fbdd9d53ef..6bc6117876 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,6 @@ -|screencast| +|screencast_basic| +More screencasts: `installation`_, `advanced usage`_ What is BorgBackup? ------------------- @@ -172,9 +173,13 @@ see ``docs/suppport.rst`` in the source distribution). :alt: Test Coverage :target: https://codecov.io/github/borgbackup/borg?branch=master -.. |screencast| image:: https://asciinema.org/a/28691.png - :alt: BorgBackup Installation and Basic Usage - :target: https://asciinema.org/a/28691?autoplay=1&speed=2 +.. |screencast_basic| image:: https://asciinema.org/a/133292.png + :alt: BorgBackup Basic Usage + :target: https://asciinema.org/a/133292?autoplay=1&speed=1 + +.. _installation: https://asciinema.org/a/133291?autoplay=1&speed=1 + +.. _advanced usage: https://asciinema.org/a/133293?autoplay=1&speed=1 .. |bestpractices| image:: https://bestpractices.coreinfrastructure.org/projects/271/badge :alt: Best Practices Score From a99060508f0c6260a18bb897255e12b87d0ae005 Mon Sep 17 00:00:00 2001 From: TW Date: Fri, 8 Sep 2017 08:31:51 +0200 Subject: [PATCH 040/798] document utf-8 locale requirement for json mode, #2273 (#3009) (cherry picked from commit 133e847f8ee42a2bf6da3f4b6a14d98a27c01be8) --- docs/internals/frontends.rst | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/internals/frontends.rst b/docs/internals/frontends.rst index cfe9730fb6..75831c95e3 100644 --- a/docs/internals/frontends.rst +++ b/docs/internals/frontends.rst @@ -11,7 +11,23 @@ but does mean that there are no release-to-release guarantees on what you might even for point releases (1.1.x), and there is no documentation beyond the code and the internals documents. Borg does on the other hand provide an API on a command-line level. In other words, a frontend should to -(for example) create a backup archive just invoke :ref:`borg_create`. +(for example) create a backup archive just invoke :ref:`borg_create`, give commandline parameters/options +as needed and parse JSON output from borg. + +Important: JSON output is expected to be UTF-8, but currently borg depends on the locale being configured +for that (must be a UTF-8 locale and *not* "C" or "ascii"), so that Python will choose to encode to UTF-8. +The same applies to any inputs read by borg, they are expected to be UTF-8 encoded also. + +We consider this a bug (see :issue:`2273`) and might fix it later, so borg will use UTF-8 independent of +the locale. + +On POSIX systems, you can usually set environment vars to choose a UTF-8 locale: + +:: + + export LANG=en_US.UTF-8 + export LC_CTYPE=en_US.UTF-8 + Logging ------- From ecc5ceec07a25ca8f170cab31e08763627ae2ac9 Mon Sep 17 00:00:00 2001 From: enkore Date: Sat, 9 Sep 2017 23:08:23 +0200 Subject: [PATCH 041/798] delete: support naming multiple archives (#3017) (cherry picked from commit 7c5a9d89b2a8deac84a641bb4ccad00a575a2765) --- src/borg/archiver.py | 16 +++++++++++++--- src/borg/testsuite/archiver.py | 11 +++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 0e567b9111..f6b177d366 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -1183,7 +1183,12 @@ def do_rename(self, args, repository, manifest, key, cache, archive): @with_repository(exclusive=True, manifest=False) def do_delete(self, args, repository): """Delete an existing repository or archives""" - if any((args.location.archive, args.first, args.last, args.prefix, args.glob_archives)): + archive_filter_specified = args.first or args.last or args.prefix or args.glob_archives + explicit_archives_specified = args.location.archive or args.archives + if archive_filter_specified and explicit_archives_specified: + self.print_error('Mixing archive filters and explicitly named archives is not supported.') + return self.exit_code + if archive_filter_specified or explicit_archives_specified: return self._delete_archives(args, repository) else: return self._delete_repository(args, repository) @@ -1192,8 +1197,11 @@ def _delete_archives(self, args, repository): """Delete archives""" manifest, key = Manifest.load(repository, (Manifest.Operation.DELETE,)) - if args.location.archive: - archive_names = (args.location.archive,) + if args.location.archive or args.archives: + archives = list(args.archives) + if args.location.archive: + archives.insert(0, args.location.archive) + archive_names = tuple(archives) else: archive_names = tuple(x.name for x in manifest.archives.list_considering(args)) if not archive_names: @@ -3096,6 +3104,8 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): subparser.add_argument('location', metavar='TARGET', nargs='?', default='', type=location_validator(), help='archive or repository to delete') + subparser.add_argument('archives', metavar='ARCHIVE', nargs='*', + help='archives to delete') define_archive_filters_group(subparser) list_epilog = process_epilog(""" diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index b3575ddb29..1fea6df93f 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -1367,6 +1367,17 @@ def test_delete(self): with Repository(self.repository_path) as repository: self.assert_equal(len(repository), 1) + def test_delete_multiple(self): + self.create_regular_file('file1', size=1024 * 80) + self.cmd('init', '--encryption=repokey', self.repository_location) + self.cmd('create', self.repository_location + '::test1', 'input') + self.cmd('create', self.repository_location + '::test2', 'input') + self.cmd('create', self.repository_location + '::test3', 'input') + self.cmd('delete', self.repository_location + '::test1', 'test2') + self.cmd('extract', '--dry-run', self.repository_location + '::test3') + self.cmd('delete', self.repository_location, 'test3') + assert not self.cmd('list', self.repository_location) + def test_delete_repo(self): self.create_regular_file('file1', size=1024 * 80) self.create_regular_file('dir2/file2', size=1024 * 80) From 3ed6f2367cb75dcd6b7a63fb10b49655637f4a7f Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 8 Sep 2017 01:54:55 +0200 Subject: [PATCH 042/798] update CHANGES (1.1-maint) --- docs/changes.rst | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/docs/changes.rst b/docs/changes.rst index bbdb09fc19..6cec0ca206 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -131,6 +131,33 @@ The best check that everything is ok is to run a dry-run extraction:: Changelog ========= +Version 1.1.0rc3 (2017-09-10) +----------------------------- + +New features: + +- delete: support naming multiple archives, #2958 + +Fixes: + +- repo cleanup/write: invalidate cached FDs, #2982 +- fix datetime.isoformat() microseconds issues, #2994 +- recover_segment: use mmap(), lower memory needs, #2987 + +Other changes: + +- with-lock: close segment file before invoking subprocess +- keymanager: don't depend on optional readline module, #2976 +- docs: + + - fix macOS keychain integration command + - show/link new screencasts in README, #2936 + - document utf-8 locale requirement for json mode, #2273 +- vagrant: clean up shell profile init, user name, #2977 +- test_detect_attic_repo: don't test mount, #2975 +- add debug logging for repository cleanup + + Version 1.1.0rc2 (2017-08-28) ----------------------------- From 2d7dc08c8ce797b4b0a250f14191b339f414a302 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 10 Sep 2017 00:07:45 +0200 Subject: [PATCH 043/798] build_usage --- docs/usage/delete.rst.inc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/usage/delete.rst.inc b/docs/usage/delete.rst.inc index e965b830f6..93a3ec3cb8 100644 --- a/docs/usage/delete.rst.inc +++ b/docs/usage/delete.rst.inc @@ -6,7 +6,7 @@ borg delete ----------- .. code-block:: none - borg [common options] delete [options] [TARGET] + borg [common options] delete [options] [TARGET] [ARCHIVE...] .. only:: html @@ -17,6 +17,8 @@ borg delete +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``TARGET`` | archive or repository to delete | +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``ARCHIVE`` | archives to delete | + +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ | **optional arguments** | +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``-s``, ``--stats`` | print statistics for the deleted archive | @@ -56,6 +58,8 @@ borg delete TARGET archive or repository to delete + ARCHIVE + archives to delete optional arguments From 313529b929a99e6a38c89355255f801259078005 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 10 Sep 2017 00:08:25 +0200 Subject: [PATCH 044/798] build_man --- docs/man/borg-benchmark-crud.1 | 2 +- docs/man/borg-benchmark.1 | 2 +- docs/man/borg-break-lock.1 | 2 +- docs/man/borg-change-passphrase.1 | 2 +- docs/man/borg-check.1 | 2 +- docs/man/borg-common.1 | 2 +- docs/man/borg-compression.1 | 2 +- docs/man/borg-create.1 | 2 +- docs/man/borg-delete.1 | 7 +++++-- docs/man/borg-diff.1 | 2 +- docs/man/borg-export-tar.1 | 2 +- docs/man/borg-extract.1 | 2 +- docs/man/borg-info.1 | 2 +- docs/man/borg-init.1 | 2 +- docs/man/borg-key-change-passphrase.1 | 2 +- docs/man/borg-key-export.1 | 2 +- docs/man/borg-key-import.1 | 2 +- docs/man/borg-key-migrate-to-repokey.1 | 2 +- docs/man/borg-key.1 | 2 +- docs/man/borg-list.1 | 2 +- docs/man/borg-mount.1 | 2 +- docs/man/borg-patterns.1 | 2 +- docs/man/borg-placeholders.1 | 2 +- docs/man/borg-prune.1 | 2 +- docs/man/borg-recreate.1 | 2 +- docs/man/borg-rename.1 | 2 +- docs/man/borg-serve.1 | 2 +- docs/man/borg-umount.1 | 2 +- docs/man/borg-upgrade.1 | 2 +- docs/man/borg-with-lock.1 | 2 +- 30 files changed, 34 insertions(+), 31 deletions(-) diff --git a/docs/man/borg-benchmark-crud.1 b/docs/man/borg-benchmark-crud.1 index 31b059f61d..cf83afb7d9 100644 --- a/docs/man/borg-benchmark-crud.1 +++ b/docs/man/borg-benchmark-crud.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-BENCHMARK-CRUD 1 "2017-08-27" "" "borg backup tool" +.TH BORG-BENCHMARK-CRUD 1 "2017-09-09" "" "borg backup tool" .SH NAME borg-benchmark-crud \- Benchmark Create, Read, Update, Delete for archives. . diff --git a/docs/man/borg-benchmark.1 b/docs/man/borg-benchmark.1 index a56a0bc8c5..7a1febe4d0 100644 --- a/docs/man/borg-benchmark.1 +++ b/docs/man/borg-benchmark.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-BENCHMARK 1 "2017-08-27" "" "borg backup tool" +.TH BORG-BENCHMARK 1 "2017-09-09" "" "borg backup tool" .SH NAME borg-benchmark \- benchmark command . diff --git a/docs/man/borg-break-lock.1 b/docs/man/borg-break-lock.1 index 7a3b9c2534..31c2bdba6b 100644 --- a/docs/man/borg-break-lock.1 +++ b/docs/man/borg-break-lock.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-BREAK-LOCK 1 "2017-08-27" "" "borg backup tool" +.TH BORG-BREAK-LOCK 1 "2017-09-09" "" "borg backup tool" .SH NAME borg-break-lock \- Break the repository lock (e.g. in case it was left by a dead borg. . diff --git a/docs/man/borg-change-passphrase.1 b/docs/man/borg-change-passphrase.1 index fc85a7da5e..2c3a6dca85 100644 --- a/docs/man/borg-change-passphrase.1 +++ b/docs/man/borg-change-passphrase.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-CHANGE-PASSPHRASE 1 "2017-08-27" "" "borg backup tool" +.TH BORG-CHANGE-PASSPHRASE 1 "2017-09-09" "" "borg backup tool" .SH NAME borg-change-passphrase \- Change repository key file passphrase . diff --git a/docs/man/borg-check.1 b/docs/man/borg-check.1 index a05dcf23d3..af3a105f00 100644 --- a/docs/man/borg-check.1 +++ b/docs/man/borg-check.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-CHECK 1 "2017-08-27" "" "borg backup tool" +.TH BORG-CHECK 1 "2017-09-09" "" "borg backup tool" .SH NAME borg-check \- Check repository consistency . diff --git a/docs/man/borg-common.1 b/docs/man/borg-common.1 index cd6cfc7572..68f703283a 100644 --- a/docs/man/borg-common.1 +++ b/docs/man/borg-common.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-COMMON 1 "2017-08-27" "" "borg backup tool" +.TH BORG-COMMON 1 "2017-09-09" "" "borg backup tool" .SH NAME borg-common \- Common options of Borg commands . diff --git a/docs/man/borg-compression.1 b/docs/man/borg-compression.1 index 3c6c53468f..d2fbba59c3 100644 --- a/docs/man/borg-compression.1 +++ b/docs/man/borg-compression.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-COMPRESSION 1 "2017-08-27" "" "borg backup tool" +.TH BORG-COMPRESSION 1 "2017-09-09" "" "borg backup tool" .SH NAME borg-compression \- Details regarding compression . diff --git a/docs/man/borg-create.1 b/docs/man/borg-create.1 index f41536784b..ae8f532756 100644 --- a/docs/man/borg-create.1 +++ b/docs/man/borg-create.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-CREATE 1 "2017-08-27" "" "borg backup tool" +.TH BORG-CREATE 1 "2017-09-09" "" "borg backup tool" .SH NAME borg-create \- Create new archive . diff --git a/docs/man/borg-delete.1 b/docs/man/borg-delete.1 index 318d8b16d0..01513e50c3 100644 --- a/docs/man/borg-delete.1 +++ b/docs/man/borg-delete.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-DELETE 1 "2017-08-27" "" "borg backup tool" +.TH BORG-DELETE 1 "2017-09-09" "" "borg backup tool" .SH NAME borg-delete \- Delete an existing repository or archives . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -borg [common options] delete [options] [TARGET] +borg [common options] delete [options] [TARGET] [ARCHIVE...] .SH DESCRIPTION .sp This command deletes an archive from the repository or the complete repository. @@ -46,6 +46,9 @@ See \fIborg\-common(1)\fP for common options of Borg commands. .TP .B TARGET archive or repository to delete +.TP +.B ARCHIVE +archives to delete .UNINDENT .SS optional arguments .INDENT 0.0 diff --git a/docs/man/borg-diff.1 b/docs/man/borg-diff.1 index 555e6cc7a7..587bf56d7f 100644 --- a/docs/man/borg-diff.1 +++ b/docs/man/borg-diff.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-DIFF 1 "2017-08-27" "" "borg backup tool" +.TH BORG-DIFF 1 "2017-09-09" "" "borg backup tool" .SH NAME borg-diff \- Diff contents of two archives . diff --git a/docs/man/borg-export-tar.1 b/docs/man/borg-export-tar.1 index ba509ac4dd..b2de54dce8 100644 --- a/docs/man/borg-export-tar.1 +++ b/docs/man/borg-export-tar.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-EXPORT-TAR 1 "2017-08-27" "" "borg backup tool" +.TH BORG-EXPORT-TAR 1 "2017-09-09" "" "borg backup tool" .SH NAME borg-export-tar \- Export archive contents as a tarball . diff --git a/docs/man/borg-extract.1 b/docs/man/borg-extract.1 index ee261b497e..df5c48da52 100644 --- a/docs/man/borg-extract.1 +++ b/docs/man/borg-extract.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-EXTRACT 1 "2017-08-27" "" "borg backup tool" +.TH BORG-EXTRACT 1 "2017-09-09" "" "borg backup tool" .SH NAME borg-extract \- Extract archive contents . diff --git a/docs/man/borg-info.1 b/docs/man/borg-info.1 index 7016d2cf44..51721f49ad 100644 --- a/docs/man/borg-info.1 +++ b/docs/man/borg-info.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-INFO 1 "2017-08-27" "" "borg backup tool" +.TH BORG-INFO 1 "2017-09-09" "" "borg backup tool" .SH NAME borg-info \- Show archive details such as disk space used . diff --git a/docs/man/borg-init.1 b/docs/man/borg-init.1 index dc7a930235..556002946f 100644 --- a/docs/man/borg-init.1 +++ b/docs/man/borg-init.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-INIT 1 "2017-08-27" "" "borg backup tool" +.TH BORG-INIT 1 "2017-09-09" "" "borg backup tool" .SH NAME borg-init \- Initialize an empty repository . diff --git a/docs/man/borg-key-change-passphrase.1 b/docs/man/borg-key-change-passphrase.1 index f6da306e3b..2161921e7c 100644 --- a/docs/man/borg-key-change-passphrase.1 +++ b/docs/man/borg-key-change-passphrase.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-CHANGE-PASSPHRASE 1 "2017-08-27" "" "borg backup tool" +.TH BORG-KEY-CHANGE-PASSPHRASE 1 "2017-09-09" "" "borg backup tool" .SH NAME borg-key-change-passphrase \- Change repository key file passphrase . diff --git a/docs/man/borg-key-export.1 b/docs/man/borg-key-export.1 index f4e52608c1..974e1aba0e 100644 --- a/docs/man/borg-key-export.1 +++ b/docs/man/borg-key-export.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-EXPORT 1 "2017-08-27" "" "borg backup tool" +.TH BORG-KEY-EXPORT 1 "2017-09-09" "" "borg backup tool" .SH NAME borg-key-export \- Export the repository key for backup . diff --git a/docs/man/borg-key-import.1 b/docs/man/borg-key-import.1 index 0bd5aa482c..fb2bf6357c 100644 --- a/docs/man/borg-key-import.1 +++ b/docs/man/borg-key-import.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-IMPORT 1 "2017-08-27" "" "borg backup tool" +.TH BORG-KEY-IMPORT 1 "2017-09-09" "" "borg backup tool" .SH NAME borg-key-import \- Import the repository key from backup . diff --git a/docs/man/borg-key-migrate-to-repokey.1 b/docs/man/borg-key-migrate-to-repokey.1 index b6f5efac24..10340f048f 100644 --- a/docs/man/borg-key-migrate-to-repokey.1 +++ b/docs/man/borg-key-migrate-to-repokey.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-MIGRATE-TO-REPOKEY 1 "2017-08-27" "" "borg backup tool" +.TH BORG-KEY-MIGRATE-TO-REPOKEY 1 "2017-09-09" "" "borg backup tool" .SH NAME borg-key-migrate-to-repokey \- Migrate passphrase -> repokey . diff --git a/docs/man/borg-key.1 b/docs/man/borg-key.1 index f24211c6e2..881fbb70cd 100644 --- a/docs/man/borg-key.1 +++ b/docs/man/borg-key.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY 1 "2017-08-27" "" "borg backup tool" +.TH BORG-KEY 1 "2017-09-09" "" "borg backup tool" .SH NAME borg-key \- Manage a keyfile or repokey of a repository . diff --git a/docs/man/borg-list.1 b/docs/man/borg-list.1 index d26bcaba86..06f2686076 100644 --- a/docs/man/borg-list.1 +++ b/docs/man/borg-list.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-LIST 1 "2017-08-27" "" "borg backup tool" +.TH BORG-LIST 1 "2017-09-09" "" "borg backup tool" .SH NAME borg-list \- List archive or repository contents . diff --git a/docs/man/borg-mount.1 b/docs/man/borg-mount.1 index 0e5d36ee90..1e664a24bf 100644 --- a/docs/man/borg-mount.1 +++ b/docs/man/borg-mount.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-MOUNT 1 "2017-08-27" "" "borg backup tool" +.TH BORG-MOUNT 1 "2017-09-09" "" "borg backup tool" .SH NAME borg-mount \- Mount archive or an entire repository as a FUSE filesystem . diff --git a/docs/man/borg-patterns.1 b/docs/man/borg-patterns.1 index acbfdb4641..3be0c4ba3f 100644 --- a/docs/man/borg-patterns.1 +++ b/docs/man/borg-patterns.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-PATTERNS 1 "2017-08-27" "" "borg backup tool" +.TH BORG-PATTERNS 1 "2017-09-09" "" "borg backup tool" .SH NAME borg-patterns \- Details regarding patterns . diff --git a/docs/man/borg-placeholders.1 b/docs/man/borg-placeholders.1 index 23ec79353c..e8346dbaee 100644 --- a/docs/man/borg-placeholders.1 +++ b/docs/man/borg-placeholders.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-PLACEHOLDERS 1 "2017-08-27" "" "borg backup tool" +.TH BORG-PLACEHOLDERS 1 "2017-09-09" "" "borg backup tool" .SH NAME borg-placeholders \- Details regarding placeholders . diff --git a/docs/man/borg-prune.1 b/docs/man/borg-prune.1 index 92b2cd35fa..e6d46b6fb4 100644 --- a/docs/man/borg-prune.1 +++ b/docs/man/borg-prune.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-PRUNE 1 "2017-08-27" "" "borg backup tool" +.TH BORG-PRUNE 1 "2017-09-09" "" "borg backup tool" .SH NAME borg-prune \- Prune repository archives according to specified rules . diff --git a/docs/man/borg-recreate.1 b/docs/man/borg-recreate.1 index 226e79f1bf..cd90406fa9 100644 --- a/docs/man/borg-recreate.1 +++ b/docs/man/borg-recreate.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-RECREATE 1 "2017-08-27" "" "borg backup tool" +.TH BORG-RECREATE 1 "2017-09-09" "" "borg backup tool" .SH NAME borg-recreate \- Re-create archives . diff --git a/docs/man/borg-rename.1 b/docs/man/borg-rename.1 index a118cd1668..772d96a0ec 100644 --- a/docs/man/borg-rename.1 +++ b/docs/man/borg-rename.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-RENAME 1 "2017-08-27" "" "borg backup tool" +.TH BORG-RENAME 1 "2017-09-09" "" "borg backup tool" .SH NAME borg-rename \- Rename an existing archive . diff --git a/docs/man/borg-serve.1 b/docs/man/borg-serve.1 index 8ad1216df6..262eb56f02 100644 --- a/docs/man/borg-serve.1 +++ b/docs/man/borg-serve.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-SERVE 1 "2017-08-27" "" "borg backup tool" +.TH BORG-SERVE 1 "2017-09-09" "" "borg backup tool" .SH NAME borg-serve \- Start in server mode. This command is usually not used manually. . diff --git a/docs/man/borg-umount.1 b/docs/man/borg-umount.1 index 6f7471ead9..27d4fbf371 100644 --- a/docs/man/borg-umount.1 +++ b/docs/man/borg-umount.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-UMOUNT 1 "2017-08-27" "" "borg backup tool" +.TH BORG-UMOUNT 1 "2017-09-09" "" "borg backup tool" .SH NAME borg-umount \- un-mount the FUSE filesystem . diff --git a/docs/man/borg-upgrade.1 b/docs/man/borg-upgrade.1 index b716f23eea..38b739e3c5 100644 --- a/docs/man/borg-upgrade.1 +++ b/docs/man/borg-upgrade.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-UPGRADE 1 "2017-08-27" "" "borg backup tool" +.TH BORG-UPGRADE 1 "2017-09-09" "" "borg backup tool" .SH NAME borg-upgrade \- upgrade a repository from a previous version . diff --git a/docs/man/borg-with-lock.1 b/docs/man/borg-with-lock.1 index 1ad8ae6b8a..7b0e2d6b7e 100644 --- a/docs/man/borg-with-lock.1 +++ b/docs/man/borg-with-lock.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-WITH-LOCK 1 "2017-08-27" "" "borg backup tool" +.TH BORG-WITH-LOCK 1 "2017-09-09" "" "borg backup tool" .SH NAME borg-with-lock \- run a user specified command with the repository lock held . From f44fd6ba193bac99fb48ebd7264e7724d269f3d0 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 18 Sep 2017 04:42:30 +0200 Subject: [PATCH 045/798] flush json mode progress stderr output if borg stderr is not connected to a tty, but to ssh (when using borg client/server), sys.stderr is block buffered (tty: line buffered). thus we better flush explicitly after emitting a line as the receiving side can not handle partial json at the end of the block. also, it might solve some delays, when output didn't arrive at receiving side in time. (cherry picked from commit 2b75b278dab59b18a64aeb627bfcfc259804e825) --- src/borg/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 0432db2f0e..77c87228aa 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -1444,7 +1444,7 @@ def output_json(self, *, finished=False, **kwargs): finished=finished, time=time.time(), )) - print(json.dumps(kwargs), file=sys.stderr) + print(json.dumps(kwargs), file=sys.stderr, flush=True) def finish(self): if self.json: From 3f6c14e8543f5c9d76077c577af156e73d0a6a2a Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 22 Sep 2017 02:22:39 +0200 Subject: [PATCH 046/798] use python 3.5.4 to build the binaries (cherry picked from commit 507203a759d15d145c055ffbc3e820abe778ae2b) --- Vagrantfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 9b516a9320..0f03ebd0cd 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -233,7 +233,7 @@ def install_pythons(boxname) pyenv install 3.4.0 # tests pyenv install 3.5.0 # tests pyenv install 3.6.0 # tests - pyenv install 3.5.3 # binary build, use latest 3.5.x release + pyenv install 3.5.4 # binary build, use latest 3.5.x release pyenv rehash EOF end @@ -251,8 +251,8 @@ def build_pyenv_venv(boxname) . ~/.bash_profile cd /vagrant/borg # use the latest 3.5 release - pyenv global 3.5.3 - pyenv virtualenv 3.5.3 borg-env + pyenv global 3.5.4 + pyenv virtualenv 3.5.4 borg-env ln -s ~/.pyenv/versions/borg-env . EOF end From ca34a33e7df673b2b2b54994716f1a84566fbce6 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 22 Sep 2017 03:24:44 +0200 Subject: [PATCH 047/798] upgrade to FUSE for macOS 3.7.1 --- Vagrantfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 0f03ebd0cd..ad931c77a3 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -62,9 +62,9 @@ def packages_darwin sudo softwareupdate --ignore iTunes sudo softwareupdate --install --all # get osxfuse 3.x release code from github: - curl -s -L https://github.com/osxfuse/osxfuse/releases/download/osxfuse-3.6.3/osxfuse-3.6.3.dmg >osxfuse.dmg + curl -s -L https://github.com/osxfuse/osxfuse/releases/download/osxfuse-3.7.1/osxfuse-3.7.1.dmg >osxfuse.dmg MOUNTDIR=$(echo `hdiutil mount osxfuse.dmg | tail -1 | awk '{$1="" ; print $0}'` | xargs -0 echo) \ - && sudo installer -pkg "${MOUNTDIR}/Extras/FUSE for macOS 3.6.3.pkg" -target / + && sudo installer -pkg "${MOUNTDIR}/Extras/FUSE for macOS 3.7.1.pkg" -target / sudo chown -R vagrant /usr/local # brew must be able to create stuff here ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" brew update From b6abee4d6852f2b8b569228e45751e1ae8dc96b3 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 22 Sep 2017 22:53:55 +0200 Subject: [PATCH 048/798] docs: change-passphrase only changes the passphrase, fixes #2990 (cherry picked from commit 713be765d1855adfd6dd2949324f5a2251c7f3c2) --- src/borg/archiver.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index f6b177d366..9b67b99d83 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -2717,6 +2717,11 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): change_passphrase_epilog = process_epilog(""" The key files used for repository encryption are optionally passphrase protected. This command can be used to change this passphrase. + + Please note that this command only changes the passphrase, but not any + secret protected by it (like e.g. encryption/MAC keys or chunker seed). + Thus, changing the passphrase after passphrase and borg key got compromised + does not protect future (nor past) backups to the same repository. """) subparser = key_parsers.add_parser('change-passphrase', parents=[common_parser], add_help=False, description=self.do_change_passphrase.__doc__, From eacea02d8c497437af22734982fe219beecefa0e Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 22 Sep 2017 23:08:33 +0200 Subject: [PATCH 049/798] docs: add note about metadata dedup and --no[ac]time, fixes #2518 (cherry picked from commit d8766df998642007d00ea818f2d97766a5aa29a2) --- docs/usage/notes.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/usage/notes.rst b/docs/usage/notes.rst index 85648382e8..c45ef3f8ca 100644 --- a/docs/usage/notes.rst +++ b/docs/usage/notes.rst @@ -50,6 +50,17 @@ a new repository when changing chunker params. For more details, see :ref:`chunker_details`. +``--noatime / --noctime`` +~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can use these ``borg create`` options to not store the respective timestamp +into the archive, in case you do not really need it. + +Besides saving a little space for the not archived timestamp, it might also +affect metadata stream deduplication: if only this timestamp changes between +backups and is stored into the metadata stream, the metadata stream chunks +won't deduplicate just because of that. + ``--umask`` ~~~~~~~~~~~ From 055f34f025f57c754869a9e9dd84161525bf53d9 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 22 Sep 2017 23:36:48 +0200 Subject: [PATCH 050/798] docs: twitter account @borgbackup, fixes #2948 (cherry picked from commit 112bf4395641b7481e92456b4313a6b596ed62a8) --- docs/development.rst | 2 +- docs/support.rst | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/development.rst b/docs/development.rst index 37cc9ad68d..831c8ef2fd 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -340,7 +340,7 @@ Checklist: - announce on: - Mailing list - - Twitter (follow @ThomasJWaldmann for these tweets) + - Twitter - IRC channel (change ``/topic``) - create a Github release, include: diff --git a/docs/support.rst b/docs/support.rst index 293264f51e..8bd3fcbedf 100644 --- a/docs/support.rst +++ b/docs/support.rst @@ -38,6 +38,15 @@ unsubscribe and where you can find the archives of the list, see the `mailing list homepage `_. +Twitter +------- + +Follow @borgbackup for announcements. You can also add @borgbackup if you +would like to get retweeted for a borg related tweet. + +Please understand that Twitter is not suitable for longer / more complex +discussions - use one of the other channels for that. + Bounties and Fundraisers ------------------------ From 354d3f0b4f9952fc0868c253f0b03bf698fe99d9 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 21 Sep 2017 04:26:52 +0200 Subject: [PATCH 051/798] fix docs: --compression lz4 is the default now, fixes #3034 (cherry picked from commit c88528512f146a5e9a144c4f35b940d8d45041a1) --- docs/usage/create.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/usage/create.rst b/docs/usage/create.rst index 8e4a2e0361..4586b30f5c 100644 --- a/docs/usage/create.rst +++ b/docs/usage/create.rst @@ -41,11 +41,11 @@ Examples # Backup a raw device (must not be active/in use/mounted at that time) $ dd if=/dev/sdx bs=10M | borg create /path/to/repo::my-sdx - - # No compression (default) - $ borg create /path/to/repo::arch ~ + # No compression + $ borg create --compression none /path/to/repo::arch ~ - # Super fast, low compression - $ borg create --compression lz4 /path/to/repo::arch ~ + # Super fast, low compression (default) + $ borg create /path/to/repo::arch ~ # Less fast, higher compression (N = 0..9) $ borg create --compression zlib,N /path/to/repo::arch ~ From c616009724801e80edd4e556c31e9fae8202c388 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 21 Sep 2017 04:28:35 +0200 Subject: [PATCH 052/798] docs: add compressor names to be more clear (cherry picked from commit 0ec6c920b9e70a8bcbc6340be9b867d833fb912a) --- docs/usage/create.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/usage/create.rst b/docs/usage/create.rst index 4586b30f5c..6f707c9859 100644 --- a/docs/usage/create.rst +++ b/docs/usage/create.rst @@ -41,16 +41,16 @@ Examples # Backup a raw device (must not be active/in use/mounted at that time) $ dd if=/dev/sdx bs=10M | borg create /path/to/repo::my-sdx - - # No compression + # No compression (none) $ borg create --compression none /path/to/repo::arch ~ - # Super fast, low compression (default) + # Super fast, low compression (lz4, default) $ borg create /path/to/repo::arch ~ - # Less fast, higher compression (N = 0..9) + # Less fast, higher compression (zlib, N = 0..9) $ borg create --compression zlib,N /path/to/repo::arch ~ - # Even slower, even higher compression (N = 0..9) + # Even slower, even higher compression (lzma, N = 0..9) $ borg create --compression lzma,N /path/to/repo::arch ~ # Use short hostname, user name and current time in archive name From cfc48d28992d7c2ff1029681568aaca32dae7d81 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 21 Sep 2017 14:00:50 +0200 Subject: [PATCH 053/798] docs: add auto compression example to borg create examples (cherry picked from commit 3f16d91fd3ff4f83d46978ce858750f1091f45fa) --- docs/usage/create.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/usage/create.rst b/docs/usage/create.rst index 6f707c9859..ec49f5fa83 100644 --- a/docs/usage/create.rst +++ b/docs/usage/create.rst @@ -53,6 +53,9 @@ Examples # Even slower, even higher compression (lzma, N = 0..9) $ borg create --compression lzma,N /path/to/repo::arch ~ + # Only compress compressible data with lzma,N (N = 0..9) + $ borg create --compression auto,lzma,N /path/to/repo::arch ~ + # Use short hostname, user name and current time in archive name $ borg create /path/to/repo::{hostname}-{user}-{now} ~ # Similar, use the same datetime format as borg 1.1 will have as default From f48f86444dfd5cbf3094d9fb7a6fce51bb8981b1 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 21 Sep 2017 04:11:59 +0200 Subject: [PATCH 054/798] remove client_supports_log_v3 flag, fixes #3033 the client_supports_log_v3 flag was added to differentiate 1.1.0 beta3 to beta5 clients (which did not support parsing json log format from server) from >= 1.1.0beta6 clients (which support it). for clients older than 1.1.0b3, no json log format will be negotiated anyway. by removing the client_supports_log_v3 flag support, we drop support for clients using 1.1.0beta3..5. thus, a client is now expected to either support old log format (like borg 1.0.x) or new json format (like borg 1.1.0 >= beta6). client server comment =========================================== any 0.29+ uses $LOG plain remote log format any 1.0.x uses $LOG plain remote log format 1.0.x 1.1.0 uses $LOG plain remote log format 1.1.0b1/b2 1.1.0 (uses $LOG plain remote log format) 1.1.0b3-b5 1.1.0 (malfunction) 1.1.0b6 1.1.0 (uses json remote log format) 1.1.0rc 1.1.0 uses json remote log format 1.1.x 1.1.0 uses json remote log format (beta testing is over and betas are unsupported now) Note: client_supports_log_v3 flag was added in changeset 18a2902c9c0f4ebeeb458f72ed02ba29d1053169 (cherry picked from commit 54c5049fb9159d9695dc21a56bdf96e44657d9db) --- src/borg/remote.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/borg/remote.py b/src/borg/remote.py index a508e04f97..d3f262c408 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -304,12 +304,12 @@ def negotiate(self, client_data): if client_data == RPC_PROTOCOL_VERSION: return RPC_PROTOCOL_VERSION # clients since 1.1.0b3 use a dict as client_data + # clients since 1.1.0b6 support json log format from server if isinstance(client_data, dict): self.client_version = client_data[b'client_version'] - if client_data.get(b'client_supports_log_v3', False): - level = logging.getLevelName(logging.getLogger('').level) - setup_logging(is_serve=True, json=True, level=level) - logger.debug('Initialized logging system for new (v3) protocol') + level = logging.getLevelName(logging.getLogger('').level) + setup_logging(is_serve=True, json=True, level=level) + logger.debug('Initialized logging system for JSON-based protocol') else: self.client_version = BORG_VERSION # seems to be newer than current version (no known old format) @@ -567,7 +567,6 @@ def __init__(self, location, create=False, exclusive=False, lock_wait=None, lock try: version = self.call('negotiate', {'client_data': { b'client_version': BORG_VERSION, - b'client_supports_log_v3': True, }}) except ConnectionClosed: raise ConnectionClosedWithHint('Is borg working on the server?') from None @@ -1012,16 +1011,10 @@ def handle_remote_line(line): # of local progress displays. sys.stderr.write('Remote: ' + msg['message'] + '\r') elif line.startswith('$LOG '): - # This format is used by Borg since 1.1.0b1. + # This format is used by borg serve 0.xx, 1.0.x and 1.1.0b1..b5. # It prefixed log lines with $LOG as a marker, followed by the log level # and optionally a logger name, then "Remote:" as a separator followed by the original # message. - # - # It is the oldest format supported by these servers, so it was important to make - # it readable with older (1.0.x) clients. - # - # TODO: Remove this block (so it'll be handled by the "else:" below) with a Borg 1.1 RC. - # Also check whether client_supports_log_v3 should be removed. _, level, msg = line.split(' ', 2) level = getattr(logging, level, logging.CRITICAL) # str -> int if msg.startswith('Remote:'): From 5dd16672c0e82e1bde24c53a45629527bda77d2a Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 23 Sep 2017 19:04:46 +0200 Subject: [PATCH 055/798] refactor/fix subprocess env preparation refactor: make a generally usable function fix: remove support code for ancient pyinstaller the "else" branch was needed for pyinstaller < 20160820 because it did not have the LD_LIBRARY_PATH_ORIG env var, so we just killed LDLP because we had no better way. but with borg tests running under fakeroot, this is troublesome as fakeroot uses this also and can't find its library without it. so, just remove it, we do not need to support old pyinstaller. (cherry picked from commit ba941b08016e9931eeaef5063a0fdc879d85f234) --- src/borg/helpers.py | 26 ++++++++++++++++++++++++++ src/borg/remote.py | 16 ++++------------ 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 77c87228aa..647a054aa8 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -2303,6 +2303,32 @@ def popen_with_error_handling(cmd_line: str, log_prefix='', **kwargs): return +def prepare_subprocess_env(system, env=None): + """ + Prepare the environment for a subprocess we are going to create. + + :param system: True for preparing to invoke system-installed binaries, + False for stuff inside the pyinstaller environment (like borg, python). + :param env: optionally give a environment dict here. if not given, default to os.environ. + :return: a modified copy of the environment + """ + env = dict(env if env is not None else os.environ) + if system: + # a pyinstaller binary's bootloader modifies LD_LIBRARY_PATH=/tmp/_ME..., + # but we do not want that system binaries (like ssh or other) pick up + # (non-matching) libraries from there. + # thus we install the original LDLP, before pyinstaller has modified it: + lp_key = 'LD_LIBRARY_PATH' + lp_orig = env.get(lp_key + '_ORIG') # pyinstaller >= 20160820 has this + if lp_orig is not None: + env[lp_key] = lp_orig + # security: do not give secrets to subprocess + env.pop('BORG_PASSPHRASE', None) + # for information, give borg version to the subprocess + env['BORG_VERSION'] = borg_version + return env + + def dash_open(path, mode): assert '+' not in mode # the streams are either r or w, but never both if path == '-': diff --git a/src/borg/remote.py b/src/borg/remote.py index d3f262c408..9e15780728 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -30,6 +30,7 @@ from .helpers import sysinfo from .helpers import format_file_size from .helpers import truncate_and_unlink +from .helpers import prepare_subprocess_env from .logger import create_logger, setup_logging from .repository import Repository from .version import parse_version, format_version @@ -537,21 +538,12 @@ def __init__(self, location, create=False, exclusive=False, lock_wait=None, lock self.server_version = parse_version('1.0.8') # fallback version if server is too old to send version information self.p = None testing = location.host == '__testsuite__' + # when testing, we invoke and talk to a borg process directly (no ssh). + # when not testing, we invoke the system-installed ssh binary to talk to a remote borg. + env = prepare_subprocess_env(system=not testing) borg_cmd = self.borg_cmd(args, testing) - env = dict(os.environ) if not testing: borg_cmd = self.ssh_cmd(location) + borg_cmd - # pyinstaller binary modifies LD_LIBRARY_PATH=/tmp/_ME... but we do not want - # that the system's ssh binary picks up (non-matching) libraries from there. - # thus we install the original LDLP, before pyinstaller has modified it: - lp_key = 'LD_LIBRARY_PATH' - lp_orig = env.get(lp_key + '_ORIG') # pyinstaller >= 20160820 has this - if lp_orig is not None: - env[lp_key] = lp_orig - else: - env.pop(lp_key, None) - env.pop('BORG_PASSPHRASE', None) # security: do not give secrets to subprocess - env['BORG_VERSION'] = __version__ logger.debug('SSH command line: %s', borg_cmd) self.p = Popen(borg_cmd, bufsize=0, stdin=PIPE, stdout=PIPE, stderr=PIPE, env=env) self.stdin_fd = self.p.stdin.fileno() From 2f7d7bdbb8db9bb43cb498f7cf129599539135be Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 23 Sep 2017 20:04:47 +0200 Subject: [PATCH 056/798] use prepared env for calling BORG_PASSCOMMAND, fixes #3050 (cherry picked from commit 6a6fd318045ceea254871de8042944875781bb7f) --- src/borg/crypto/key.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/borg/crypto/key.py b/src/borg/crypto/key.py index 02cfed6e7b..58f928023f 100644 --- a/src/borg/crypto/key.py +++ b/src/borg/crypto/key.py @@ -23,6 +23,7 @@ from ..helpers import get_keys_dir, get_security_dir from ..helpers import get_limited_unpacker from ..helpers import bin_to_hex +from ..helpers import prepare_subprocess_env from ..item import Key, EncryptedKey from ..platform import SaveFile from .nonces import NonceManager @@ -431,8 +432,10 @@ def env_passphrase(cls, default=None): def env_passcommand(cls, default=None): passcommand = os.environ.get('BORG_PASSCOMMAND', None) if passcommand is not None: + # passcommand is a system command (not inside pyinstaller env) + env = prepare_subprocess_env(system=True) try: - passphrase = subprocess.check_output(shlex.split(passcommand), universal_newlines=True) + passphrase = subprocess.check_output(shlex.split(passcommand), universal_newlines=True, env=env) except (subprocess.CalledProcessError, FileNotFoundError) as e: raise PasscommandFailure(e) return cls(passphrase.rstrip('\n')) From 3b20c18c3ba754d0e6348ec0fde87a360753b215 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 23 Sep 2017 22:35:10 +0200 Subject: [PATCH 057/798] use prepared env for borg with-lock (cherry picked from commit 6da5bf4b850fe1e197e8e58b46e9a6a9ca4e4770) --- src/borg/archiver.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 9b67b99d83..df10cef599 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -64,7 +64,7 @@ from .helpers import basic_json_data, json_print from .helpers import replace_placeholders from .helpers import ChunkIteratorFileWrapper -from .helpers import popen_with_error_handling +from .helpers import popen_with_error_handling, prepare_subprocess_env from .helpers import dash_open from .helpers import umount from .nanorst import rst_to_terminal @@ -1677,9 +1677,10 @@ def do_with_lock(self, args, repository): # we can only do this for local repositories (with .io), though: if hasattr(repository, 'io'): repository.io.close_segment() + env = prepare_subprocess_env(system=True) try: # we exit with the return code we get from the subprocess - return subprocess.call([args.command] + args.args) + return subprocess.call([args.command] + args.args, env=env) finally: # we need to commit the "no change" operation we did to the manifest # because it created a new segment file in the repository. if we would From d6f810560cd036d2e60fe3485e849dba48010a38 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 23 Sep 2017 22:38:26 +0200 Subject: [PATCH 058/798] use prepared env for borg umount (cherry picked from commit b88da1064155f159163a78cfbafa1adc0eb5ec3a) --- src/borg/helpers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 647a054aa8..54ea00fc84 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -2343,7 +2343,8 @@ def is_terminal(fd=sys.stdout): def umount(mountpoint): + env = prepare_subprocess_env(system=True) try: - return subprocess.call(['fusermount', '-u', mountpoint]) + return subprocess.call(['fusermount', '-u', mountpoint], env=env) except FileNotFoundError: - return subprocess.call(['umount', mountpoint]) + return subprocess.call(['umount', mountpoint], env=env) From d7a5ca2e6a772ae18c5aaaa0fc9fbf8e67b0bbbf Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 23 Sep 2017 22:54:52 +0200 Subject: [PATCH 059/798] use prepared env for borg export-tar --tar-filter subprocess (cherry picked from commit cf59f653e5821ddee7af2c4d4fc8806744fab017) --- src/borg/archiver.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index df10cef599..d899edca55 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -790,10 +790,12 @@ def do_export_tar(self, args, repository, manifest, key, archive): # The decision whether to close that or not remains the same. filterout = tarstream filterout_close = tarstream_close + env = prepare_subprocess_env(system=True) # There is no deadlock potential here (the subprocess docs warn about this), because # communication with the process is a one-way road, i.e. the process can never block # for us to do something while we block on the process for something different. - filterproc = popen_with_error_handling(filter, stdin=subprocess.PIPE, stdout=filterout, log_prefix='--tar-filter: ') + filterproc = popen_with_error_handling(filter, stdin=subprocess.PIPE, stdout=filterout, + log_prefix='--tar-filter: ', env=env) if not filterproc: return EXIT_ERROR # Always close the pipe, otherwise the filter process would not notice when we are done. From 37a5442bef024afd92e2c3156d8becd9ab01385d Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 23 Sep 2017 23:27:14 +0200 Subject: [PATCH 060/798] use prepared env for xattr module's fakeroot version check (cherry picked from commit a57e23fdb35096c40b4849849c3752ae83090f06) --- src/borg/xattr.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/borg/xattr.py b/src/borg/xattr.py index 03c5eae260..c1f3314659 100644 --- a/src/borg/xattr.py +++ b/src/borg/xattr.py @@ -10,7 +10,7 @@ from ctypes.util import find_library from distutils.version import LooseVersion -from .helpers import Buffer +from .helpers import Buffer, prepare_subprocess_env try: @@ -88,7 +88,9 @@ def get_all(path, follow_symlinks=True): preloads = re.split("[ :]", LD_PRELOAD) for preload in preloads: if preload.startswith("libfakeroot"): - fakeroot_version = LooseVersion(subprocess.check_output(['fakeroot', '-v']).decode('ascii').split()[-1]) + env = prepare_subprocess_env(system=True) + fakeroot_output = subprocess.check_output(['fakeroot', '-v'], env=env) + fakeroot_version = LooseVersion(fakeroot_output.decode('ascii').split()[-1]) if fakeroot_version >= LooseVersion("1.20.2"): # 1.20.2 has been confirmed to have xattr support # 1.18.2 has been confirmed not to have xattr support From f2b48cd5c8947806306dbb8a7ee7c98d78a4c729 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 25 Sep 2017 01:15:13 +0200 Subject: [PATCH 061/798] remote: deal with partial lines, fixes #2637 due to block buffering (in borg, pipes, sshd, ssh) partial lines might be received. for plain text, this causes cosmetic issues, for json it causes tracebacks due to parse errors. the code now makes sure handle_remote_line() only gets called with a complete line (which is terminated by any universal newline char, a pure \r seems to be needed for remote progress displays). it also fixes a yet undiscovered partial utf-8-sequence decoding issue that might occur for the same reason. (cherry picked from commit 8646216a062364efb3179285d21884d95b87f1c8) --- src/borg/remote.py | 15 ++++++++++++--- src/borg/testsuite/repository.py | 18 ++++++++++++------ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/borg/remote.py b/src/borg/remote.py index 9e15780728..db811c2760 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -528,6 +528,7 @@ def __init__(self, location, create=False, exclusive=False, lock_wait=None, lock self.rx_bytes = 0 self.tx_bytes = 0 self.to_send = b'' + self.stderr_received = b'' # incomplete stderr line bytes received (no \n yet) self.chunkid_to_msgids = {} self.ignore_responses = set() self.responses = {} @@ -828,9 +829,16 @@ def handle_error(unpacked): if not data: raise ConnectionClosed() self.rx_bytes += len(data) - data = data.decode('utf-8') - for line in data.splitlines(keepends=True): - handle_remote_line(line) + # deal with incomplete lines (may appear due to block buffering) + if self.stderr_received: + data = self.stderr_received + data + self.stderr_received = b'' + lines = data.splitlines(keepends=True) + if lines and not lines[-1].endswith((b'\r', b'\n')): + self.stderr_received = lines.pop() + # now we have complete lines in and any partial line in self.stderr_received. + for line in lines: + handle_remote_line(line.decode('utf-8')) # decode late, avoid partial utf-8 sequences if w: while not self.to_send and (calls or self.preload_ids) and len(waiting_for) < MAX_INFLIGHT: if calls: @@ -961,6 +969,7 @@ def handle_remote_line(line): This function is remarkably complex because it handles multiple wire formats. """ + assert line.endswith(('\r', '\n')) if line.startswith('{'): # This format is used by Borg since 1.1.0b6 for new-protocol clients. # It is the same format that is exposed by --log-json. diff --git a/src/borg/testsuite/repository.py b/src/borg/testsuite/repository.py index 1bec140069..49f2e1f109 100644 --- a/src/borg/testsuite/repository.py +++ b/src/borg/testsuite/repository.py @@ -944,16 +944,22 @@ def tearDown(self): sys.stderr = self.old_stderr def test_stderr_messages(self): - handle_remote_line("unstructured stderr message") + handle_remote_line("unstructured stderr message\n") self.assert_equal(self.stream.getvalue(), '') # stderr messages don't get an implicit newline - self.assert_equal(self.stderr.getvalue(), 'Remote: unstructured stderr message') + self.assert_equal(self.stderr.getvalue(), 'Remote: unstructured stderr message\n') + + def test_stderr_progress_messages(self): + handle_remote_line("unstructured stderr progress message\r") + self.assert_equal(self.stream.getvalue(), '') + # stderr messages don't get an implicit newline + self.assert_equal(self.stderr.getvalue(), 'Remote: unstructured stderr progress message\r') def test_pre11_format_messages(self): self.handler.setLevel(logging.DEBUG) logging.getLogger().setLevel(logging.DEBUG) - handle_remote_line("$LOG INFO Remote: borg < 1.1 format message") + handle_remote_line("$LOG INFO Remote: borg < 1.1 format message\n") self.assert_equal(self.stream.getvalue(), 'Remote: borg < 1.1 format message\n') self.assert_equal(self.stderr.getvalue(), '') @@ -961,7 +967,7 @@ def test_post11_format_messages(self): self.handler.setLevel(logging.DEBUG) logging.getLogger().setLevel(logging.DEBUG) - handle_remote_line("$LOG INFO borg.repository Remote: borg >= 1.1 format message") + handle_remote_line("$LOG INFO borg.repository Remote: borg >= 1.1 format message\n") self.assert_equal(self.stream.getvalue(), 'Remote: borg >= 1.1 format message\n') self.assert_equal(self.stderr.getvalue(), '') @@ -970,7 +976,7 @@ def test_remote_messages_screened(self): self.handler.setLevel(logging.WARNING) logging.getLogger().setLevel(logging.WARNING) - handle_remote_line("$LOG INFO borg.repository Remote: new format info message") + handle_remote_line("$LOG INFO borg.repository Remote: new format info message\n") self.assert_equal(self.stream.getvalue(), '') self.assert_equal(self.stderr.getvalue(), '') @@ -990,7 +996,7 @@ def test_info_to_correct_local_child(self): foo_handler.setLevel(logging.INFO) logging.getLogger('borg.repository.foo').handlers[:] = [foo_handler] - handle_remote_line("$LOG INFO borg.repository Remote: new format child message") + handle_remote_line("$LOG INFO borg.repository Remote: new format child message\n") self.assert_equal(foo_stream.getvalue(), '') self.assert_equal(child_stream.getvalue(), 'Remote: new format child message\n') self.assert_equal(self.stream.getvalue(), '') From d3533de5f7f113dbcf4668524c89a04467947ee8 Mon Sep 17 00:00:00 2001 From: Alexander Meshcheryakov Date: Mon, 25 Sep 2017 15:43:07 +0300 Subject: [PATCH 062/798] Simplified rate limiting wrapper in FAQ Exporting $RATE as environment variable is not need in this case. And example does not use any bash specific features. It should use default system shell instead. (cherry picked from commit f7ec13eabb3751de93bda08003b58e5a76be6f9e) --- docs/faq.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 0cc0444354..fd9c6e50ec 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -624,10 +624,10 @@ can be accomplished with pipeviewer_: Create a wrapper script: /usr/local/bin/pv-wrapper :: - #!/bin/bash + #!/bin/sh ## -q, --quiet do not output any transfer information at all ## -L, --rate-limit RATE limit transfer to RATE bytes per second - export RATE=307200 + RATE=307200 pv -q -L $RATE | "$@" Add BORG_RSH environment variable to use pipeviewer wrapper script with ssh. :: From f170641bd46b0b36aedf2fcc997f92d29e891779 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 29 Sep 2017 01:36:22 +0200 Subject: [PATCH 063/798] exclude Cython 0.27(.0) in requirements, fixes #3066 https://github.com/cython/cython/issues/1880 (cherry picked from commit 7e94d42853b29639169d308853f14d0454a33123) --- requirements.d/development.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.d/development.txt b/requirements.d/development.txt index 5805cb4a11..88535435ae 100644 --- a/requirements.d/development.txt +++ b/requirements.d/development.txt @@ -7,4 +7,4 @@ pytest pytest-xdist pytest-cov pytest-benchmark -Cython +Cython!=0.27 From eab9f5a07b568d41c7f44058530a7c1f60a36aa7 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 11 Sep 2017 02:54:52 +0200 Subject: [PATCH 064/798] implement files cache mode control, fixes #911 You can now control the files cache mode using this option: --files-cache={ctime,mtime,size,inode,rechunk,disabled}* (only some combinations are supported) Previously, only these modes were supported: - mtime,size,inode (default of borg < 1.1.0rc4) - mtime,size (by using --ignore-inode) - disabled (by using --no-files-cache) Now, you additionally get: - ctime alternatively to mtime (more safe), e.g.: ctime,size,inode (this is the new default of borg >= 1.1.0rc4) - rechunk (consider all files as changed, rechunk them) Deprecated: - --ignore-inodes (use modes without "inode") - --no-files-cache (use "disabled" mode) The tests needed some changes: - previously, we use os.utime() to set a files mtime (atime) to specific values, but that does not work for ctime. - now use time.sleep() to create the "latest file" that usually does not end up in the files cache (see FAQ) (cherry picked from commit 5e2de8ba67ded568fcedf9812669e5e1e7c1c90b) --- docs/internals/data-structures.rst | 2 +- src/borg/archive.py | 6 +-- src/borg/archiver.py | 59 ++++++++++++++++++------ src/borg/cache.py | 73 +++++++++++++++++------------- src/borg/constants.py | 4 ++ src/borg/helpers.py | 16 +++++++ src/borg/testsuite/archiver.py | 61 +++++++++++++++++++------ 7 files changed, 158 insertions(+), 63 deletions(-) diff --git a/docs/internals/data-structures.rst b/docs/internals/data-structures.rst index dd7eaefd2f..faa1736fb5 100644 --- a/docs/internals/data-structures.rst +++ b/docs/internals/data-structures.rst @@ -735,7 +735,7 @@ b) with ``create --chunker-params 19,23,21,4095`` (default): mem_usage = 0.31GiB -.. note:: There is also the ``--no-files-cache`` option to switch off the files cache. +.. note:: There is also the ``--files-cache=disabled`` option to disable the files cache. You'll save some memory, but it will need to read / chunk all the files as it can not skip unmodified files then. diff --git a/src/borg/archive.py b/src/borg/archive.py index 6d78aff413..4366419cf5 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -953,13 +953,13 @@ def process_stdin(self, path, cache): self.add_item(item) return 'i' # stdin - def process_file(self, path, st, cache, ignore_inode=False): + def process_file(self, path, st, cache, ignore_inode=False, files_cache_mode=DEFAULT_FILES_CACHE_MODE): with self.create_helper(path, st, None) as (item, status, hardlinked, hardlink_master): # no status yet is_special_file = is_special(st.st_mode) if not hardlinked or hardlink_master: if not is_special_file: path_hash = self.key.id_hash(safe_encode(os.path.join(self.cwd, path))) - ids = cache.file_known_and_unchanged(path_hash, st, ignore_inode) + ids = cache.file_known_and_unchanged(path_hash, st, ignore_inode, files_cache_mode) else: # in --read-special mode, we may be called for special files. # there should be no information in the cache about special files processed in @@ -992,7 +992,7 @@ def process_file(self, path, st, cache, ignore_inode=False): if not is_special_file: # we must not memorize special files, because the contents of e.g. a # block or char device will change without its mtime/size/inode changing. - cache.memorize_file(path_hash, st, [c.id for c in item.chunks]) + cache.memorize_file(path_hash, st, [c.id for c in item.chunks], files_cache_mode) status = status or 'M' # regular file, modified (if not 'A' already) self.stats.nfiles += 1 item.update(self.stat_attrs(st, path)) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index d899edca55..7797c65c39 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -45,7 +45,7 @@ from .helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR from .helpers import Error, NoManifestError, set_ec from .helpers import positive_int_validator, location_validator, archivename_validator, ChunkerParams -from .helpers import PrefixSpec, SortBySpec, HUMAN_SORT_KEYS +from .helpers import PrefixSpec, SortBySpec, HUMAN_SORT_KEYS, FilesCacheMode from .helpers import BaseFormatter, ItemFormatter, ArchiveFormatter from .helpers import format_timedelta, format_file_size, parse_file_size, format_archive from .helpers import safe_encode, remove_surrogates, bin_to_hex, prepare_dump_dict @@ -373,7 +373,7 @@ def measurement_run(repo, path): compression = '--compression=none' # measure create perf (without files cache to always have it chunking) t_start = time.monotonic() - rc = self.do_create(self.parse_args(['create', compression, '--no-files-cache', archive + '1', path])) + rc = self.do_create(self.parse_args(['create', compression, '--files-cache=disabled', archive + '1', path])) t_end = time.monotonic() dt_create = t_end - t_start assert rc == 0 @@ -512,6 +512,7 @@ def create_inner(archive, cache): self.output_filter = args.output_filter self.output_list = args.output_list self.ignore_inode = args.ignore_inode + self.files_cache_mode = args.files_cache_mode dry_run = args.dry_run t0 = datetime.utcnow() t0_monotonic = time.monotonic() @@ -567,7 +568,7 @@ def _process(self, archive, cache, matcher, exclude_caches, exclude_if_present, return if stat.S_ISREG(st.st_mode): if not dry_run: - status = archive.process_file(path, st, cache, self.ignore_inode) + status = archive.process_file(path, st, cache, self.ignore_inode, self.files_cache_mode) elif stat.S_ISDIR(st.st_mode): if recurse: tag_paths = dir_is_tagged(path, exclude_caches, exclude_if_present) @@ -2157,14 +2158,17 @@ def do_subcommand_help(self, parser, args): def preprocess_args(self, args): deprecations = [ - # ('--old', '--new', 'Warning: "--old" has been deprecated. Use "--new" instead.'), + # ('--old', '--new' or None, 'Warning: "--old" has been deprecated. Use "--new" instead.'), ('--list-format', '--format', 'Warning: "--list-format" has been deprecated. Use "--format" instead.'), ('--keep-tag-files', '--keep-exclude-tags', 'Warning: "--keep-tag-files" has been deprecated. Use "--keep-exclude-tags" instead.'), + ('--ignore-inode', None, 'Warning: "--ignore-inode" has been deprecated. Use "--files-cache=ctime,size" or "...=mtime,size" instead.'), + ('--no-files-cache', None, 'Warning: "--no-files-cache" has been deprecated. Use "--files-cache=disabled" instead.'), ] for i, arg in enumerate(args[:]): for old_name, new_name, warning in deprecations: if arg.startswith(old_name): - args[i] = arg.replace(old_name, new_name) + if new_name is not None: + args[i] = arg.replace(old_name, new_name) print(warning, file=sys.stderr) return args @@ -2792,13 +2796,39 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): {now}, {utcnow}, {fqdn}, {hostname}, {user} and some others. Backup speed is increased by not reprocessing files that are already part of - existing archives and weren't modified. Normally, detecting file modifications - will take inode information into consideration. This is problematic for files - located on sshfs and similar network file systems which do not provide stable - inode numbers, such files will always be considered modified. The - ``--ignore-inode`` flag can be used to prevent this and improve performance. - This flag will reduce reliability of change detection however, with files - considered unmodified as long as their size and modification time are unchanged. + existing archives and weren't modified. The detection of unmodified files is + done by comparing multiple file metadata values with previous values kept in + the files cache. + + This comparison can operate in different modes as given by ``--files-cache``: + + - ctime,size,inode (default) + - mtime,size,inode (default behaviour of borg versions older than 1.1.0rc4) + - ctime,size (ignore the inode number) + - mtime,size (ignore the inode number) + - rechunk,ctime (all files are considered modified - rechunk, cache ctime) + - rechunk,mtime (all files are considered modified - rechunk, cache mtime) + - disabled (disable the files cache, all files considered modified - rechunk) + + inode number: better safety, but often unstable on network filesystems + + Normally, detecting file modifications will take inode information into + consideration to improve the reliability of file change detection. + This is problematic for files located on sshfs and similar network file + systems which do not provide stable inode numbers, such files will always + be considered modified. You can use modes without `inode` in this case to + improve performance, but reliability of change detection might be reduced. + + ctime vs. mtime: safety vs. speed + + - ctime is a rather safe way to detect changes to a file (metadata and contents) + as it can not be set from userspace. But, a metadata-only change will already + update the ctime, so there might be some unnecessary chunking/hashing even + without content changes. Some filesystems do not support ctime (change time). + - mtime usually works and only updates if file contents were changed. But mtime + can be arbitrarily set from userspace, e.g. to set mtime back to the same value + it had before a content change happened. This can be used maliciously as well as + well-meant, but in both cases mtime based cache modes can be problematic. The mount points of filesystems or filesystem snapshots should be the same for every creation of a new archive to ensure fast operation. This is because the file cache that @@ -2889,7 +2919,7 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): subparser.add_argument('--json', action='store_true', help='output stats as JSON. Implies ``--stats``.') subparser.add_argument('--no-cache-sync', dest='no_cache_sync', action='store_true', - help='experimental: do not synchronize the cache. Implies ``--no-files-cache``.') + help='experimental: do not synchronize the cache. Implies not using the files cache.') define_exclusion_group(subparser, tag_files=True) @@ -2904,6 +2934,9 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): help='do not store ctime into archive') fs_group.add_argument('--ignore-inode', dest='ignore_inode', action='store_true', help='ignore inode data in the file metadata cache used to detect unchanged files.') + fs_group.add_argument('--files-cache', metavar='MODE', dest='files_cache_mode', + type=FilesCacheMode, default=DEFAULT_FILES_CACHE_MODE_UI, + help='operate files cache in MODE. default: %s' % DEFAULT_FILES_CACHE_MODE_UI) fs_group.add_argument('--read-special', dest='read_special', action='store_true', help='open and read block and char device files as well as FIFOs as if they were ' 'regular files. Also follows symlinks pointing to these kinds of files.') diff --git a/src/borg/cache.py b/src/borg/cache.py index b1b440ac08..3d8883797a 100644 --- a/src/borg/cache.py +++ b/src/borg/cache.py @@ -12,7 +12,7 @@ logger = create_logger() -from .constants import CACHE_README +from .constants import CACHE_README, DEFAULT_FILES_CACHE_MODE from .hashindex import ChunkIndex, ChunkIndexEntry, CacheSynchronizer from .helpers import Location from .helpers import Error @@ -34,7 +34,8 @@ from .remote import cache_if_remote from .repository import LIST_SCAN_LIMIT -FileCacheEntry = namedtuple('FileCacheEntry', 'age inode size mtime chunk_ids') +# note: cmtime might me either a ctime or a mtime timestamp +FileCacheEntry = namedtuple('FileCacheEntry', 'age inode size cmtime chunk_ids') class SecurityManager: @@ -492,7 +493,7 @@ def close(self): def _read_files(self): self.files = {} - self._newest_mtime = None + self._newest_cmtime = None logger.debug('Reading files cache ...') with IntegrityCheckedFile(path=os.path.join(self.path, 'files'), write=False, @@ -538,18 +539,18 @@ def commit(self): self.security_manager.save(self.manifest, self.key) pi = ProgressIndicatorMessage(msgid='cache.commit') if self.files is not None: - if self._newest_mtime is None: + if self._newest_cmtime is None: # was never set because no files were modified/added - self._newest_mtime = 2 ** 63 - 1 # nanoseconds, good until y2262 + self._newest_cmtime = 2 ** 63 - 1 # nanoseconds, good until y2262 ttl = int(os.environ.get('BORG_FILES_CACHE_TTL', 20)) pi.output('Saving files cache') with IntegrityCheckedFile(path=os.path.join(self.path, 'files'), write=True) as fd: for path_hash, item in self.files.items(): - # Only keep files seen in this backup that are older than newest mtime seen in this backup - - # this is to avoid issues with filesystem snapshots and mtime granularity. + # Only keep files seen in this backup that are older than newest cmtime seen in this backup - + # this is to avoid issues with filesystem snapshots and cmtime granularity. # Also keep files from older backups that have not reached BORG_FILES_CACHE_TTL yet. entry = FileCacheEntry(*msgpack.unpackb(item)) - if entry.age == 0 and bigint_to_int(entry.mtime) < self._newest_mtime or \ + if entry.age == 0 and bigint_to_int(entry.cmtime) < self._newest_cmtime or \ entry.age > 0 and entry.age < ttl: msgpack.pack((path_hash, entry), fd) self.cache_config.integrity['files'] = fd.integrity_data @@ -902,37 +903,47 @@ def chunk_decref(self, id, stats, wait=True): else: stats.update(-size, -csize, False) - def file_known_and_unchanged(self, path_hash, st, ignore_inode=False): - if not (self.do_files and stat.S_ISREG(st.st_mode)): + def file_known_and_unchanged(self, path_hash, st, ignore_inode=False, cache_mode=DEFAULT_FILES_CACHE_MODE): + if 'd' in cache_mode or not self.do_files or not stat.S_ISREG(st.st_mode): # d(isabled) return None if self.files is None: self._read_files() + if 'r' in cache_mode: # r(echunk) + return None entry = self.files.get(path_hash) if not entry: return None entry = FileCacheEntry(*msgpack.unpackb(entry)) - if (entry.size == st.st_size and bigint_to_int(entry.mtime) == st.st_mtime_ns and - (ignore_inode or entry.inode == st.st_ino)): - # we ignored the inode number in the comparison above or it is still same. - # if it is still the same, replacing it in the tuple doesn't change it. - # if we ignored it, a reason for doing that is that files were moved to a new - # disk / new fs (so a one-time change of inode number is expected) and we wanted - # to avoid everything getting chunked again. to be able to re-enable the inode - # number comparison in a future backup run (and avoid chunking everything - # again at that time), we need to update the inode number in the cache with what - # we see in the filesystem. - self.files[path_hash] = msgpack.packb(entry._replace(inode=st.st_ino, age=0)) - return entry.chunk_ids - else: + if 's' in cache_mode and entry.size != st.st_size: return None - - def memorize_file(self, path_hash, st, ids): - if not (self.do_files and stat.S_ISREG(st.st_mode)): + if 'i' in cache_mode and not ignore_inode and entry.inode != st.st_ino: + return None + if 'c' in cache_mode and bigint_to_int(entry.cmtime) != st.st_ctime_ns: + return None + elif 'm' in cache_mode and bigint_to_int(entry.cmtime) != st.st_mtime_ns: + return None + # we ignored the inode number in the comparison above or it is still same. + # if it is still the same, replacing it in the tuple doesn't change it. + # if we ignored it, a reason for doing that is that files were moved to a new + # disk / new fs (so a one-time change of inode number is expected) and we wanted + # to avoid everything getting chunked again. to be able to re-enable the inode + # number comparison in a future backup run (and avoid chunking everything + # again at that time), we need to update the inode number in the cache with what + # we see in the filesystem. + self.files[path_hash] = msgpack.packb(entry._replace(inode=st.st_ino, age=0)) + return entry.chunk_ids + + def memorize_file(self, path_hash, st, ids, cache_mode=DEFAULT_FILES_CACHE_MODE): + # note: r(echunk) modes will update the files cache, d(isabled) mode won't + if 'd' in cache_mode or not self.do_files or not stat.S_ISREG(st.st_mode): return - mtime_ns = safe_ns(st.st_mtime_ns) - entry = FileCacheEntry(age=0, inode=st.st_ino, size=st.st_size, mtime=int_to_bigint(mtime_ns), chunk_ids=ids) + if 'c' in cache_mode: + cmtime_ns = safe_ns(st.st_ctime_ns) + elif 'm' in cache_mode: + cmtime_ns = safe_ns(st.st_mtime_ns) + entry = FileCacheEntry(age=0, inode=st.st_ino, size=st.st_size, cmtime=int_to_bigint(cmtime_ns), chunk_ids=ids) self.files[path_hash] = msgpack.packb(entry) - self._newest_mtime = max(self._newest_mtime or 0, mtime_ns) + self._newest_cmtime = max(self._newest_cmtime or 0, cmtime_ns) class AdHocCache(CacheStatsMixin): @@ -973,10 +984,10 @@ def __exit__(self, exc_type, exc_val, exc_tb): files = None do_files = False - def file_known_and_unchanged(self, path_hash, st, ignore_inode=False): + def file_known_and_unchanged(self, path_hash, st, ignore_inode=False, cache_mode=DEFAULT_FILES_CACHE_MODE): return None - def memorize_file(self, path_hash, st, ids): + def memorize_file(self, path_hash, st, ids, cache_mode=DEFAULT_FILES_CACHE_MODE): pass def add_chunk(self, id, chunk, stats, overwrite=False, wait=True): diff --git a/src/borg/constants.py b/src/borg/constants.py index 36d6ee45bc..4f2a430da7 100644 --- a/src/borg/constants.py +++ b/src/borg/constants.py @@ -60,6 +60,10 @@ # chunker params for the items metadata stream, finer granularity ITEMS_CHUNKER_PARAMS = (15, 19, 17, HASH_WINDOW_SIZE) +# operating mode of the files cache (for fast skipping of unchanged files) +DEFAULT_FILES_CACHE_MODE_UI = 'ctime,size,inode' +DEFAULT_FILES_CACHE_MODE = 'cis' # == CacheMode(DEFAULT_FILES_CACHE_MODE_UI) + # return codes returned by borg command # when borg is killed by signal N, rc = 128 + N EXIT_SUCCESS = 0 # everything done, no problems diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 54ea00fc84..6b66c786b8 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -561,6 +561,22 @@ def ChunkerParams(s): return int(chunk_min), int(chunk_max), int(chunk_mask), int(window_size) +def FilesCacheMode(s): + ENTRIES_MAP = dict(ctime='c', mtime='m', size='s', inode='i', rechunk='r', disabled='d') + VALID_MODES = ('cis', 'ims', 'cs', 'ms', 'cr', 'mr', 'd') # letters in alpha order + entries = set(s.strip().split(',')) + if not entries <= set(ENTRIES_MAP): + raise ValueError('cache mode must be a comma-separated list of: %s' % ','.join(sorted(ENTRIES_MAP))) + short_entries = {ENTRIES_MAP[entry] for entry in entries} + mode = ''.join(sorted(short_entries)) + if mode not in VALID_MODES: + raise ValueError('cache mode short must be one of: %s' % ','.join(VALID_MODES)) + return mode + + +assert FilesCacheMode(DEFAULT_FILES_CACHE_MODE_UI) == DEFAULT_FILES_CACHE_MODE # keep these 2 values in sync! + + def dir_is_cachedir(path): """Determines whether the specified path is a cache directory (and therefore should potentially be excluded from the backup) according to diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 1fea6df93f..164c2226e7 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -318,8 +318,6 @@ def create_test_files(self): """Create a minimal test case including all supported file types """ # File - self.create_regular_file('empty', size=0) - os.utime('input/empty', (MAX_S, MAX_S)) self.create_regular_file('file1', size=1024 * 80) self.create_regular_file('flagfile', size=1024) # Directory @@ -370,6 +368,8 @@ def create_test_files(self): if e.errno not in (errno.EINVAL, errno.ENOSYS): raise have_root = False + time.sleep(1) # "empty" must have newer timestamp than other files + self.create_regular_file('empty', size=0) return have_root @@ -1591,9 +1591,8 @@ def test_file_status(self): """test that various file status show expected results clearly incomplete: only tests for the weird "unchanged" status for now""" - now = time.time() self.create_regular_file('file1', size=1024 * 80) - os.utime('input/file1', (now - 5, now - 5)) # 5 seconds ago + time.sleep(1) # file2 must have newer timestamps than file1 self.create_regular_file('file2', size=1024 * 80) self.cmd('init', '--encryption=repokey', self.repository_location) output = self.cmd('create', '--list', self.repository_location + '::test', 'input') @@ -1606,12 +1605,51 @@ def test_file_status(self): # https://borgbackup.readthedocs.org/en/latest/faq.html#i-am-seeing-a-added-status-for-a-unchanged-file self.assert_in("A input/file2", output) + def test_file_status_cs_cache_mode(self): + """test that a changed file with faked "previous" mtime still gets backed up in ctime,size cache_mode""" + self.create_regular_file('file1', contents=b'123') + time.sleep(1) # file2 must have newer timestamps than file1 + self.create_regular_file('file2', size=10) + self.cmd('init', '--encryption=repokey', self.repository_location) + output = self.cmd('create', '--list', '--files-cache=ctime,size', self.repository_location + '::test1', 'input') + # modify file1, but cheat with the mtime (and atime) and also keep same size: + st = os.stat('input/file1') + self.create_regular_file('file1', contents=b'321') + os.utime('input/file1', ns=(st.st_atime_ns, st.st_mtime_ns)) + # this mode uses ctime for change detection, so it should find file1 as modified + output = self.cmd('create', '--list', '--files-cache=ctime,size', self.repository_location + '::test2', 'input') + self.assert_in("A input/file1", output) + + def test_file_status_ms_cache_mode(self): + """test that a chmod'ed file with no content changes does not get chunked again in mtime,size cache_mode""" + self.create_regular_file('file1', size=10) + time.sleep(1) # file2 must have newer timestamps than file1 + self.create_regular_file('file2', size=10) + self.cmd('init', '--encryption=repokey', self.repository_location) + output = self.cmd('create', '--list', '--files-cache=mtime,size', self.repository_location + '::test1', 'input') + # change mode of file1, no content change: + st = os.stat('input/file1') + os.chmod('input/file1', st.st_mode ^ stat.S_IRWXO) # this triggers a ctime change, but mtime is unchanged + # this mode uses mtime for change detection, so it should find file1 as unmodified + output = self.cmd('create', '--list', '--files-cache=mtime,size', self.repository_location + '::test2', 'input') + self.assert_in("U input/file1", output) + + def test_file_status_rc_cache_mode(self): + """test that files get rechunked unconditionally in rechunk,ctime cache mode""" + self.create_regular_file('file1', size=10) + time.sleep(1) # file2 must have newer timestamps than file1 + self.create_regular_file('file2', size=10) + self.cmd('init', '--encryption=repokey', self.repository_location) + output = self.cmd('create', '--list', '--files-cache=rechunk,ctime', self.repository_location + '::test1', 'input') + # no changes here, but this mode rechunks unconditionally + output = self.cmd('create', '--list', '--files-cache=rechunk,ctime', self.repository_location + '::test2', 'input') + self.assert_in("A input/file1", output) + def test_file_status_excluded(self): """test that excluded paths are listed""" - now = time.time() self.create_regular_file('file1', size=1024 * 80) - os.utime('input/file1', (now - 5, now - 5)) # 5 seconds ago + time.sleep(1) # file2 must have newer timestamps than file1 self.create_regular_file('file2', size=1024 * 80) if has_lchflags: self.create_regular_file('file3', size=1024 * 80) @@ -1647,9 +1685,8 @@ def test_create_json(self): assert 'stats' in archive def test_create_topical(self): - now = time.time() self.create_regular_file('file1', size=1024 * 80) - os.utime('input/file1', (now-5, now-5)) + time.sleep(1) # file2 must have newer timestamps than file1 self.create_regular_file('file2', size=1024 * 80) self.cmd('init', '--encryption=repokey', self.repository_location) # no listing by default @@ -2363,7 +2400,7 @@ def test_recreate_rechunkify(self): fd.write(b'b' * 280) self.cmd('init', '--encryption=repokey', self.repository_location) self.cmd('create', '--chunker-params', '7,9,8,128', self.repository_location + '::test1', 'input') - self.cmd('create', self.repository_location + '::test2', 'input', '--no-files-cache') + self.cmd('create', self.repository_location + '::test2', 'input', '--files-cache=disabled') list = self.cmd('list', self.repository_location + '::test1', 'input/large_file', '--format', '{num_chunks} {unique_chunks}') num_chunks, unique_chunks = map(int, list.split(' ')) @@ -3513,7 +3550,6 @@ def define_common_options(add_common_option): add_common_option('-p', '--progress', dest='progress', action='store_true', help='foo') add_common_option('--lock-wait', dest='lock_wait', type=int, metavar='N', default=1, help='(default: %(default)d).') - add_common_option('--no-files-cache', dest='no_files_cache', action='store_false', help='foo') @pytest.fixture def basic_parser(self): @@ -3555,7 +3591,6 @@ def parse_vars_from_line(*line): def test_simple(self, parse_vars_from_line): assert parse_vars_from_line('--error') == { - 'no_files_cache': True, 'append': [], 'lock_wait': 1, 'log_level': 'error', @@ -3563,7 +3598,6 @@ def test_simple(self, parse_vars_from_line): } assert parse_vars_from_line('--error', 'subcommand', '--critical') == { - 'no_files_cache': True, 'append': [], 'lock_wait': 1, 'log_level': 'critical', @@ -3576,7 +3610,6 @@ def test_simple(self, parse_vars_from_line): parse_vars_from_line('--append-only', 'subcommand') assert parse_vars_from_line('--append=foo', '--append', 'bar', 'subcommand', '--append', 'baz') == { - 'no_files_cache': True, 'append': ['foo', 'bar', 'baz'], 'lock_wait': 1, 'log_level': 'warning', @@ -3589,7 +3622,6 @@ def test_simple(self, parse_vars_from_line): @pytest.mark.parametrize('flag,args_key,args_value', ( ('-p', 'progress', True), ('--lock-wait=3', 'lock_wait', 3), - ('--no-files-cache', 'no_files_cache', False), )) def test_flag_position_independence(self, parse_vars_from_line, position, flag, args_key, args_value): line = [] @@ -3600,7 +3632,6 @@ def test_flag_position_independence(self, parse_vars_from_line, position, flag, line.append(flag) result = { - 'no_files_cache': True, 'append': [], 'lock_wait': 1, 'log_level': 'warning', From 9723f388552377ad9b0590bc070fe2b009961a11 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 28 Sep 2017 05:08:30 +0200 Subject: [PATCH 065/798] update CHANGES (1.1-maint) --- docs/changes.rst | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/docs/changes.rst b/docs/changes.rst index 6cec0ca206..eff07f1846 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -131,6 +131,50 @@ The best check that everything is ok is to run a dry-run extraction:: Changelog ========= + +Version 1.1.0rc4 (2017-10-01) +----------------------------- + +Compatibility notes: + +- A borg server >= 1.1.0rc4 does not support borg clients 1.1.0b3-b5. #3033 +- The files cache is now controlled differently and has a new default mode: + + - the files cache now uses ctime by default for improved file change + detection safety. You can still use mtime for more speed and less safety. + - --ignore-inode is deprecated (use --files-cache=... without "inode") + - --no-files-cache is deprecated (use --files-cache=disabled) + +New features: + +- --files-cache - implement files cache mode control, #911 + You can now control the files cache mode using this option: + --files-cache={ctime,mtime,size,inode,rechunk,disabled} + (only some combinations are supported). See the docs for details. + +Fixes: + +- remote progress/logging: deal with partial lines, #2637 +- remote progress: flush json mode output +- fix subprocess environments, #3050 (and more) + +Other changes: + +- remove client_supports_log_v3 flag, #3033 +- exclude broken Cython 0.27(.0) in requirements, #3066 +- vagrant: + + - upgrade to FUSE for macOS 3.7.1 + - use Python 3.5.4 to build the binaries +- docs: + + - security: change-passphrase only changes the passphrase, #2990 + - fixed/improved borg create --compression examples, #3034 + - add note about metadata dedup and --no[ac]time, #2518 + - twitter account @borgbackup now, better visible, #2948 + - simplified rate limiting wrapper in FAQ + + Version 1.1.0rc3 (2017-09-10) ----------------------------- From 0263c9dd8a3b3b1c0915b983f7da7bf421d06e8e Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 1 Oct 2017 02:24:54 +0200 Subject: [PATCH 066/798] build_usage --- docs/usage/change-passphrase.rst.inc | 7 +++- docs/usage/create.rst.inc | 47 +++++++++++++++++++----- docs/usage/key_change-passphrase.rst.inc | 7 +++- 3 files changed, 50 insertions(+), 11 deletions(-) diff --git a/docs/usage/change-passphrase.rst.inc b/docs/usage/change-passphrase.rst.inc index 1114d35039..44edba19ba 100644 --- a/docs/usage/change-passphrase.rst.inc +++ b/docs/usage/change-passphrase.rst.inc @@ -43,4 +43,9 @@ Description ~~~~~~~~~~~ The key files used for repository encryption are optionally passphrase -protected. This command can be used to change this passphrase. \ No newline at end of file +protected. This command can be used to change this passphrase. + +Please note that this command only changes the passphrase, but not any +secret protected by it (like e.g. encryption/MAC keys or chunker seed). +Thus, changing the passphrase after passphrase and borg key got compromised +does not protect future (nor past) backups to the same repository. \ No newline at end of file diff --git a/docs/usage/create.rst.inc b/docs/usage/create.rst.inc index 3fc59dff4d..444e4e2e94 100644 --- a/docs/usage/create.rst.inc +++ b/docs/usage/create.rst.inc @@ -31,7 +31,7 @@ borg create +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``--json`` | output stats as JSON. Implies ``--stats``. | +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------+ - | | ``--no-cache-sync`` | experimental: do not synchronize the cache. Implies ``--no-files-cache``. | + | | ``--no-cache-sync`` | experimental: do not synchronize the cache. Implies not using the files cache. | +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------+ | .. class:: borg-common-opt-ref | | | @@ -65,6 +65,8 @@ borg create +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``--ignore-inode`` | ignore inode data in the file metadata cache used to detect unchanged files. | +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``--files-cache MODE`` | operate files cache in MODE. default: ctime,size,inode | + +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``--read-special`` | open and read block and char device files as well as FIFOs as if they were regular files. Also follows symlinks pointing to these kinds of files. | +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------+ | **Archive options** | @@ -102,7 +104,7 @@ borg create --list output verbose list of items (files, dirs, ...) --filter STATUSCHARS only display items with the given status characters (see description) --json output stats as JSON. Implies ``--stats``. - --no-cache-sync experimental: do not synchronize the cache. Implies ``--no-files-cache``. + --no-cache-sync experimental: do not synchronize the cache. Implies not using the files cache. :ref:`common_options` @@ -124,6 +126,7 @@ borg create --noatime do not store atime into archive --noctime do not store ctime into archive --ignore-inode ignore inode data in the file metadata cache used to detect unchanged files. + --files-cache MODE operate files cache in MODE. default: ctime,size,inode --read-special open and read block and char device files as well as FIFOs as if they were regular files. Also follows symlinks pointing to these kinds of files. @@ -157,13 +160,39 @@ In the archive name, you may use the following placeholders: {now}, {utcnow}, {fqdn}, {hostname}, {user} and some others. Backup speed is increased by not reprocessing files that are already part of -existing archives and weren't modified. Normally, detecting file modifications -will take inode information into consideration. This is problematic for files -located on sshfs and similar network file systems which do not provide stable -inode numbers, such files will always be considered modified. The -``--ignore-inode`` flag can be used to prevent this and improve performance. -This flag will reduce reliability of change detection however, with files -considered unmodified as long as their size and modification time are unchanged. +existing archives and weren't modified. The detection of unmodified files is +done by comparing multiple file metadata values with previous values kept in +the files cache. + +This comparison can operate in different modes as given by ``--files-cache``: + +- ctime,size,inode (default) +- mtime,size,inode (default behaviour of borg versions older than 1.1.0rc4) +- ctime,size (ignore the inode number) +- mtime,size (ignore the inode number) +- rechunk,ctime (all files are considered modified - rechunk, cache ctime) +- rechunk,mtime (all files are considered modified - rechunk, cache mtime) +- disabled (disable the files cache, all files considered modified - rechunk) + +inode number: better safety, but often unstable on network filesystems + +Normally, detecting file modifications will take inode information into +consideration to improve the reliability of file change detection. +This is problematic for files located on sshfs and similar network file +systems which do not provide stable inode numbers, such files will always +be considered modified. You can use modes without `inode` in this case to +improve performance, but reliability of change detection might be reduced. + +ctime vs. mtime: safety vs. speed + +- ctime is a rather safe way to detect changes to a file (metadata and contents) + as it can not be set from userspace. But, a metadata-only change will already + update the ctime, so there might be some unnecessary chunking/hashing even + without content changes. Some filesystems do not support ctime (change time). +- mtime usually works and only updates if file contents were changed. But mtime + can be arbitrarily set from userspace, e.g. to set mtime back to the same value + it had before a content change happened. This can be used maliciously as well as + well-meant, but in both cases mtime based cache modes can be problematic. The mount points of filesystems or filesystem snapshots should be the same for every creation of a new archive to ensure fast operation. This is because the file cache that diff --git a/docs/usage/key_change-passphrase.rst.inc b/docs/usage/key_change-passphrase.rst.inc index 64bc409c52..ab31b9fdeb 100644 --- a/docs/usage/key_change-passphrase.rst.inc +++ b/docs/usage/key_change-passphrase.rst.inc @@ -43,4 +43,9 @@ Description ~~~~~~~~~~~ The key files used for repository encryption are optionally passphrase -protected. This command can be used to change this passphrase. \ No newline at end of file +protected. This command can be used to change this passphrase. + +Please note that this command only changes the passphrase, but not any +secret protected by it (like e.g. encryption/MAC keys or chunker seed). +Thus, changing the passphrase after passphrase and borg key got compromised +does not protect future (nor past) backups to the same repository. \ No newline at end of file From 180a37db461965fcb3e619fb1a12a8e7fdda40d8 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 1 Oct 2017 02:26:55 +0200 Subject: [PATCH 067/798] build_man --- docs/man/borg-benchmark-crud.1 | 2 +- docs/man/borg-benchmark.1 | 2 +- docs/man/borg-break-lock.1 | 2 +- docs/man/borg-change-passphrase.1 | 7 ++- docs/man/borg-check.1 | 2 +- docs/man/borg-common.1 | 2 +- docs/man/borg-compression.1 | 2 +- docs/man/borg-create.1 | 73 ++++++++++++++++++++------ docs/man/borg-delete.1 | 2 +- docs/man/borg-diff.1 | 2 +- docs/man/borg-export-tar.1 | 2 +- docs/man/borg-extract.1 | 2 +- docs/man/borg-info.1 | 2 +- docs/man/borg-init.1 | 2 +- docs/man/borg-key-change-passphrase.1 | 7 ++- docs/man/borg-key-export.1 | 2 +- docs/man/borg-key-import.1 | 2 +- docs/man/borg-key-migrate-to-repokey.1 | 2 +- docs/man/borg-key.1 | 2 +- docs/man/borg-list.1 | 2 +- docs/man/borg-mount.1 | 2 +- docs/man/borg-patterns.1 | 2 +- docs/man/borg-placeholders.1 | 2 +- docs/man/borg-prune.1 | 2 +- docs/man/borg-recreate.1 | 2 +- docs/man/borg-rename.1 | 2 +- docs/man/borg-serve.1 | 2 +- docs/man/borg-umount.1 | 2 +- docs/man/borg-upgrade.1 | 2 +- docs/man/borg-with-lock.1 | 2 +- 30 files changed, 97 insertions(+), 44 deletions(-) diff --git a/docs/man/borg-benchmark-crud.1 b/docs/man/borg-benchmark-crud.1 index cf83afb7d9..758dfdafdc 100644 --- a/docs/man/borg-benchmark-crud.1 +++ b/docs/man/borg-benchmark-crud.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-BENCHMARK-CRUD 1 "2017-09-09" "" "borg backup tool" +.TH BORG-BENCHMARK-CRUD 1 "2017-10-01" "" "borg backup tool" .SH NAME borg-benchmark-crud \- Benchmark Create, Read, Update, Delete for archives. . diff --git a/docs/man/borg-benchmark.1 b/docs/man/borg-benchmark.1 index 7a1febe4d0..fbacaf8f26 100644 --- a/docs/man/borg-benchmark.1 +++ b/docs/man/borg-benchmark.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-BENCHMARK 1 "2017-09-09" "" "borg backup tool" +.TH BORG-BENCHMARK 1 "2017-10-01" "" "borg backup tool" .SH NAME borg-benchmark \- benchmark command . diff --git a/docs/man/borg-break-lock.1 b/docs/man/borg-break-lock.1 index 31c2bdba6b..709159ef2d 100644 --- a/docs/man/borg-break-lock.1 +++ b/docs/man/borg-break-lock.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-BREAK-LOCK 1 "2017-09-09" "" "borg backup tool" +.TH BORG-BREAK-LOCK 1 "2017-10-01" "" "borg backup tool" .SH NAME borg-break-lock \- Break the repository lock (e.g. in case it was left by a dead borg. . diff --git a/docs/man/borg-change-passphrase.1 b/docs/man/borg-change-passphrase.1 index 2c3a6dca85..62dc45ef9c 100644 --- a/docs/man/borg-change-passphrase.1 +++ b/docs/man/borg-change-passphrase.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-CHANGE-PASSPHRASE 1 "2017-09-09" "" "borg backup tool" +.TH BORG-CHANGE-PASSPHRASE 1 "2017-10-01" "" "borg backup tool" .SH NAME borg-change-passphrase \- Change repository key file passphrase . @@ -37,6 +37,11 @@ borg [common options] change\-passphrase [options] [REPOSITORY] .sp The key files used for repository encryption are optionally passphrase protected. This command can be used to change this passphrase. +.sp +Please note that this command only changes the passphrase, but not any +secret protected by it (like e.g. encryption/MAC keys or chunker seed). +Thus, changing the passphrase after passphrase and borg key got compromised +does not protect future (nor past) backups to the same repository. .SH OPTIONS .sp See \fIborg\-common(1)\fP for common options of Borg commands. diff --git a/docs/man/borg-check.1 b/docs/man/borg-check.1 index af3a105f00..ee47d52240 100644 --- a/docs/man/borg-check.1 +++ b/docs/man/borg-check.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-CHECK 1 "2017-09-09" "" "borg backup tool" +.TH BORG-CHECK 1 "2017-10-01" "" "borg backup tool" .SH NAME borg-check \- Check repository consistency . diff --git a/docs/man/borg-common.1 b/docs/man/borg-common.1 index 68f703283a..ea18859e7f 100644 --- a/docs/man/borg-common.1 +++ b/docs/man/borg-common.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-COMMON 1 "2017-09-09" "" "borg backup tool" +.TH BORG-COMMON 1 "2017-10-01" "" "borg backup tool" .SH NAME borg-common \- Common options of Borg commands . diff --git a/docs/man/borg-compression.1 b/docs/man/borg-compression.1 index d2fbba59c3..9cba840c2f 100644 --- a/docs/man/borg-compression.1 +++ b/docs/man/borg-compression.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-COMPRESSION 1 "2017-09-09" "" "borg backup tool" +.TH BORG-COMPRESSION 1 "2017-10-01" "" "borg backup tool" .SH NAME borg-compression \- Details regarding compression . diff --git a/docs/man/borg-create.1 b/docs/man/borg-create.1 index ae8f532756..7e339795e7 100644 --- a/docs/man/borg-create.1 +++ b/docs/man/borg-create.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-CREATE 1 "2017-09-09" "" "borg backup tool" +.TH BORG-CREATE 1 "2017-10-01" "" "borg backup tool" .SH NAME borg-create \- Create new archive . @@ -54,13 +54,50 @@ In the archive name, you may use the following placeholders: {now}, {utcnow}, {fqdn}, {hostname}, {user} and some others. .sp Backup speed is increased by not reprocessing files that are already part of -existing archives and weren\(aqt modified. Normally, detecting file modifications -will take inode information into consideration. This is problematic for files -located on sshfs and similar network file systems which do not provide stable -inode numbers, such files will always be considered modified. The -\fB\-\-ignore\-inode\fP flag can be used to prevent this and improve performance. -This flag will reduce reliability of change detection however, with files -considered unmodified as long as their size and modification time are unchanged. +existing archives and weren\(aqt modified. The detection of unmodified files is +done by comparing multiple file metadata values with previous values kept in +the files cache. +.sp +This comparison can operate in different modes as given by \fB\-\-files\-cache\fP: +.INDENT 0.0 +.IP \(bu 2 +ctime,size,inode (default) +.IP \(bu 2 +mtime,size,inode (default behaviour of borg versions older than 1.1.0rc4) +.IP \(bu 2 +ctime,size (ignore the inode number) +.IP \(bu 2 +mtime,size (ignore the inode number) +.IP \(bu 2 +rechunk,ctime (all files are considered modified \- rechunk, cache ctime) +.IP \(bu 2 +rechunk,mtime (all files are considered modified \- rechunk, cache mtime) +.IP \(bu 2 +disabled (disable the files cache, all files considered modified \- rechunk) +.UNINDENT +.sp +inode number: better safety, but often unstable on network filesystems +.sp +Normally, detecting file modifications will take inode information into +consideration to improve the reliability of file change detection. +This is problematic for files located on sshfs and similar network file +systems which do not provide stable inode numbers, such files will always +be considered modified. You can use modes without \fIinode\fP in this case to +improve performance, but reliability of change detection might be reduced. +.sp +ctime vs. mtime: safety vs. speed +.INDENT 0.0 +.IP \(bu 2 +ctime is a rather safe way to detect changes to a file (metadata and contents) +as it can not be set from userspace. But, a metadata\-only change will already +update the ctime, so there might be some unnecessary chunking/hashing even +without content changes. Some filesystems do not support ctime (change time). +.IP \(bu 2 +mtime usually works and only updates if file contents were changed. But mtime +can be arbitrarily set from userspace, e.g. to set mtime back to the same value +it had before a content change happened. This can be used maliciously as well as +well\-meant, but in both cases mtime based cache modes can be problematic. +.UNINDENT .sp The mount points of filesystems or filesystem snapshots should be the same for every creation of a new archive to ensure fast operation. This is because the file cache that @@ -104,7 +141,7 @@ only display items with the given status characters (see description) output stats as JSON. Implies \fB\-\-stats\fP\&. .TP .B \-\-no\-cache\-sync -experimental: do not synchronize the cache. Implies \fB\-\-no\-files\-cache\fP\&. +experimental: do not synchronize the cache. Implies not using the files cache. .UNINDENT .SS Exclusion options .INDENT 0.0 @@ -148,6 +185,9 @@ do not store ctime into archive .B \-\-ignore\-inode ignore inode data in the file metadata cache used to detect unchanged files. .TP +.BI \-\-files\-cache \ MODE +operate files cache in MODE. default: ctime,size,inode +.TP .B \-\-read\-special open and read block and char device files as well as FIFOs as if they were regular files. Also follows symlinks pointing to these kinds of files. .UNINDENT @@ -212,18 +252,21 @@ $ borg create \-\-chunker\-params 10,23,16,4095 /path/to/repo::small /smallstuff # Backup a raw device (must not be active/in use/mounted at that time) $ dd if=/dev/sdx bs=10M | borg create /path/to/repo::my\-sdx \- -# No compression (default) -$ borg create /path/to/repo::arch ~ +# No compression (none) +$ borg create \-\-compression none /path/to/repo::arch ~ -# Super fast, low compression -$ borg create \-\-compression lz4 /path/to/repo::arch ~ +# Super fast, low compression (lz4, default) +$ borg create /path/to/repo::arch ~ -# Less fast, higher compression (N = 0..9) +# Less fast, higher compression (zlib, N = 0..9) $ borg create \-\-compression zlib,N /path/to/repo::arch ~ -# Even slower, even higher compression (N = 0..9) +# Even slower, even higher compression (lzma, N = 0..9) $ borg create \-\-compression lzma,N /path/to/repo::arch ~ +# Only compress compressible data with lzma,N (N = 0..9) +$ borg create \-\-compression auto,lzma,N /path/to/repo::arch ~ + # Use short hostname, user name and current time in archive name $ borg create /path/to/repo::{hostname}\-{user}\-{now} ~ # Similar, use the same datetime format as borg 1.1 will have as default diff --git a/docs/man/borg-delete.1 b/docs/man/borg-delete.1 index 01513e50c3..52cb0a18b2 100644 --- a/docs/man/borg-delete.1 +++ b/docs/man/borg-delete.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-DELETE 1 "2017-09-09" "" "borg backup tool" +.TH BORG-DELETE 1 "2017-10-01" "" "borg backup tool" .SH NAME borg-delete \- Delete an existing repository or archives . diff --git a/docs/man/borg-diff.1 b/docs/man/borg-diff.1 index 587bf56d7f..9edbd0edbc 100644 --- a/docs/man/borg-diff.1 +++ b/docs/man/borg-diff.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-DIFF 1 "2017-09-09" "" "borg backup tool" +.TH BORG-DIFF 1 "2017-10-01" "" "borg backup tool" .SH NAME borg-diff \- Diff contents of two archives . diff --git a/docs/man/borg-export-tar.1 b/docs/man/borg-export-tar.1 index b2de54dce8..59f447a827 100644 --- a/docs/man/borg-export-tar.1 +++ b/docs/man/borg-export-tar.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-EXPORT-TAR 1 "2017-09-09" "" "borg backup tool" +.TH BORG-EXPORT-TAR 1 "2017-10-01" "" "borg backup tool" .SH NAME borg-export-tar \- Export archive contents as a tarball . diff --git a/docs/man/borg-extract.1 b/docs/man/borg-extract.1 index df5c48da52..2c575e7d49 100644 --- a/docs/man/borg-extract.1 +++ b/docs/man/borg-extract.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-EXTRACT 1 "2017-09-09" "" "borg backup tool" +.TH BORG-EXTRACT 1 "2017-10-01" "" "borg backup tool" .SH NAME borg-extract \- Extract archive contents . diff --git a/docs/man/borg-info.1 b/docs/man/borg-info.1 index 51721f49ad..599371eb70 100644 --- a/docs/man/borg-info.1 +++ b/docs/man/borg-info.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-INFO 1 "2017-09-09" "" "borg backup tool" +.TH BORG-INFO 1 "2017-10-01" "" "borg backup tool" .SH NAME borg-info \- Show archive details such as disk space used . diff --git a/docs/man/borg-init.1 b/docs/man/borg-init.1 index 556002946f..04ad10ff70 100644 --- a/docs/man/borg-init.1 +++ b/docs/man/borg-init.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-INIT 1 "2017-09-09" "" "borg backup tool" +.TH BORG-INIT 1 "2017-10-01" "" "borg backup tool" .SH NAME borg-init \- Initialize an empty repository . diff --git a/docs/man/borg-key-change-passphrase.1 b/docs/man/borg-key-change-passphrase.1 index 2161921e7c..2d16248eb6 100644 --- a/docs/man/borg-key-change-passphrase.1 +++ b/docs/man/borg-key-change-passphrase.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-CHANGE-PASSPHRASE 1 "2017-09-09" "" "borg backup tool" +.TH BORG-KEY-CHANGE-PASSPHRASE 1 "2017-10-01" "" "borg backup tool" .SH NAME borg-key-change-passphrase \- Change repository key file passphrase . @@ -37,6 +37,11 @@ borg [common options] key change\-passphrase [options] [REPOSITORY] .sp The key files used for repository encryption are optionally passphrase protected. This command can be used to change this passphrase. +.sp +Please note that this command only changes the passphrase, but not any +secret protected by it (like e.g. encryption/MAC keys or chunker seed). +Thus, changing the passphrase after passphrase and borg key got compromised +does not protect future (nor past) backups to the same repository. .SH OPTIONS .sp See \fIborg\-common(1)\fP for common options of Borg commands. diff --git a/docs/man/borg-key-export.1 b/docs/man/borg-key-export.1 index 974e1aba0e..61c945520e 100644 --- a/docs/man/borg-key-export.1 +++ b/docs/man/borg-key-export.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-EXPORT 1 "2017-09-09" "" "borg backup tool" +.TH BORG-KEY-EXPORT 1 "2017-10-01" "" "borg backup tool" .SH NAME borg-key-export \- Export the repository key for backup . diff --git a/docs/man/borg-key-import.1 b/docs/man/borg-key-import.1 index fb2bf6357c..fad2cdc02b 100644 --- a/docs/man/borg-key-import.1 +++ b/docs/man/borg-key-import.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-IMPORT 1 "2017-09-09" "" "borg backup tool" +.TH BORG-KEY-IMPORT 1 "2017-10-01" "" "borg backup tool" .SH NAME borg-key-import \- Import the repository key from backup . diff --git a/docs/man/borg-key-migrate-to-repokey.1 b/docs/man/borg-key-migrate-to-repokey.1 index 10340f048f..3d343f27ee 100644 --- a/docs/man/borg-key-migrate-to-repokey.1 +++ b/docs/man/borg-key-migrate-to-repokey.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-MIGRATE-TO-REPOKEY 1 "2017-09-09" "" "borg backup tool" +.TH BORG-KEY-MIGRATE-TO-REPOKEY 1 "2017-10-01" "" "borg backup tool" .SH NAME borg-key-migrate-to-repokey \- Migrate passphrase -> repokey . diff --git a/docs/man/borg-key.1 b/docs/man/borg-key.1 index 881fbb70cd..240c36a4f2 100644 --- a/docs/man/borg-key.1 +++ b/docs/man/borg-key.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY 1 "2017-09-09" "" "borg backup tool" +.TH BORG-KEY 1 "2017-10-01" "" "borg backup tool" .SH NAME borg-key \- Manage a keyfile or repokey of a repository . diff --git a/docs/man/borg-list.1 b/docs/man/borg-list.1 index 06f2686076..743a41b5e4 100644 --- a/docs/man/borg-list.1 +++ b/docs/man/borg-list.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-LIST 1 "2017-09-09" "" "borg backup tool" +.TH BORG-LIST 1 "2017-10-01" "" "borg backup tool" .SH NAME borg-list \- List archive or repository contents . diff --git a/docs/man/borg-mount.1 b/docs/man/borg-mount.1 index 1e664a24bf..fb28ae4443 100644 --- a/docs/man/borg-mount.1 +++ b/docs/man/borg-mount.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-MOUNT 1 "2017-09-09" "" "borg backup tool" +.TH BORG-MOUNT 1 "2017-10-01" "" "borg backup tool" .SH NAME borg-mount \- Mount archive or an entire repository as a FUSE filesystem . diff --git a/docs/man/borg-patterns.1 b/docs/man/borg-patterns.1 index 3be0c4ba3f..8251c8ddef 100644 --- a/docs/man/borg-patterns.1 +++ b/docs/man/borg-patterns.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-PATTERNS 1 "2017-09-09" "" "borg backup tool" +.TH BORG-PATTERNS 1 "2017-10-01" "" "borg backup tool" .SH NAME borg-patterns \- Details regarding patterns . diff --git a/docs/man/borg-placeholders.1 b/docs/man/borg-placeholders.1 index e8346dbaee..5baa5a6637 100644 --- a/docs/man/borg-placeholders.1 +++ b/docs/man/borg-placeholders.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-PLACEHOLDERS 1 "2017-09-09" "" "borg backup tool" +.TH BORG-PLACEHOLDERS 1 "2017-10-01" "" "borg backup tool" .SH NAME borg-placeholders \- Details regarding placeholders . diff --git a/docs/man/borg-prune.1 b/docs/man/borg-prune.1 index e6d46b6fb4..7adaa21d5f 100644 --- a/docs/man/borg-prune.1 +++ b/docs/man/borg-prune.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-PRUNE 1 "2017-09-09" "" "borg backup tool" +.TH BORG-PRUNE 1 "2017-10-01" "" "borg backup tool" .SH NAME borg-prune \- Prune repository archives according to specified rules . diff --git a/docs/man/borg-recreate.1 b/docs/man/borg-recreate.1 index cd90406fa9..67508adf6b 100644 --- a/docs/man/borg-recreate.1 +++ b/docs/man/borg-recreate.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-RECREATE 1 "2017-09-09" "" "borg backup tool" +.TH BORG-RECREATE 1 "2017-10-01" "" "borg backup tool" .SH NAME borg-recreate \- Re-create archives . diff --git a/docs/man/borg-rename.1 b/docs/man/borg-rename.1 index 772d96a0ec..718b417b68 100644 --- a/docs/man/borg-rename.1 +++ b/docs/man/borg-rename.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-RENAME 1 "2017-09-09" "" "borg backup tool" +.TH BORG-RENAME 1 "2017-10-01" "" "borg backup tool" .SH NAME borg-rename \- Rename an existing archive . diff --git a/docs/man/borg-serve.1 b/docs/man/borg-serve.1 index 262eb56f02..485449e0d5 100644 --- a/docs/man/borg-serve.1 +++ b/docs/man/borg-serve.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-SERVE 1 "2017-09-09" "" "borg backup tool" +.TH BORG-SERVE 1 "2017-10-01" "" "borg backup tool" .SH NAME borg-serve \- Start in server mode. This command is usually not used manually. . diff --git a/docs/man/borg-umount.1 b/docs/man/borg-umount.1 index 27d4fbf371..9aa78a3125 100644 --- a/docs/man/borg-umount.1 +++ b/docs/man/borg-umount.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-UMOUNT 1 "2017-09-09" "" "borg backup tool" +.TH BORG-UMOUNT 1 "2017-10-01" "" "borg backup tool" .SH NAME borg-umount \- un-mount the FUSE filesystem . diff --git a/docs/man/borg-upgrade.1 b/docs/man/borg-upgrade.1 index 38b739e3c5..235f149a78 100644 --- a/docs/man/borg-upgrade.1 +++ b/docs/man/borg-upgrade.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-UPGRADE 1 "2017-09-09" "" "borg backup tool" +.TH BORG-UPGRADE 1 "2017-10-01" "" "borg backup tool" .SH NAME borg-upgrade \- upgrade a repository from a previous version . diff --git a/docs/man/borg-with-lock.1 b/docs/man/borg-with-lock.1 index 7b0e2d6b7e..c0bc117b8b 100644 --- a/docs/man/borg-with-lock.1 +++ b/docs/man/borg-with-lock.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-WITH-LOCK 1 "2017-09-09" "" "borg backup tool" +.TH BORG-WITH-LOCK 1 "2017-10-01" "" "borg backup tool" .SH NAME borg-with-lock \- run a user specified command with the repository lock held . From 7e4d6cac0f099b980be42d72257bb6d6657f89c8 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 1 Oct 2017 19:17:23 +0200 Subject: [PATCH 068/798] fix LDLP restoration for subprocesses, fixes #3077 (cherry picked from commit b5069770b7d4eb454cd429a0e4e7b2347994add0) --- src/borg/helpers.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 6b66c786b8..0e71ad5b56 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -2330,14 +2330,27 @@ def prepare_subprocess_env(system, env=None): """ env = dict(env if env is not None else os.environ) if system: - # a pyinstaller binary's bootloader modifies LD_LIBRARY_PATH=/tmp/_ME..., + # a pyinstaller binary's bootloader modifies LD_LIBRARY_PATH=/tmp/_MEIXXXXXX, # but we do not want that system binaries (like ssh or other) pick up # (non-matching) libraries from there. # thus we install the original LDLP, before pyinstaller has modified it: lp_key = 'LD_LIBRARY_PATH' - lp_orig = env.get(lp_key + '_ORIG') # pyinstaller >= 20160820 has this + lp_orig = env.get(lp_key + '_ORIG') # pyinstaller >= 20160820 / v3.2.1 has this if lp_orig is not None: env[lp_key] = lp_orig + else: + # We get here in 2 cases: + # 1. when not running a pyinstaller-made binary. + # in this case, we must not kill LDLP. + # 2. when running a pyinstaller-made binary and there was no LDLP + # in the original env (in this case, the pyinstaller bootloader + # does *not* put ..._ORIG into the env either). + # in this case, we must kill LDLP. + # The directory used by pyinstaller is created by mkdtemp("_MEIXXXXXX"), + # we can use that to differentiate between the cases. + lp = env.get(lp_key) + if lp is not None and re.search(r'/_MEI......', lp): + env.pop(lp_key) # security: do not give secrets to subprocess env.pop('BORG_PASSPHRASE', None) # for information, give borg version to the subprocess From bfdb6c4558a253ca9bbf2003ee6a0100556ef7a3 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 6 Oct 2017 19:51:49 +0200 Subject: [PATCH 069/798] add some comments about recent fuse versions to setup.py --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index dd85db6a1e..899b10ee5d 100644 --- a/setup.py +++ b/setup.py @@ -37,6 +37,8 @@ # llfuse 0.42 (tested shortly, looks ok), needs FUSE version >= 2.8.0 # llfuse 1.0 (tested shortly, looks ok), needs FUSE version >= 2.8.0 # llfuse 1.1.1 (tested shortly, looks ok), needs FUSE version >= 2.8.0 + # llfuse 1.2 (tested shortly, looks ok), needs FUSE version >= 2.8.0 + # llfuse 1.3 (tested shortly, looks ok), needs FUSE version >= 2.8.0 # llfuse 2.0 will break API 'fuse': ['llfuse<2.0', ], } From 1e370d4d23033214e0bcd52ad8daca58beda0d19 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 5 Oct 2017 22:19:39 +0200 Subject: [PATCH 070/798] faq: we do not implement futile attempts of ETA / progress displays (cherry picked from commit fe0843733767432d8af40423fd207231f4805ce1) --- docs/faq.rst | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/faq.rst b/docs/faq.rst index fd9c6e50ec..e23f45a307 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -682,6 +682,31 @@ the corruption is caused by a one time event such as a power outage, running `borg check --repair` will fix most problems. +Why isn't there more progress / ETA information displayed? +---------------------------------------------------------- + +Some borg runs take quite a bit, so it would be nice to see a progress display, +maybe even including a ETA (expected time of "arrival" [here rather "completion"]). + +For some functionality, this can be done: if the total amount of work is more or +less known, we can display progress. So check if there is a ``--progress`` option. + +But sometimes, the total amount is unknown (e.g. for ``borg create`` we just do +a single pass over the filesystem, so we do not know the total file count or data +volume before reaching the end). Adding another pass just to determine that would +take additional time and could be incorrect, if the filesystem is changing. + +Even if the fs does not change and we knew count and size of all files, we still +could not compute the ``borg create`` ETA as we do not know the amount of changed +chunks, how the bandwidth of source and destination or system performance might +fluctuate. + +You see, trying to display ETA would be futile. The borg developers prefer to +rather not implement progress / ETA display than doing futile attempts. + +See also: https://xkcd.com/612/ + + Miscellaneous ############# From 164bb059ec387db02b2f30019dd84283b4296c58 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 7 Oct 2017 03:34:03 +0200 Subject: [PATCH 071/798] manpage: fix typos, update homepage (cherry picked from commit 62e0f7a64e8950863ab53c05f9bf8c4597d16e6e) --- docs/man_intro.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/man_intro.rst b/docs/man_intro.rst index 85ba9bd323..44dee959e5 100644 --- a/docs/man_intro.rst +++ b/docs/man_intro.rst @@ -32,8 +32,8 @@ fully trusted targets. Borg stores a set of files in an *archive*. A *repository* is a collection of *archives*. The format of repositories is Borg-specific. Borg does not -distinguish archives from each other in a any way other than their name, -it does not matter when or where archives where created (eg. different hosts). +distinguish archives from each other in any way other than their name, +it does not matter when or where archives were created (e.g. different hosts). EXAMPLES -------- @@ -61,7 +61,7 @@ SEE ALSO `borg-compression(1)`, `borg-patterns(1)`, `borg-placeholders(1)` -* Main web site https://borgbackup.readthedocs.org/ +* Main web site https://www.borgbackup.org/ * Releases https://github.com/borgbackup/borg/releases * Changelog https://github.com/borgbackup/borg/blob/master/docs/changes.rst * GitHub https://github.com/borgbackup/borg From 355191ab0b3affa49eca4c12792a5e8d1e36df33 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 7 Oct 2017 04:11:29 +0200 Subject: [PATCH 072/798] implement simple "issue" role for manpage generation, fixes #3075 (cherry picked from commit bf3f8e567283f43f297670554d158588df38bf78) --- setup.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/setup.py b/setup.py index 899b10ee5d..42b272613e 100644 --- a/setup.py +++ b/setup.py @@ -653,6 +653,13 @@ def write_see_also(self, write, man_title): def gen_man_page(self, name, rst): from docutils.writers import manpage from docutils.core import publish_string + from docutils.nodes import inline + from docutils.parsers.rst import roles + + def issue(name, rawtext, text, lineno, inliner, options={}, content=[]): + return [inline(rawtext, '#' + text)], [] + + roles.register_local_role('issue', issue) # We give the source_path so that docutils can find relative includes # as-if the document where located in the docs/ directory. man_page = publish_string(source=rst, source_path='docs/virtmanpage.rst', writer=manpage.Writer()) From 5e3758fc7a78bf8e90b9385a202b625c71fe7f37 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 3 Oct 2017 21:11:43 +0200 Subject: [PATCH 073/798] auto compression: make sure expensive compression is actually better if it is not significantly better compressed, we just store lz4 compressed data (which we already have computed anyway), because that at least decompressed super fast. (cherry picked from commit 011e0fd3faf2730681a17403e8fd575bd3ea4b08) --- src/borg/compress.pyx | 21 ++++++++++++++++++--- src/borg/testsuite/compress.py | 18 ++++++++++++------ 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/src/borg/compress.pyx b/src/borg/compress.pyx index 6c1d4c31ff..8e509213e3 100644 --- a/src/borg/compress.pyx +++ b/src/borg/compress.pyx @@ -244,7 +244,7 @@ class Auto(CompressorBase): lz4_data = self.lz4.compress(data) ratio = len(lz4_data) / len(data) if ratio < 0.97: - return self.compressor, None + return self.compressor, lz4_data elif ratio < 1: return self.lz4, lz4_data else: @@ -255,9 +255,24 @@ class Auto(CompressorBase): def compress(self, data): compressor, lz4_data = self._decide(data) - if lz4_data is None: - return compressor.compress(data) + if compressor is self.lz4: + # we know that trying to compress with expensive compressor is likely pointless, + # but lz4 managed to at least squeeze the data a bit. + return lz4_data + if compressor is self.none: + # we know that trying to compress with expensive compressor is likely pointless + # and also lz4 did not manage to squeeze the data (not even a bit). + uncompressed_data = compressor.compress(data) + return uncompressed_data + # if we get here, the decider decided to try the expensive compressor. + # we also know that lz4_data is smaller than uncompressed data. + exp_compressed_data = compressor.compress(data) + ratio = len(exp_compressed_data) / len(lz4_data) + if ratio < 0.99: + # the expensive compressor managed to squeeze the data significantly better than lz4. + return exp_compressed_data else: + # otherwise let's just store the lz4 data, which decompresses extremely fast. return lz4_data def decompress(self, data): diff --git a/src/borg/testsuite/compress.py b/src/borg/testsuite/compress.py index ee6da55a16..f881ad2c7c 100644 --- a/src/borg/testsuite/compress.py +++ b/src/borg/testsuite/compress.py @@ -110,12 +110,18 @@ def test_compressor(): def test_auto(): - compressor = CompressionSpec('auto,zlib,9').compressor - - compressed = compressor.compress(bytes(500)) - assert Compressor.detect(compressed) == ZLIB - - compressed = compressor.compress(b'\x00\xb8\xa3\xa2-O\xe1i\xb6\x12\x03\xc21\xf3\x8a\xf78\\\x01\xa5b\x07\x95\xbeE\xf8\xa3\x9ahm\xb1~') + compressor_auto_zlib = CompressionSpec('auto,zlib,9').compressor + compressor_lz4 = CompressionSpec('lz4').compressor + compressor_zlib = CompressionSpec('zlib,9').compressor + data = bytes(500) + compressed_auto_zlib = compressor_auto_zlib.compress(data) + compressed_lz4 = compressor_lz4.compress(data) + compressed_zlib = compressor_zlib.compress(data) + ratio = len(compressed_zlib) / len(compressed_lz4) + assert Compressor.detect(compressed_auto_zlib) == ZLIB if ratio < 0.99 else LZ4 + + data = b'\x00\xb8\xa3\xa2-O\xe1i\xb6\x12\x03\xc21\xf3\x8a\xf78\\\x01\xa5b\x07\x95\xbeE\xf8\xa3\x9ahm\xb1~' + compressed = compressor_auto_zlib.compress(data) assert Compressor.detect(compressed) == CNONE From ce121246cabe684d34968549d49a0de46530bed7 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 7 Oct 2017 22:13:44 +0200 Subject: [PATCH 074/798] update CHANGES --- docs/changes.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/changes.rst b/docs/changes.rst index eff07f1846..3e10a5aa2e 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -132,6 +132,24 @@ Changelog ========= +Version 1.1.0 (2017-10-07) +-------------------------- + +Fixes: + +- fix LD_LIBRARY_PATH restoration for subprocesses, #3077 +- "auto" compression: make sure expensive compression is actually better, + otherwise store lz4 compressed data we already computed. + +Other changes: + +- docs: + + - FAQ: we do not implement futile attempts of ETA / progress displays + - manpage: fix typos, update homepage + - implement simple "issue" role for manpage generation, #3075 + + Version 1.1.0rc4 (2017-10-01) ----------------------------- From 8685c24a951084e7c5fe2c016846c41469c4b392 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 7 Oct 2017 22:15:27 +0200 Subject: [PATCH 075/798] build_man --- docs/man/borg-benchmark-crud.1 | 2 +- docs/man/borg-benchmark.1 | 2 +- docs/man/borg-break-lock.1 | 2 +- docs/man/borg-change-passphrase.1 | 2 +- docs/man/borg-check.1 | 2 +- docs/man/borg-common.1 | 2 +- docs/man/borg-compression.1 | 2 +- docs/man/borg-create.1 | 2 +- docs/man/borg-delete.1 | 2 +- docs/man/borg-diff.1 | 2 +- docs/man/borg-export-tar.1 | 2 +- docs/man/borg-extract.1 | 2 +- docs/man/borg-info.1 | 2 +- docs/man/borg-init.1 | 2 +- docs/man/borg-key-change-passphrase.1 | 2 +- docs/man/borg-key-export.1 | 2 +- docs/man/borg-key-import.1 | 2 +- docs/man/borg-key-migrate-to-repokey.1 | 2 +- docs/man/borg-key.1 | 2 +- docs/man/borg-list.1 | 2 +- docs/man/borg-mount.1 | 2 +- docs/man/borg-patterns.1 | 2 +- docs/man/borg-placeholders.1 | 2 +- docs/man/borg-prune.1 | 2 +- docs/man/borg-recreate.1 | 2 +- docs/man/borg-rename.1 | 2 +- docs/man/borg-serve.1 | 2 +- docs/man/borg-umount.1 | 2 +- docs/man/borg-upgrade.1 | 2 +- docs/man/borg-with-lock.1 | 2 +- docs/man/borg.1 | 30 ++++++-------------------- 31 files changed, 36 insertions(+), 54 deletions(-) diff --git a/docs/man/borg-benchmark-crud.1 b/docs/man/borg-benchmark-crud.1 index 758dfdafdc..98e58266f1 100644 --- a/docs/man/borg-benchmark-crud.1 +++ b/docs/man/borg-benchmark-crud.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-BENCHMARK-CRUD 1 "2017-10-01" "" "borg backup tool" +.TH BORG-BENCHMARK-CRUD 1 "2017-10-07" "" "borg backup tool" .SH NAME borg-benchmark-crud \- Benchmark Create, Read, Update, Delete for archives. . diff --git a/docs/man/borg-benchmark.1 b/docs/man/borg-benchmark.1 index fbacaf8f26..7cdd889edb 100644 --- a/docs/man/borg-benchmark.1 +++ b/docs/man/borg-benchmark.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-BENCHMARK 1 "2017-10-01" "" "borg backup tool" +.TH BORG-BENCHMARK 1 "2017-10-07" "" "borg backup tool" .SH NAME borg-benchmark \- benchmark command . diff --git a/docs/man/borg-break-lock.1 b/docs/man/borg-break-lock.1 index 709159ef2d..a7b87aeaa7 100644 --- a/docs/man/borg-break-lock.1 +++ b/docs/man/borg-break-lock.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-BREAK-LOCK 1 "2017-10-01" "" "borg backup tool" +.TH BORG-BREAK-LOCK 1 "2017-10-07" "" "borg backup tool" .SH NAME borg-break-lock \- Break the repository lock (e.g. in case it was left by a dead borg. . diff --git a/docs/man/borg-change-passphrase.1 b/docs/man/borg-change-passphrase.1 index 62dc45ef9c..87be29b543 100644 --- a/docs/man/borg-change-passphrase.1 +++ b/docs/man/borg-change-passphrase.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-CHANGE-PASSPHRASE 1 "2017-10-01" "" "borg backup tool" +.TH BORG-CHANGE-PASSPHRASE 1 "2017-10-07" "" "borg backup tool" .SH NAME borg-change-passphrase \- Change repository key file passphrase . diff --git a/docs/man/borg-check.1 b/docs/man/borg-check.1 index ee47d52240..1f675b19b7 100644 --- a/docs/man/borg-check.1 +++ b/docs/man/borg-check.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-CHECK 1 "2017-10-01" "" "borg backup tool" +.TH BORG-CHECK 1 "2017-10-07" "" "borg backup tool" .SH NAME borg-check \- Check repository consistency . diff --git a/docs/man/borg-common.1 b/docs/man/borg-common.1 index ea18859e7f..df5eca490c 100644 --- a/docs/man/borg-common.1 +++ b/docs/man/borg-common.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-COMMON 1 "2017-10-01" "" "borg backup tool" +.TH BORG-COMMON 1 "2017-10-07" "" "borg backup tool" .SH NAME borg-common \- Common options of Borg commands . diff --git a/docs/man/borg-compression.1 b/docs/man/borg-compression.1 index 9cba840c2f..93645abc20 100644 --- a/docs/man/borg-compression.1 +++ b/docs/man/borg-compression.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-COMPRESSION 1 "2017-10-01" "" "borg backup tool" +.TH BORG-COMPRESSION 1 "2017-10-07" "" "borg backup tool" .SH NAME borg-compression \- Details regarding compression . diff --git a/docs/man/borg-create.1 b/docs/man/borg-create.1 index 7e339795e7..2afc4e60bb 100644 --- a/docs/man/borg-create.1 +++ b/docs/man/borg-create.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-CREATE 1 "2017-10-01" "" "borg backup tool" +.TH BORG-CREATE 1 "2017-10-07" "" "borg backup tool" .SH NAME borg-create \- Create new archive . diff --git a/docs/man/borg-delete.1 b/docs/man/borg-delete.1 index 52cb0a18b2..76daae71a0 100644 --- a/docs/man/borg-delete.1 +++ b/docs/man/borg-delete.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-DELETE 1 "2017-10-01" "" "borg backup tool" +.TH BORG-DELETE 1 "2017-10-07" "" "borg backup tool" .SH NAME borg-delete \- Delete an existing repository or archives . diff --git a/docs/man/borg-diff.1 b/docs/man/borg-diff.1 index 9edbd0edbc..a953b184d5 100644 --- a/docs/man/borg-diff.1 +++ b/docs/man/borg-diff.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-DIFF 1 "2017-10-01" "" "borg backup tool" +.TH BORG-DIFF 1 "2017-10-07" "" "borg backup tool" .SH NAME borg-diff \- Diff contents of two archives . diff --git a/docs/man/borg-export-tar.1 b/docs/man/borg-export-tar.1 index 59f447a827..d2ee75a2b9 100644 --- a/docs/man/borg-export-tar.1 +++ b/docs/man/borg-export-tar.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-EXPORT-TAR 1 "2017-10-01" "" "borg backup tool" +.TH BORG-EXPORT-TAR 1 "2017-10-07" "" "borg backup tool" .SH NAME borg-export-tar \- Export archive contents as a tarball . diff --git a/docs/man/borg-extract.1 b/docs/man/borg-extract.1 index 2c575e7d49..2594178275 100644 --- a/docs/man/borg-extract.1 +++ b/docs/man/borg-extract.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-EXTRACT 1 "2017-10-01" "" "borg backup tool" +.TH BORG-EXTRACT 1 "2017-10-07" "" "borg backup tool" .SH NAME borg-extract \- Extract archive contents . diff --git a/docs/man/borg-info.1 b/docs/man/borg-info.1 index 599371eb70..c15b62ddd0 100644 --- a/docs/man/borg-info.1 +++ b/docs/man/borg-info.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-INFO 1 "2017-10-01" "" "borg backup tool" +.TH BORG-INFO 1 "2017-10-07" "" "borg backup tool" .SH NAME borg-info \- Show archive details such as disk space used . diff --git a/docs/man/borg-init.1 b/docs/man/borg-init.1 index 04ad10ff70..9c897e5e96 100644 --- a/docs/man/borg-init.1 +++ b/docs/man/borg-init.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-INIT 1 "2017-10-01" "" "borg backup tool" +.TH BORG-INIT 1 "2017-10-07" "" "borg backup tool" .SH NAME borg-init \- Initialize an empty repository . diff --git a/docs/man/borg-key-change-passphrase.1 b/docs/man/borg-key-change-passphrase.1 index 2d16248eb6..345d43c959 100644 --- a/docs/man/borg-key-change-passphrase.1 +++ b/docs/man/borg-key-change-passphrase.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-CHANGE-PASSPHRASE 1 "2017-10-01" "" "borg backup tool" +.TH BORG-KEY-CHANGE-PASSPHRASE 1 "2017-10-07" "" "borg backup tool" .SH NAME borg-key-change-passphrase \- Change repository key file passphrase . diff --git a/docs/man/borg-key-export.1 b/docs/man/borg-key-export.1 index 61c945520e..5bcf0c9770 100644 --- a/docs/man/borg-key-export.1 +++ b/docs/man/borg-key-export.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-EXPORT 1 "2017-10-01" "" "borg backup tool" +.TH BORG-KEY-EXPORT 1 "2017-10-07" "" "borg backup tool" .SH NAME borg-key-export \- Export the repository key for backup . diff --git a/docs/man/borg-key-import.1 b/docs/man/borg-key-import.1 index fad2cdc02b..c2e0d1c3c6 100644 --- a/docs/man/borg-key-import.1 +++ b/docs/man/borg-key-import.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-IMPORT 1 "2017-10-01" "" "borg backup tool" +.TH BORG-KEY-IMPORT 1 "2017-10-07" "" "borg backup tool" .SH NAME borg-key-import \- Import the repository key from backup . diff --git a/docs/man/borg-key-migrate-to-repokey.1 b/docs/man/borg-key-migrate-to-repokey.1 index 3d343f27ee..6ee88b09f5 100644 --- a/docs/man/borg-key-migrate-to-repokey.1 +++ b/docs/man/borg-key-migrate-to-repokey.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-MIGRATE-TO-REPOKEY 1 "2017-10-01" "" "borg backup tool" +.TH BORG-KEY-MIGRATE-TO-REPOKEY 1 "2017-10-07" "" "borg backup tool" .SH NAME borg-key-migrate-to-repokey \- Migrate passphrase -> repokey . diff --git a/docs/man/borg-key.1 b/docs/man/borg-key.1 index 240c36a4f2..9a6abcdd4c 100644 --- a/docs/man/borg-key.1 +++ b/docs/man/borg-key.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY 1 "2017-10-01" "" "borg backup tool" +.TH BORG-KEY 1 "2017-10-07" "" "borg backup tool" .SH NAME borg-key \- Manage a keyfile or repokey of a repository . diff --git a/docs/man/borg-list.1 b/docs/man/borg-list.1 index 743a41b5e4..6583f3fe34 100644 --- a/docs/man/borg-list.1 +++ b/docs/man/borg-list.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-LIST 1 "2017-10-01" "" "borg backup tool" +.TH BORG-LIST 1 "2017-10-07" "" "borg backup tool" .SH NAME borg-list \- List archive or repository contents . diff --git a/docs/man/borg-mount.1 b/docs/man/borg-mount.1 index fb28ae4443..d10ca21ae6 100644 --- a/docs/man/borg-mount.1 +++ b/docs/man/borg-mount.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-MOUNT 1 "2017-10-01" "" "borg backup tool" +.TH BORG-MOUNT 1 "2017-10-07" "" "borg backup tool" .SH NAME borg-mount \- Mount archive or an entire repository as a FUSE filesystem . diff --git a/docs/man/borg-patterns.1 b/docs/man/borg-patterns.1 index 8251c8ddef..3e5a718e40 100644 --- a/docs/man/borg-patterns.1 +++ b/docs/man/borg-patterns.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-PATTERNS 1 "2017-10-01" "" "borg backup tool" +.TH BORG-PATTERNS 1 "2017-10-07" "" "borg backup tool" .SH NAME borg-patterns \- Details regarding patterns . diff --git a/docs/man/borg-placeholders.1 b/docs/man/borg-placeholders.1 index 5baa5a6637..f6177d915b 100644 --- a/docs/man/borg-placeholders.1 +++ b/docs/man/borg-placeholders.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-PLACEHOLDERS 1 "2017-10-01" "" "borg backup tool" +.TH BORG-PLACEHOLDERS 1 "2017-10-07" "" "borg backup tool" .SH NAME borg-placeholders \- Details regarding placeholders . diff --git a/docs/man/borg-prune.1 b/docs/man/borg-prune.1 index 7adaa21d5f..96cd24419f 100644 --- a/docs/man/borg-prune.1 +++ b/docs/man/borg-prune.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-PRUNE 1 "2017-10-01" "" "borg backup tool" +.TH BORG-PRUNE 1 "2017-10-07" "" "borg backup tool" .SH NAME borg-prune \- Prune repository archives according to specified rules . diff --git a/docs/man/borg-recreate.1 b/docs/man/borg-recreate.1 index 67508adf6b..584d56f411 100644 --- a/docs/man/borg-recreate.1 +++ b/docs/man/borg-recreate.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-RECREATE 1 "2017-10-01" "" "borg backup tool" +.TH BORG-RECREATE 1 "2017-10-07" "" "borg backup tool" .SH NAME borg-recreate \- Re-create archives . diff --git a/docs/man/borg-rename.1 b/docs/man/borg-rename.1 index 718b417b68..1050795214 100644 --- a/docs/man/borg-rename.1 +++ b/docs/man/borg-rename.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-RENAME 1 "2017-10-01" "" "borg backup tool" +.TH BORG-RENAME 1 "2017-10-07" "" "borg backup tool" .SH NAME borg-rename \- Rename an existing archive . diff --git a/docs/man/borg-serve.1 b/docs/man/borg-serve.1 index 485449e0d5..a5cd6c9aa1 100644 --- a/docs/man/borg-serve.1 +++ b/docs/man/borg-serve.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-SERVE 1 "2017-10-01" "" "borg backup tool" +.TH BORG-SERVE 1 "2017-10-07" "" "borg backup tool" .SH NAME borg-serve \- Start in server mode. This command is usually not used manually. . diff --git a/docs/man/borg-umount.1 b/docs/man/borg-umount.1 index 9aa78a3125..5c53d501ac 100644 --- a/docs/man/borg-umount.1 +++ b/docs/man/borg-umount.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-UMOUNT 1 "2017-10-01" "" "borg backup tool" +.TH BORG-UMOUNT 1 "2017-10-07" "" "borg backup tool" .SH NAME borg-umount \- un-mount the FUSE filesystem . diff --git a/docs/man/borg-upgrade.1 b/docs/man/borg-upgrade.1 index 235f149a78..a9cd99cbce 100644 --- a/docs/man/borg-upgrade.1 +++ b/docs/man/borg-upgrade.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-UPGRADE 1 "2017-10-01" "" "borg backup tool" +.TH BORG-UPGRADE 1 "2017-10-07" "" "borg backup tool" .SH NAME borg-upgrade \- upgrade a repository from a previous version . diff --git a/docs/man/borg-with-lock.1 b/docs/man/borg-with-lock.1 index c0bc117b8b..0f84a7ff7e 100644 --- a/docs/man/borg-with-lock.1 +++ b/docs/man/borg-with-lock.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-WITH-LOCK 1 "2017-10-01" "" "borg backup tool" +.TH BORG-WITH-LOCK 1 "2017-10-07" "" "borg backup tool" .SH NAME borg-with-lock \- run a user specified command with the repository lock held . diff --git a/docs/man/borg.1 b/docs/man/borg.1 index 797788a809..9f3da1e24a 100644 --- a/docs/man/borg.1 +++ b/docs/man/borg.1 @@ -48,8 +48,8 @@ fully trusted targets. .sp Borg stores a set of files in an \fIarchive\fP\&. A \fIrepository\fP is a collection of \fIarchives\fP\&. The format of repositories is Borg\-specific. Borg does not -distinguish archives from each other in a any way other than their name, -it does not matter when or where archives where created (eg. different hosts). +distinguish archives from each other in any way other than their name, +it does not matter when or where archives were created (e.g. different hosts). .SH EXAMPLES .SS A step\-by\-step example .INDENT 0.0 @@ -716,29 +716,11 @@ Note that most of the platform\-dependent features also depend on the file syste For example, ntfs\-3g on Linux isn\(aqt able to convey NTFS ACLs. .IP [1] 5 Only "nodump", "immutable", "compressed" and "append" are supported. -Feature request -.nf -:issue:\(ga618\(ga -.fi - for more flags. -.IP "System Message: ERROR/3 (docs/usage_general.rst.inc:, line 410)" -Unknown interpreted text role "issue". +Feature request #618 for more flags. .IP [2] 5 -Feature request -.nf -:issue:\(ga1332\(ga -.fi - -.IP "System Message: ERROR/3 (docs/usage_general.rst.inc:, line 412)" -Unknown interpreted text role "issue". +Feature request #1332 .IP [3] 5 -Feature request -.nf -:issue:\(ga1337\(ga -.fi - -.IP "System Message: ERROR/3 (docs/usage_general.rst.inc:, line 413)" -Unknown interpreted text role "issue". +Feature request #1337 .IP [4] 5 Cygwin tries to map NTFS ACLs to permissions with varying degress of success. .IP [5] 5 @@ -763,7 +745,7 @@ The BSDs define additional flags. \fIborg\-compression(1)\fP, \fIborg\-patterns(1)\fP, \fIborg\-placeholders(1)\fP .INDENT 0.0 .IP \(bu 2 -Main web site \fI\%https://borgbackup.readthedocs.org/\fP +Main web site \fI\%https://www.borgbackup.org/\fP .IP \(bu 2 Releases \fI\%https://github.com/borgbackup/borg/releases\fP .IP \(bu 2 From c995797640c86c976c26e85529a917ad091a9939 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 9 Oct 2017 04:58:44 +0200 Subject: [PATCH 076/798] vagrant: use self-made FreeBSD 10.3 box, fixes #3022 --- Vagrantfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index ad931c77a3..206ca54988 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -502,9 +502,8 @@ Vagrant.configure(2) do |config| end # BSD - # note: the FreeBSD-10.3-RELEASE box needs "vagrant up" twice to start. config.vm.define "freebsd64" do |b| - b.vm.box = "freebsd/FreeBSD-10.3-RELEASE" + b.vm.box = "freebsd64" # custom FreeBSD 10.3 box. official ones are broken, #3022. b.vm.provider :virtualbox do |v| v.memory = 1024 + $wmem end From a68d463041bfa90ae8de330daa224b7b170da837 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 10 Oct 2017 00:51:13 +0200 Subject: [PATCH 077/798] don't crash in first part of truncate_and_unlink, fixes #3117 (cherry picked from commit 7a689b1295ca647a7f9008df508f303214930d08) --- src/borg/helpers.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 0e71ad5b56..f1721c52bc 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -2,6 +2,7 @@ import contextlib import collections import enum +import errno import grp import hashlib import logging @@ -2283,8 +2284,13 @@ def truncate_and_unlink(path): recover. Refer to the "File system interaction" section in repository.py for further explanations. """ - with open(path, 'r+b') as fd: - fd.truncate() + try: + with open(path, 'r+b') as fd: + fd.truncate() + except OSError as err: + if err.errno != errno.ENOTSUP: + raise + # don't crash if the above ops are not supported. os.unlink(path) From 747cfadf795e0b7498bf69b16bba1cfa8f394ba2 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 10 Oct 2017 01:57:58 +0200 Subject: [PATCH 078/798] catch ENOTSUP for os.link, fixes #3107 (cherry picked from commit 203a5c8f197904b51fbced32f42cf0c383130292) --- src/borg/repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/borg/repository.py b/src/borg/repository.py index 5320b56a2e..e327ddace2 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -266,7 +266,7 @@ def save_config(self, path, config): try: os.link(config_path, old_config_path) except OSError as e: - if e.errno in (errno.EMLINK, errno.ENOSYS, errno.EPERM): + if e.errno in (errno.EMLINK, errno.ENOSYS, errno.EPERM, errno.ENOTSUP): logger.warning("Hardlink failed, cannot securely erase old config file") else: raise From fd253a281312e11b7340b4367859ec1f481fdda9 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 10 Oct 2017 01:17:56 +0200 Subject: [PATCH 079/798] recreate: don't crash on attic archives w/o time_end, fixes #3109 (cherry picked from commit 9d3daebd5fa6a7929fc3a1e556a2b7931c97b354) --- src/borg/archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index 4366419cf5..f565b1ff83 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -1697,7 +1697,7 @@ def save(self, archive, target, comment=None, replace_original=True): target.save(comment=comment, additional_metadata={ # keep some metadata as in original archive: 'time': archive.metadata.time, - 'time_end': archive.metadata.time_end, + 'time_end': archive.metadata.get('time_end') or archive.metadata.time, 'cmdline': archive.metadata.cmdline, # but also remember recreate metadata: 'recreate_cmdline': sys.argv, From d35537c19911e94e097033d243c780f289c20449 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 10 Oct 2017 01:36:44 +0200 Subject: [PATCH 080/798] fix detection of non-local path, fixes #3108 filenames like ..foobar are valid, so, to detect stuff in upper dirs, we need to include the path separator and check if it starts with '../'. (cherry picked from commit 60e924910034b86d3d9c6e9af706e5559cdb4d19) --- src/borg/archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index 4366419cf5..3f09ea4662 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -557,7 +557,7 @@ def extract_item(self, item, restore_attrs=True, dry_run=False, stdout=False, sp original_path = original_path or item.path dest = self.cwd - if item.path.startswith(('/', '..')): + if item.path.startswith(('/', '../')): raise Exception('Path should be relative and local') path = os.path.join(dest, item.path) # Attempt to remove existing files, ignore errors on failure From edcc64aeedf3dabffd72d111bc88a808643b0f87 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 10 Oct 2017 02:18:13 +0200 Subject: [PATCH 081/798] logging with fileConfig: set json attr on "borg" logger, fixes #3114 (cherry picked from commit afba813706e3f0db2688086fd553b40c770f4136) --- src/borg/logger.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/borg/logger.py b/src/borg/logger.py index 69cb86f18c..7edec891b8 100644 --- a/src/borg/logger.py +++ b/src/borg/logger.py @@ -80,6 +80,8 @@ def setup_logging(stream=None, conf_fname=None, env_var='BORG_LOGGING_CONF', lev logging.config.fileConfig(f) configured = True logger = logging.getLogger(__name__) + borg_logger = logging.getLogger('borg') + borg_logger.json = json logger.debug('using logging configuration read from "{0}"'.format(conf_fname)) warnings.showwarning = _log_warning return None From 0e44f07b8835bd3a40b690c5586bacfd084f495c Mon Sep 17 00:00:00 2001 From: Fabio Pedretti Date: Mon, 9 Oct 2017 14:09:43 +0200 Subject: [PATCH 082/798] use --format rather than --list-format in examples, the latter is deprecated (cherry picked from commit bc42f58c04edf5d318ede88629f3b5edb98e367c) --- docs/usage/list.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage/list.rst b/docs/usage/list.rst index 4268bff1ad..9b84eb01a9 100644 --- a/docs/usage/list.rst +++ b/docs/usage/list.rst @@ -19,7 +19,7 @@ Examples -rwxr-xr-x root root 2140 Fri, 2015-03-27 20:24:22 bin/bzdiff ... - $ borg list /path/to/repo::archiveA --list-format="{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}" + $ borg list /path/to/repo::archiveA --format="{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}" drwxrwxr-x user user 0 Sun, 2015-02-01 11:00:00 . drwxrwxr-x user user 0 Sun, 2015-02-01 11:00:00 code drwxrwxr-x user user 0 Sun, 2015-02-01 11:00:00 code/myproject From ad07efcb29dce87a634be786b9768aa275db6b02 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 8 Oct 2017 12:28:05 +0200 Subject: [PATCH 083/798] init: fix wrong encryption choices in command line parser (cherry picked from commit b00179ff784fbd959d7ee1e331ac62a1c9bbc25a) --- src/borg/archiver.py | 4 ++-- src/borg/crypto/key.py | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 7797c65c39..1d211d33b6 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -40,7 +40,7 @@ from .cache import Cache, assert_secure from .constants import * # NOQA from .compress import CompressionSpec -from .crypto.key import key_creator, tam_required_file, tam_required, RepoKey, PassphraseKey +from .crypto.key import key_creator, key_argument_names, tam_required_file, tam_required, RepoKey, PassphraseKey from .crypto.keymanager import KeyManager from .helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR from .helpers import Error, NoManifestError, set_ec @@ -2580,7 +2580,7 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): type=location_validator(archive=False), help='repository to create') subparser.add_argument('-e', '--encryption', metavar='MODE', dest='encryption', required=True, - choices=('none', 'keyfile', 'repokey', 'keyfile-blake2', 'repokey-blake2', 'authenticated'), + choices=key_argument_names(), help='select encryption key mode **(required)**') subparser.add_argument('--append-only', dest='append_only', action='store_true', help='create an append-only mode repository') diff --git a/src/borg/crypto/key.py b/src/borg/crypto/key.py index 58f928023f..53a4c3df91 100644 --- a/src/borg/crypto/key.py +++ b/src/borg/crypto/key.py @@ -103,11 +103,16 @@ class KeyBlobStorage: def key_creator(repository, args): for key in AVAILABLE_KEY_TYPES: if key.ARG_NAME == args.encryption: + assert key.ARG_NAME is not None return key.create(repository, args) else: raise ValueError('Invalid encryption mode "%s"' % args.encryption) +def key_argument_names(): + return [key.ARG_NAME for key in AVAILABLE_KEY_TYPES if key.ARG_NAME] + + def identify_key(manifest_data): key_type = manifest_data[0] if key_type == PassphraseKey.TYPE: From 7c1b08922a11846dc10e0fa4a1c89636db04e18b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 12 Oct 2017 16:14:32 +0200 Subject: [PATCH 084/798] readme: -e is required in borg 1.1 (cherry picked from commit c8441b5b3da4413e63c9b856b2816a6758dae2f3) --- README.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 6bc6117876..6762017fb9 100644 --- a/README.rst +++ b/README.rst @@ -89,9 +89,12 @@ Main features Easy to use ~~~~~~~~~~~ -Initialize a new backup repository and create a backup archive:: +Initialize a new backup repository (see ``borg init --help`` for encryption options):: + + $ borg init -e repokey /path/to/repo + +Create a backup archive:: - $ borg init /path/to/repo $ borg create /path/to/repo::Saturday1 ~/Documents Now doing another backup, just to show off the great deduplication:: From c768999a05d27bacd272298e3cd2a14c9deee70b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 12 Oct 2017 05:40:52 +0200 Subject: [PATCH 085/798] don't crash if only a global option is given, show help, fixes #3142 (cherry picked from commit 3be328ed709d7d893415ccc4ffc046835b74eb6b) --- src/borg/archiver.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 7797c65c39..745e757740 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -2156,6 +2156,8 @@ def do_subcommand_help(self, parser, args): parser.print_help() return EXIT_SUCCESS + do_maincommand_help = do_subcommand_help + def preprocess_args(self, args): deprecations = [ # ('--old', '--new' or None, 'Warning: "--old" has been deprecated. Use "--new" instead.'), @@ -2423,6 +2425,7 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): parser = argparse.ArgumentParser(prog=self.prog, description='Borg - Deduplicated Backups', add_help=False) + parser.set_defaults(func=functools.partial(self.do_maincommand_help, parser)) parser.common_options = self.CommonOptions(define_common_options, suffix_precedence=('_maincommand', '_midcommand', '_subcommand')) parser.add_argument('-V', '--version', action='version', version='%(prog)s ' + __version__, From fcaf764604fe0bd299fe42add5601455438ea7ae Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 12 Oct 2017 06:03:12 +0200 Subject: [PATCH 086/798] add example showing --show-version --show-rc (cherry picked from commit ed1a8b5cf1009423cb34753e9c1e0dda3b4b7b91) --- docs/usage/general.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/usage/general.rst b/docs/usage/general.rst index 127ab66762..5629aa5b85 100644 --- a/docs/usage/general.rst +++ b/docs/usage/general.rst @@ -30,3 +30,11 @@ Common options All Borg commands share these options: .. include:: common-options.rst.inc + +Examples +~~~~~~~~ +:: + + # Create an archive and log: borg version, files list, return code + $ borg create --show-version --list --show-rc /path/to/repo::my-files files + From aac190eae14aa02989e034f49509e5becd617588 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 12 Oct 2017 03:37:48 +0200 Subject: [PATCH 087/798] don't brew update, hopefully fixes #2532 (cherry picked from commit 6049a07b74b55856d5be999bac4b4d49a5317d35) --- .travis/install.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis/install.sh b/.travis/install.sh index 708b8c81b1..a4f24c1e59 100755 --- a/.travis/install.sh +++ b/.travis/install.sh @@ -4,8 +4,6 @@ set -e set -x if [[ "$(uname -s)" == 'Darwin' ]]; then - brew update || brew update - if [[ "${OPENSSL}" != "0.9.8" ]]; then brew outdated openssl || brew upgrade openssl fi From e3cdd90729616e022e365e67337bf79f15152b1e Mon Sep 17 00:00:00 2001 From: TW Date: Fri, 13 Oct 2017 20:16:30 +0200 Subject: [PATCH 088/798] move --no-files-cache from common to borg create options, fixes #3146 for borg prune, just use do_files=False (it only needs the chunks cache, not the files cache). (cherry picked from commit 19ed725a58b58901d22dbcdb74d22cf029c468d1) --- src/borg/archiver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 521bc6a205..1cead626e4 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -1523,7 +1523,7 @@ def do_prune(self, args, repository, manifest, key): keep += prune_split(archives, '%Y', args.yearly, keep) to_delete = (set(archives) | checkpoints) - (set(keep) | set(keep_checkpoints)) stats = Statistics() - with Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) as cache: + with Cache(repository, key, manifest, do_files=False, lock_wait=self.lock_wait) as cache: list_logger = logging.getLogger('borg.output.list') if args.output_list: # set up counters for the progress display @@ -2351,8 +2351,6 @@ def define_common_options(add_common_option): help='show/log the borg version') add_common_option('--show-rc', dest='show_rc', action='store_true', help='show/log the return code (rc)') - add_common_option('--no-files-cache', dest='cache_files', action='store_false', - help='do not load/update the file metadata cache used to detect unchanged files') add_common_option('--umask', metavar='M', dest='umask', type=lambda s: int(s, 8), default=UMASK_DEFAULT, help='set umask to M (local and remote, default: %(default)04o)') add_common_option('--remote-path', metavar='PATH', dest='remote_path', @@ -2923,6 +2921,8 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): help='output stats as JSON. Implies ``--stats``.') subparser.add_argument('--no-cache-sync', dest='no_cache_sync', action='store_true', help='experimental: do not synchronize the cache. Implies not using the files cache.') + subparser.add_argument('--no-files-cache', dest='cache_files', action='store_false', + help='do not load/update the file metadata cache used to detect unchanged files') define_exclusion_group(subparser, tag_files=True) From e1723ec1d970da9e4f5f2fb3ac9c1e9ffedc8f9e Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 11 Oct 2017 03:32:33 +0200 Subject: [PATCH 089/798] bsdflags support: do not open BLK/CHR/LNK files, fixes #3130 opening a device file for a non-existing device can be very slow. symlinks will make the open() call fail as it is using O_NOFOLLOW. also: lstat -> stat(..., follow_symlinks=False) like everywhere else. (cherry picked from commit a6ee4e9aedd2eb124c3a486f12131215661f77e1) --- src/borg/platform/linux.pyx | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/borg/platform/linux.pyx b/src/borg/platform/linux.pyx index 7c3cf1b0b5..25f71fa189 100644 --- a/src/borg/platform/linux.pyx +++ b/src/borg/platform/linux.pyx @@ -72,8 +72,11 @@ BSD_TO_LINUX_FLAGS = { def set_flags(path, bsd_flags, fd=None): - if fd is None and stat.S_ISLNK(os.lstat(path).st_mode): - return + if fd is None: + st = os.stat(path, follow_symlinks=False) + if stat.S_ISBLK(st.st_mode) or stat.S_ISCHR(st.st_mode) or stat.S_ISLNK(st.st_mode): + # see comment in get_flags() + return cdef int flags = 0 for bsd_flag, linux_flag in BSD_TO_LINUX_FLAGS.items(): if bsd_flags & bsd_flag: @@ -92,6 +95,10 @@ def set_flags(path, bsd_flags, fd=None): def get_flags(path, st): + if stat.S_ISBLK(st.st_mode) or stat.S_ISCHR(st.st_mode) or stat.S_ISLNK(st.st_mode): + # avoid opening devices files - trying to open non-present devices can be rather slow. + # avoid opening symlinks, O_NOFOLLOW would make the open() fail anyway. + return 0 cdef int linux_flags try: fd = os.open(path, os.O_RDONLY|os.O_NONBLOCK|os.O_NOFOLLOW) From 9a6da1d65e3a7e7e273b9ac621e9eb9496a15054 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 14 Oct 2017 04:24:26 +0200 Subject: [PATCH 090/798] borg recreate: correctly compute part file sizes, fixes #3157 when doing in-file checkpointing, borg creates *.borg_part_N files. complete_file = part_1 + part_2 + ... + part_N the source item for recreate already has a precomputed (total) size member, thus we must force recomputation from the (partial) chunks list to correct the size to be the part's size only. borg create avoided this problem by computing the size member after writing all the parts. this is now not required any more. the bug is mostly cosmetic, borg check will complain, borg extract on a part file would also complain. but all the complaints only refer to the wrong metadata of the part files, the part files' contents are correct. usually you will never extract or look at part files, but only deal with the full file, which will be completely valid, all metadata and content. you can get rid of the archives with these cosmetic errors by running borg recreate on them with a fixed borg version. the old part files will get dropped (because they are usually ignored) and any new part file created due to checkpointing will be correct. (cherry picked from commit 9d6b125e989b02487cc05f6c5371ad2058cc9579) --- src/borg/archive.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index a47c528d28..ca8bc95c47 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -897,7 +897,9 @@ def write_part_file(self, item, from_chunk, number): length = len(item.chunks) # the item should only have the *additional* chunks we processed after the last partial item: item.chunks = item.chunks[from_chunk:] - item.get_size(memorize=True) + # for borg recreate, we already have a size member in the source item (giving the total file size), + # but we consider only a part of the file here, thus we must recompute the size from the chunks: + item.get_size(memorize=True, from_chunks=True) item.path += '.borg_part_%d' % number item.part = number number += 1 From 386e4cee5926e5b79aec1bf5adb2c9dbc4d754db Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 14 Oct 2017 21:57:58 +0200 Subject: [PATCH 091/798] cache: use SaveFile for more safety, fixes #3158 Looks like under unfortunate circumstances, these files could become 0 byte files (see #3158). SaveFile usage should prevent that. (cherry picked from commit 0190abff814a8b5bc5cdb8342593ca3d4da321ce) --- src/borg/cache.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/borg/cache.py b/src/borg/cache.py index 3d8883797a..621e2d85ca 100644 --- a/src/borg/cache.py +++ b/src/borg/cache.py @@ -85,11 +85,11 @@ def save(self, manifest, key): logger.debug('security: current location %s', current_location) logger.debug('security: key type %s', str(key.TYPE)) logger.debug('security: manifest timestamp %s', manifest.timestamp) - with open(self.location_file, 'w') as fd: + with SaveFile(self.location_file) as fd: fd.write(current_location) - with open(self.key_type_file, 'w') as fd: + with SaveFile(self.key_type_file) as fd: fd.write(str(key.TYPE)) - with open(self.manifest_ts_file, 'w') as fd: + with SaveFile(self.manifest_ts_file) as fd: fd.write(manifest.timestamp) def assert_location_matches(self, cache_config=None): @@ -119,7 +119,7 @@ def assert_location_matches(self, cache_config=None): raise Cache.RepositoryAccessAborted() # adapt on-disk config immediately if the new location was accepted logger.debug('security: updating location stored in cache and security dir') - with open(self.location_file, 'w') as fd: + with SaveFile(self.location_file) as fd: fd.write(repository_location) if cache_config: cache_config.save() @@ -470,7 +470,7 @@ def create(self): self.cache_config.create() ChunkIndex().write(os.path.join(self.path, 'chunks')) os.makedirs(os.path.join(self.path, 'chunks.archive.d')) - with SaveFile(os.path.join(self.path, 'files'), binary=True) as fd: + with SaveFile(os.path.join(self.path, 'files'), binary=True): pass # empty file def _do_open(self): @@ -844,7 +844,7 @@ def wipe_cache(self): shutil.rmtree(os.path.join(self.path, 'chunks.archive.d')) os.makedirs(os.path.join(self.path, 'chunks.archive.d')) self.chunks = ChunkIndex() - with open(os.path.join(self.path, 'files'), 'wb'): + with SaveFile(os.path.join(self.path, 'files'), binary=True): pass # empty file self.cache_config.manifest_id = '' self.cache_config._config.set('cache', 'manifest', '') From 229a117ceb5d31358bc14b2d4e03c9ba816efba6 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 14 Oct 2017 05:18:21 +0200 Subject: [PATCH 092/798] implement --nobsdflags and --exclude-nodump, fixes #3160 do no read/archive bsdflags: borg create --nobsdflags ... do not extract/set bsdflags: borg extract --nobsdflags ... use cases: - fs shows wrong / random bsdflags (bug in filesystem) - fs does not support bsdflags anyway - already archived bsdflags are wrong / unwanted - borg shows any sort of unwanted effect due to get_flags, esp. on Linux the nodump flag ("do not backup this file") is not honoured any more by default because this functionality (esp. if it happened by error or unexpected) was rather confusing and unexplainable at first to users. if you want that "do not backup NODUMP-flagged files" behaviour, use: borg create --exclude-nodump ... (cherry picked from commit 10adadf685df0fa4da9f7c3d23891d997ff3395e) --- src/borg/archive.py | 11 +++++++---- src/borg/archiver.py | 27 +++++++++++++++++++-------- src/borg/testsuite/archiver.py | 12 ++++++------ 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index ca8bc95c47..e29919b79a 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -280,8 +280,8 @@ class IncompatibleFilesystemEncodingError(Error): """Failed to encode filename "{}" into file system encoding "{}". Consider configuring the LANG environment variable.""" def __init__(self, repository, key, manifest, name, cache=None, create=False, - checkpoint_interval=300, numeric_owner=False, noatime=False, noctime=False, progress=False, - chunker_params=CHUNKER_PARAMS, start=None, start_monotonic=None, end=None, + checkpoint_interval=300, numeric_owner=False, noatime=False, noctime=False, nobsdflags=False, + progress=False, chunker_params=CHUNKER_PARAMS, start=None, start_monotonic=None, end=None, consider_part_files=False, log_json=False): self.cwd = os.getcwd() self.key = key @@ -298,6 +298,7 @@ def __init__(self, repository, key, manifest, name, cache=None, create=False, self.numeric_owner = numeric_owner self.noatime = noatime self.noctime = noctime + self.nobsdflags = nobsdflags assert (start is None) == (start_monotonic is None), 'Logic error: if start is given, start_monotonic must be given as well and vice versa.' if start is None: start = datetime.utcnow() @@ -691,7 +692,7 @@ def restore_attrs(self, path, item, symlink=False, fd=None): # some systems don't support calling utime on a symlink pass acl_set(path, item, self.numeric_owner) - if 'bsdflags' in item: + if not self.nobsdflags and 'bsdflags' in item: try: set_flags(path, item.bsdflags, fd=fd) except OSError: @@ -829,9 +830,11 @@ def stat_simple_attrs(self, st): def stat_ext_attrs(self, st, path): attrs = {} + bsdflags = 0 with backup_io('extended stat'): xattrs = xattr.get_all(path, follow_symlinks=False) - bsdflags = get_flags(path, st) + if not self.nobsdflags: + bsdflags = get_flags(path, st) acl_get(path, attrs, st, self.numeric_owner) if xattrs: attrs['xattrs'] = StableDict(xattrs) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 1cead626e4..16f5836808 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -154,7 +154,9 @@ def with_archive(method): @functools.wraps(method) def wrapper(self, args, repository, key, manifest, **kwargs): archive = Archive(repository, key, manifest, args.location.archive, - numeric_owner=getattr(args, 'numeric_owner', False), cache=kwargs.get('cache'), + numeric_owner=getattr(args, 'numeric_owner', False), + nobsdflags=getattr(args, 'nobsdflags', False), + cache=kwargs.get('cache'), consider_part_files=args.consider_part_files, log_json=args.log_json) return method(self, args, repository=repository, manifest=manifest, key=key, archive=archive, **kwargs) return wrapper @@ -512,6 +514,7 @@ def create_inner(archive, cache): self.output_filter = args.output_filter self.output_list = args.output_list self.ignore_inode = args.ignore_inode + self.exclude_nodump = args.exclude_nodump self.files_cache_mode = args.files_cache_mode dry_run = args.dry_run t0 = datetime.utcnow() @@ -522,7 +525,7 @@ def create_inner(archive, cache): archive = Archive(repository, key, manifest, args.location.archive, cache=cache, create=True, checkpoint_interval=args.checkpoint_interval, numeric_owner=args.numeric_owner, noatime=args.noatime, noctime=args.noctime, - progress=args.progress, + nobsdflags=args.nobsdflags, progress=args.progress, chunker_params=args.chunker_params, start=t0, start_monotonic=t0_monotonic, log_json=args.log_json) create_inner(archive, cache) @@ -561,11 +564,12 @@ def _process(self, archive, cache, matcher, exclude_caches, exclude_if_present, # directory of the mounted filesystem that shadows the mountpoint dir). recurse = restrict_dev is None or st.st_dev == restrict_dev status = None - # Ignore if nodump flag is set - with backup_io('flags'): - if get_flags(path, st) & stat.UF_NODUMP: - self.print_file_status('x', path) - return + if self.exclude_nodump: + # Ignore if nodump flag is set + with backup_io('flags'): + if get_flags(path, st) & stat.UF_NODUMP: + self.print_file_status('x', path) + return if stat.S_ISREG(st.st_mode): if not dry_run: status = archive.process_file(path, st, cache, self.ignore_inode, self.files_cache_mode) @@ -2395,6 +2399,7 @@ def define_exclude_and_patterns(add_option, *, tag_files=False, strip_components def define_exclusion_group(subparser, **kwargs): exclude_group = subparser.add_argument_group('Exclusion options') define_exclude_and_patterns(exclude_group.add_argument, **kwargs) + return exclude_group def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): filters_group = subparser.add_argument_group('Archive filters', @@ -2924,7 +2929,9 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): subparser.add_argument('--no-files-cache', dest='cache_files', action='store_false', help='do not load/update the file metadata cache used to detect unchanged files') - define_exclusion_group(subparser, tag_files=True) + exclude_group = define_exclusion_group(subparser, tag_files=True) + exclude_group.add_argument('--exclude-nodump', dest='exclude_nodump', action='store_true', + help='exclude files flagged NODUMP') fs_group = subparser.add_argument_group('Filesystem options') fs_group.add_argument('-x', '--one-file-system', dest='one_file_system', action='store_true', @@ -2935,6 +2942,8 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): help='do not store atime into archive') fs_group.add_argument('--noctime', dest='noctime', action='store_true', help='do not store ctime into archive') + fs_group.add_argument('--nobsdflags', dest='nobsdflags', action='store_true', + help='do not read and store bsdflags (e.g. NODUMP, IMMUTABLE) into archive') fs_group.add_argument('--ignore-inode', dest='ignore_inode', action='store_true', help='ignore inode data in the file metadata cache used to detect unchanged files.') fs_group.add_argument('--files-cache', metavar='MODE', dest='files_cache_mode', @@ -3001,6 +3010,8 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): help='do not actually change any files') subparser.add_argument('--numeric-owner', dest='numeric_owner', action='store_true', help='only obey numeric user and group identifiers') + subparser.add_argument('--nobsdflags', dest='nobsdflags', action='store_true', + help='do not extract/set bsdflags (e.g. NODUMP, IMMUTABLE)') subparser.add_argument('--stdout', dest='stdout', action='store_true', help='write all extracted data to stdout') subparser.add_argument('--sparse', dest='sparse', action='store_true', diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 164c2226e7..f6ada2f8a6 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -380,8 +380,8 @@ def test_basic_functionality(self): output = self.cmd('init', '--encryption=repokey', '--show-version', '--show-rc', self.repository_location, fork=True) self.assert_in('borgbackup version', output) self.assert_in('terminating with success status, rc 0', output) - self.cmd('create', self.repository_location + '::test', 'input') - output = self.cmd('create', '--stats', self.repository_location + '::test.2', 'input') + self.cmd('create', '--exclude-nodump', self.repository_location + '::test', 'input') + output = self.cmd('create', '--exclude-nodump', '--stats', self.repository_location + '::test.2', 'input') self.assert_in('Archive name: test.2', output) self.assert_in('This archive: ', output) with changedir('output'): @@ -1655,13 +1655,13 @@ def test_file_status_excluded(self): self.create_regular_file('file3', size=1024 * 80) platform.set_flags(os.path.join(self.input_path, 'file3'), stat.UF_NODUMP) self.cmd('init', '--encryption=repokey', self.repository_location) - output = self.cmd('create', '--list', self.repository_location + '::test', 'input') + output = self.cmd('create', '--list', '--exclude-nodump', self.repository_location + '::test', 'input') self.assert_in("A input/file1", output) self.assert_in("A input/file2", output) if has_lchflags: self.assert_in("x input/file3", output) # should find second file as excluded - output = self.cmd('create', '--list', self.repository_location + '::test1', 'input', '--exclude', '*/file2') + output = self.cmd('create', '--list', '--exclude-nodump', self.repository_location + '::test1', 'input', '--exclude', '*/file2') self.assert_in("U input/file1", output) self.assert_in("x input/file2", output) if has_lchflags: @@ -2059,8 +2059,8 @@ def has_noatime(some_file): self.cmd('init', '--encryption=repokey', self.repository_location) self.create_test_files() have_noatime = has_noatime('input/file1') - self.cmd('create', self.repository_location + '::archive', 'input') - self.cmd('create', self.repository_location + '::archive2', 'input') + self.cmd('create', '--exclude-nodump', self.repository_location + '::archive', 'input') + self.cmd('create', '--exclude-nodump', self.repository_location + '::archive2', 'input') if has_lchflags: # remove the file we did not backup, so input and output become equal os.remove(os.path.join('input', 'flagfile')) From d535c6a25d7c43723979e0924fca4f6babb01377 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 17 Oct 2017 06:39:31 +0200 Subject: [PATCH 093/798] remove hardlinked symlinks warning, update docs, fixes #3175 the warning was annoying for people with a lot of such items and they can not do anything about it anyway. thus, just document this as a limitation. (cherry picked from commit e674822888d12932216cfd9ff15f18ad77676d67) --- docs/faq.rst | 3 +++ src/borg/archive.py | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index e23f45a307..6726e15bc1 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -97,6 +97,9 @@ Which file types, attributes, etc. are *not* preserved? Archive extraction has optional support to extract all-zero chunks as holes in a sparse file. * Some filesystem specific attributes, like btrfs NOCOW, see :ref:`platforms`. + * For hardlinked symlinks, the hardlinking can not be archived (and thus, + the hardlinking will not be done at extraction time). The symlinks will + be archived and extracted as non-hardlinked symlinks, see :issue:`2379`. Are there other known limitations? ---------------------------------- diff --git a/src/borg/archive.py b/src/borg/archive.py index ca8bc95c47..598cc3f153 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -883,12 +883,11 @@ def process_dev(self, path, st, dev_type): def process_symlink(self, path, st): # note: using hardlinkable=False because we can not support hardlinked symlinks, # due to the dual-use of item.source, see issue #2343: + # hardlinked symlinks will be archived [and extracted] as non-hardlinked symlinks. with self.create_helper(path, st, 's', hardlinkable=False) as (item, status, hardlinked, hardlink_master): with backup_io('readlink'): source = os.readlink(path) item.source = source - if st.st_nlink > 1: - logger.warning('hardlinked symlinks will be archived as non-hardlinked symlinks!') item.update(self.stat_attrs(st, path)) return status From 75004e9065517c674707e580a0e56ca37e165085 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 20 Oct 2017 04:54:09 +0200 Subject: [PATCH 094/798] show excluded dir with "x" for tagged dirs / caches, fixes #3189 (cherry picked from commit 93de42bb256833b441a58afab674b911baf1573b) --- src/borg/archiver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 16f5836808..ff78a360d4 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -583,6 +583,7 @@ def _process(self, archive, cache, matcher, exclude_caches, exclude_if_present, self._process(archive, cache, matcher, exclude_caches, exclude_if_present, keep_exclude_tags, skip_inodes, tag_path, restrict_dev, read_special=read_special, dry_run=dry_run) + self.print_file_status('x', path) return if not dry_run: if not recurse_excluded_dir: From 6d67a9f874c31d60d481258c26480a8c60718757 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 20 Oct 2017 21:13:11 +0200 Subject: [PATCH 095/798] fix crash with relative BORG_KEY_FILE, fixes #3197 (cherry picked from commit 7136e2c93c4de7d2500dfe5f3d1e5120b58eee21) --- src/borg/crypto/key.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/borg/crypto/key.py b/src/borg/crypto/key.py index 53a4c3df91..ed1d44fe71 100644 --- a/src/borg/crypto/key.py +++ b/src/borg/crypto/key.py @@ -695,7 +695,7 @@ def find_key(self): id = self.repository.id keyfile = os.environ.get('BORG_KEY_FILE') if keyfile: - return self.sanity_check(keyfile, id) + return self.sanity_check(os.path.abspath(keyfile), id) keys_dir = get_keys_dir() for name in os.listdir(keys_dir): filename = os.path.join(keys_dir, name) @@ -708,7 +708,7 @@ def find_key(self): def get_new_target(self, args): keyfile = os.environ.get('BORG_KEY_FILE') if keyfile: - return keyfile + return os.path.abspath(keyfile) filename = args.location.to_key_filename() path = filename i = 1 From 2aa676f85cbf24ebf310bc3d738d559da3b15a83 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 20 Oct 2017 16:47:32 +0200 Subject: [PATCH 096/798] fix server-side IndexError for 4-arg open() of old clients, fixes #3192 borg 1.1(.0) server didn't support the 4 argument open() calls made by < 1.0.7 clients. (cherry picked from commit dbcc870489a8fe3cc3517a438e21331feef63538) --- src/borg/remote.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/borg/remote.py b/src/borg/remote.py index db811c2760..7675864f7a 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -180,7 +180,15 @@ def __init__(self, restrict_to_paths, restrict_to_repositories, append_only, sto def positional_to_named(self, method, argv): """Translate from positional protocol to named protocol.""" - return {name: argv[pos] for pos, name in enumerate(compatMap[method])} + try: + return {name: argv[pos] for pos, name in enumerate(compatMap[method])} + except IndexError: + if method == 'open' and len(argv) == 4: + # borg clients < 1.0.7 use open() with 4 args + mapping = compatMap[method][:4] + else: + raise + return {name: argv[pos] for pos, name in enumerate(mapping)} def filter_args(self, f, kwargs): """Remove unknown named parameters from call, because client did (implicitly) say it's ok.""" From 8ade3a960a1362c731d3c8b436d5e51de1be20a8 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 14 Oct 2017 14:38:20 +0200 Subject: [PATCH 097/798] update CHANGES (1.1-maint) --- docs/changes.rst | 66 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/docs/changes.rst b/docs/changes.rst index 3e10a5aa2e..dcde0d2a31 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -131,6 +131,72 @@ The best check that everything is ok is to run a dry-run extraction:: Changelog ========= +Version 1.1.1 (2017-10-22) +-------------------------- + +Compatibility notes: + +- When upgrading from borg 1.0.x to 1.1.x, please note: + + - read all the compatibility notes for 1.1.0*, starting from 1.1.0b1. + - borg might ask some security-related questions once after upgrading. + You can answer them either manually or via environment variable. + One known case is if you use unencrypted repositories, then it will ask + about a unknown unencrypted repository one time. + - your first backup with 1.1.x might be significantly slower (it might + completely read, chunk, hash a lot files) - this is due to the + --files-cache mode change (and happens every time you change mode). + You can avoid the one-time slowdown by using the pre-1.1.0rc4-compatible + mode (but that is less safe for detecting changed files than the default). + See the --files-cache docs for details. +- The deprecated --no-files-cache is not a global/common option any more, + but only available for borg create (it is not needed for anything else). + Use --files-cache=disabled instead of --no-files-cache. +- The nodump flag ("do not backup this file") is not honoured any more by + default because this functionality (esp. if it happened by error or + unexpected) was rather confusing and unexplainable at first to users. + If you want that "do not backup NODUMP-flagged files" behaviour, use: + borg create --exclude-nodump ... + +Fixes: + +- borg recreate: correctly compute part file sizes. fixes cosmetic, but + annoying issue as borg check complains about size inconsistencies of part + files in affected archives. you can solve that by running borg recreate on + these archives, see also #3157. +- bsdflags support: do not open BLK/CHR/LNK files, avoid crashes and + slowness, #3130 +- recreate: don't crash on attic archives w/o time_end, #3109 +- don't crash on repository filesystems w/o hardlink support, #3107 +- don't crash in first part of truncate_and_unlink, #3117 +- fix server-side IndexError crash with clients < 1.0.7, #3192 +- don't show traceback if only a global option is given, show help, #3142 +- cache: use SaveFile for more safety, #3158 +- init: fix wrong encryption choices in command line parser, fix missing + "authenticated-blake2", #3103 +- move --no-files-cache from common to borg create options, #3146 +- fix detection of non-local path (failed on ..filename), #3108 +- logging with fileConfig: set json attr on "borg" logger, #3114 +- fix crash with relative BORG_KEY_FILE, #3197 +- show excluded dir with "x" for tagged dirs / caches, #3189 + +New features: + +- create: --nobsdflags and --exclude-nodump options, #3160 +- extract: --nobsdflags option, #3160 + +Other changes: + +- remove annoying hardlinked symlinks warning, #3175 +- vagrant: use self-made FreeBSD 10.3 box, #3022 +- travis: don't brew update, hopefully fixes #2532 +- docs: + + - readme: -e option is required in borg 1.1 + - add example showing --show-version --show-rc + - use --format rather than --list-format (deprecated) in example + - update docs about hardlinked symlinks limitation + Version 1.1.0 (2017-10-07) -------------------------- From f53bd78b334807dcb5b893bcaf49c78fd7735833 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 22 Oct 2017 03:18:40 +0200 Subject: [PATCH 098/798] build_usage --- docs/usage/common-options.rst.inc | 1 - docs/usage/create.rst.inc | 9 +++++++++ docs/usage/extract.rst.inc | 3 +++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/usage/common-options.rst.inc b/docs/usage/common-options.rst.inc index 093041c47c..f6e191b0b2 100644 --- a/docs/usage/common-options.rst.inc +++ b/docs/usage/common-options.rst.inc @@ -10,7 +10,6 @@ --lock-wait SECONDS wait at most SECONDS for acquiring a repository/cache lock (default: 1). --show-version show/log the borg version --show-rc show/log the return code (rc) ---no-files-cache do not load/update the file metadata cache used to detect unchanged files --umask M set umask to M (local and remote, default: 0077) --remote-path PATH use PATH as borg executable on the remote (default: "borg") --remote-ratelimit RATE set remote network upload rate limit in kiByte/s (default: 0=unlimited) diff --git a/docs/usage/create.rst.inc b/docs/usage/create.rst.inc index 444e4e2e94..1ef6c23b5b 100644 --- a/docs/usage/create.rst.inc +++ b/docs/usage/create.rst.inc @@ -33,6 +33,8 @@ borg create +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``--no-cache-sync`` | experimental: do not synchronize the cache. Implies not using the files cache. | +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``--no-files-cache`` | do not load/update the file metadata cache used to detect unchanged files | + +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------+ | .. class:: borg-common-opt-ref | | | | :ref:`common_options` | @@ -53,6 +55,8 @@ borg create +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``--keep-exclude-tags``, ``--keep-tag-files`` | if tag objects are specified with ``--exclude-if-present``, don't omit the tag objects themselves from the backup archive | +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``--exclude-nodump`` | exclude files flagged NODUMP | + +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------+ | **Filesystem options** | +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``-x``, ``--one-file-system`` | stay in the same file system and do not store mount points of other file systems | @@ -63,6 +67,8 @@ borg create +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``--noctime`` | do not store ctime into archive | +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``--nobsdflags`` | do not read and store bsdflags (e.g. NODUMP, IMMUTABLE) into archive | + +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``--ignore-inode`` | ignore inode data in the file metadata cache used to detect unchanged files. | +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``--files-cache MODE`` | operate files cache in MODE. default: ctime,size,inode | @@ -105,6 +111,7 @@ borg create --filter STATUSCHARS only display items with the given status characters (see description) --json output stats as JSON. Implies ``--stats``. --no-cache-sync experimental: do not synchronize the cache. Implies not using the files cache. + --no-files-cache do not load/update the file metadata cache used to detect unchanged files :ref:`common_options` @@ -118,6 +125,7 @@ borg create --exclude-caches exclude directories that contain a CACHEDIR.TAG file (http://www.brynosaurus.com/cachedir/spec.html) --exclude-if-present NAME exclude directories that are tagged by containing a filesystem object with the given NAME --keep-exclude-tags, --keep-tag-files if tag objects are specified with ``--exclude-if-present``, don't omit the tag objects themselves from the backup archive + --exclude-nodump exclude files flagged NODUMP Filesystem options @@ -125,6 +133,7 @@ borg create --numeric-owner only store numeric user and group identifiers --noatime do not store atime into archive --noctime do not store ctime into archive + --nobsdflags do not read and store bsdflags (e.g. NODUMP, IMMUTABLE) into archive --ignore-inode ignore inode data in the file metadata cache used to detect unchanged files. --files-cache MODE operate files cache in MODE. default: ctime,size,inode --read-special open and read block and char device files as well as FIFOs as if they were regular files. Also follows symlinks pointing to these kinds of files. diff --git a/docs/usage/extract.rst.inc b/docs/usage/extract.rst.inc index ef9a0c75c9..495f20fca9 100644 --- a/docs/usage/extract.rst.inc +++ b/docs/usage/extract.rst.inc @@ -27,6 +27,8 @@ borg extract +-------------------------------------------------------+---------------------------------------+-----------------------------------------------------------------------------------------------------------+ | | ``--numeric-owner`` | only obey numeric user and group identifiers | +-------------------------------------------------------+---------------------------------------+-----------------------------------------------------------------------------------------------------------+ + | | ``--nobsdflags`` | do not extract/set bsdflags (e.g. NODUMP, IMMUTABLE) | + +-------------------------------------------------------+---------------------------------------+-----------------------------------------------------------------------------------------------------------+ | | ``--stdout`` | write all extracted data to stdout | +-------------------------------------------------------+---------------------------------------+-----------------------------------------------------------------------------------------------------------+ | | ``--sparse`` | create holes in output sparse file from all-zero chunks | @@ -68,6 +70,7 @@ borg extract --list output verbose list of items (files, dirs, ...) -n, --dry-run do not actually change any files --numeric-owner only obey numeric user and group identifiers + --nobsdflags do not extract/set bsdflags (e.g. NODUMP, IMMUTABLE) --stdout write all extracted data to stdout --sparse create holes in output sparse file from all-zero chunks From 23ac694e3dc6eb8a14ae22f588cb0a409a52d9f0 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 22 Oct 2017 03:19:23 +0200 Subject: [PATCH 099/798] build_man --- docs/man/borg-benchmark-crud.1 | 2 +- docs/man/borg-benchmark.1 | 2 +- docs/man/borg-break-lock.1 | 2 +- docs/man/borg-change-passphrase.1 | 2 +- docs/man/borg-check.1 | 2 +- docs/man/borg-common.1 | 5 +---- docs/man/borg-compression.1 | 2 +- docs/man/borg-create.1 | 11 ++++++++++- docs/man/borg-delete.1 | 2 +- docs/man/borg-diff.1 | 2 +- docs/man/borg-export-tar.1 | 2 +- docs/man/borg-extract.1 | 5 ++++- docs/man/borg-info.1 | 2 +- docs/man/borg-init.1 | 2 +- docs/man/borg-key-change-passphrase.1 | 2 +- docs/man/borg-key-export.1 | 2 +- docs/man/borg-key-import.1 | 2 +- docs/man/borg-key-migrate-to-repokey.1 | 2 +- docs/man/borg-key.1 | 2 +- docs/man/borg-list.1 | 4 ++-- docs/man/borg-mount.1 | 2 +- docs/man/borg-patterns.1 | 2 +- docs/man/borg-placeholders.1 | 2 +- docs/man/borg-prune.1 | 2 +- docs/man/borg-recreate.1 | 2 +- docs/man/borg-rename.1 | 2 +- docs/man/borg-serve.1 | 2 +- docs/man/borg-umount.1 | 2 +- docs/man/borg-upgrade.1 | 2 +- docs/man/borg-with-lock.1 | 2 +- 30 files changed, 43 insertions(+), 34 deletions(-) diff --git a/docs/man/borg-benchmark-crud.1 b/docs/man/borg-benchmark-crud.1 index 98e58266f1..c9f1334229 100644 --- a/docs/man/borg-benchmark-crud.1 +++ b/docs/man/borg-benchmark-crud.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-BENCHMARK-CRUD 1 "2017-10-07" "" "borg backup tool" +.TH BORG-BENCHMARK-CRUD 1 "2017-10-22" "" "borg backup tool" .SH NAME borg-benchmark-crud \- Benchmark Create, Read, Update, Delete for archives. . diff --git a/docs/man/borg-benchmark.1 b/docs/man/borg-benchmark.1 index 7cdd889edb..1feed69bed 100644 --- a/docs/man/borg-benchmark.1 +++ b/docs/man/borg-benchmark.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-BENCHMARK 1 "2017-10-07" "" "borg backup tool" +.TH BORG-BENCHMARK 1 "2017-10-22" "" "borg backup tool" .SH NAME borg-benchmark \- benchmark command . diff --git a/docs/man/borg-break-lock.1 b/docs/man/borg-break-lock.1 index a7b87aeaa7..8f85ade5d5 100644 --- a/docs/man/borg-break-lock.1 +++ b/docs/man/borg-break-lock.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-BREAK-LOCK 1 "2017-10-07" "" "borg backup tool" +.TH BORG-BREAK-LOCK 1 "2017-10-22" "" "borg backup tool" .SH NAME borg-break-lock \- Break the repository lock (e.g. in case it was left by a dead borg. . diff --git a/docs/man/borg-change-passphrase.1 b/docs/man/borg-change-passphrase.1 index 87be29b543..a377dfaba7 100644 --- a/docs/man/borg-change-passphrase.1 +++ b/docs/man/borg-change-passphrase.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-CHANGE-PASSPHRASE 1 "2017-10-07" "" "borg backup tool" +.TH BORG-CHANGE-PASSPHRASE 1 "2017-10-22" "" "borg backup tool" .SH NAME borg-change-passphrase \- Change repository key file passphrase . diff --git a/docs/man/borg-check.1 b/docs/man/borg-check.1 index 1f675b19b7..e0eadf5ab6 100644 --- a/docs/man/borg-check.1 +++ b/docs/man/borg-check.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-CHECK 1 "2017-10-07" "" "borg backup tool" +.TH BORG-CHECK 1 "2017-10-22" "" "borg backup tool" .SH NAME borg-check \- Check repository consistency . diff --git a/docs/man/borg-common.1 b/docs/man/borg-common.1 index df5eca490c..91e0f7ab71 100644 --- a/docs/man/borg-common.1 +++ b/docs/man/borg-common.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-COMMON 1 "2017-10-07" "" "borg backup tool" +.TH BORG-COMMON 1 "2017-10-22" "" "borg backup tool" .SH NAME borg-common \- Common options of Borg commands . @@ -69,9 +69,6 @@ show/log the borg version .B \-\-show\-rc show/log the return code (rc) .TP -.B \-\-no\-files\-cache -do not load/update the file metadata cache used to detect unchanged files -.TP .BI \-\-umask \ M set umask to M (local and remote, default: 0077) .TP diff --git a/docs/man/borg-compression.1 b/docs/man/borg-compression.1 index 93645abc20..c4a444a3a1 100644 --- a/docs/man/borg-compression.1 +++ b/docs/man/borg-compression.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-COMPRESSION 1 "2017-10-07" "" "borg backup tool" +.TH BORG-COMPRESSION 1 "2017-10-22" "" "borg backup tool" .SH NAME borg-compression \- Details regarding compression . diff --git a/docs/man/borg-create.1 b/docs/man/borg-create.1 index 2afc4e60bb..0c47dc6c99 100644 --- a/docs/man/borg-create.1 +++ b/docs/man/borg-create.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-CREATE 1 "2017-10-07" "" "borg backup tool" +.TH BORG-CREATE 1 "2017-10-22" "" "borg backup tool" .SH NAME borg-create \- Create new archive . @@ -142,6 +142,9 @@ output stats as JSON. Implies \fB\-\-stats\fP\&. .TP .B \-\-no\-cache\-sync experimental: do not synchronize the cache. Implies not using the files cache. +.TP +.B \-\-no\-files\-cache +do not load/update the file metadata cache used to detect unchanged files .UNINDENT .SS Exclusion options .INDENT 0.0 @@ -166,6 +169,9 @@ exclude directories that are tagged by containing a filesystem object with the g .TP .B \-\-keep\-exclude\-tags\fP,\fB \-\-keep\-tag\-files if tag objects are specified with \fB\-\-exclude\-if\-present\fP, don\(aqt omit the tag objects themselves from the backup archive +.TP +.B \-\-exclude\-nodump +exclude files flagged NODUMP .UNINDENT .SS Filesystem options .INDENT 0.0 @@ -182,6 +188,9 @@ do not store atime into archive .B \-\-noctime do not store ctime into archive .TP +.B \-\-nobsdflags +do not read and store bsdflags (e.g. NODUMP, IMMUTABLE) into archive +.TP .B \-\-ignore\-inode ignore inode data in the file metadata cache used to detect unchanged files. .TP diff --git a/docs/man/borg-delete.1 b/docs/man/borg-delete.1 index 76daae71a0..013ab288a5 100644 --- a/docs/man/borg-delete.1 +++ b/docs/man/borg-delete.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-DELETE 1 "2017-10-07" "" "borg backup tool" +.TH BORG-DELETE 1 "2017-10-22" "" "borg backup tool" .SH NAME borg-delete \- Delete an existing repository or archives . diff --git a/docs/man/borg-diff.1 b/docs/man/borg-diff.1 index a953b184d5..9a96832443 100644 --- a/docs/man/borg-diff.1 +++ b/docs/man/borg-diff.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-DIFF 1 "2017-10-07" "" "borg backup tool" +.TH BORG-DIFF 1 "2017-10-22" "" "borg backup tool" .SH NAME borg-diff \- Diff contents of two archives . diff --git a/docs/man/borg-export-tar.1 b/docs/man/borg-export-tar.1 index d2ee75a2b9..3691b3c3b9 100644 --- a/docs/man/borg-export-tar.1 +++ b/docs/man/borg-export-tar.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-EXPORT-TAR 1 "2017-10-07" "" "borg backup tool" +.TH BORG-EXPORT-TAR 1 "2017-10-22" "" "borg backup tool" .SH NAME borg-export-tar \- Export archive contents as a tarball . diff --git a/docs/man/borg-extract.1 b/docs/man/borg-extract.1 index 2594178275..e17adddf5a 100644 --- a/docs/man/borg-extract.1 +++ b/docs/man/borg-extract.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-EXTRACT 1 "2017-10-07" "" "borg backup tool" +.TH BORG-EXTRACT 1 "2017-10-22" "" "borg backup tool" .SH NAME borg-extract \- Extract archive contents . @@ -80,6 +80,9 @@ do not actually change any files .B \-\-numeric\-owner only obey numeric user and group identifiers .TP +.B \-\-nobsdflags +do not extract/set bsdflags (e.g. NODUMP, IMMUTABLE) +.TP .B \-\-stdout write all extracted data to stdout .TP diff --git a/docs/man/borg-info.1 b/docs/man/borg-info.1 index c15b62ddd0..0c9cce9566 100644 --- a/docs/man/borg-info.1 +++ b/docs/man/borg-info.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-INFO 1 "2017-10-07" "" "borg backup tool" +.TH BORG-INFO 1 "2017-10-22" "" "borg backup tool" .SH NAME borg-info \- Show archive details such as disk space used . diff --git a/docs/man/borg-init.1 b/docs/man/borg-init.1 index 9c897e5e96..ccf81dbfd7 100644 --- a/docs/man/borg-init.1 +++ b/docs/man/borg-init.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-INIT 1 "2017-10-07" "" "borg backup tool" +.TH BORG-INIT 1 "2017-10-22" "" "borg backup tool" .SH NAME borg-init \- Initialize an empty repository . diff --git a/docs/man/borg-key-change-passphrase.1 b/docs/man/borg-key-change-passphrase.1 index 345d43c959..f76707ddb0 100644 --- a/docs/man/borg-key-change-passphrase.1 +++ b/docs/man/borg-key-change-passphrase.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-CHANGE-PASSPHRASE 1 "2017-10-07" "" "borg backup tool" +.TH BORG-KEY-CHANGE-PASSPHRASE 1 "2017-10-22" "" "borg backup tool" .SH NAME borg-key-change-passphrase \- Change repository key file passphrase . diff --git a/docs/man/borg-key-export.1 b/docs/man/borg-key-export.1 index 5bcf0c9770..6cea595f24 100644 --- a/docs/man/borg-key-export.1 +++ b/docs/man/borg-key-export.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-EXPORT 1 "2017-10-07" "" "borg backup tool" +.TH BORG-KEY-EXPORT 1 "2017-10-22" "" "borg backup tool" .SH NAME borg-key-export \- Export the repository key for backup . diff --git a/docs/man/borg-key-import.1 b/docs/man/borg-key-import.1 index c2e0d1c3c6..ee08b97954 100644 --- a/docs/man/borg-key-import.1 +++ b/docs/man/borg-key-import.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-IMPORT 1 "2017-10-07" "" "borg backup tool" +.TH BORG-KEY-IMPORT 1 "2017-10-22" "" "borg backup tool" .SH NAME borg-key-import \- Import the repository key from backup . diff --git a/docs/man/borg-key-migrate-to-repokey.1 b/docs/man/borg-key-migrate-to-repokey.1 index 6ee88b09f5..154fcfe12d 100644 --- a/docs/man/borg-key-migrate-to-repokey.1 +++ b/docs/man/borg-key-migrate-to-repokey.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-MIGRATE-TO-REPOKEY 1 "2017-10-07" "" "borg backup tool" +.TH BORG-KEY-MIGRATE-TO-REPOKEY 1 "2017-10-22" "" "borg backup tool" .SH NAME borg-key-migrate-to-repokey \- Migrate passphrase -> repokey . diff --git a/docs/man/borg-key.1 b/docs/man/borg-key.1 index 9a6abcdd4c..32b4d701d8 100644 --- a/docs/man/borg-key.1 +++ b/docs/man/borg-key.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY 1 "2017-10-07" "" "borg backup tool" +.TH BORG-KEY 1 "2017-10-22" "" "borg backup tool" .SH NAME borg-key \- Manage a keyfile or repokey of a repository . diff --git a/docs/man/borg-list.1 b/docs/man/borg-list.1 index 6583f3fe34..409a74b23f 100644 --- a/docs/man/borg-list.1 +++ b/docs/man/borg-list.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-LIST 1 "2017-10-07" "" "borg backup tool" +.TH BORG-LIST 1 "2017-10-22" "" "borg backup tool" .SH NAME borg-list \- List archive or repository contents . @@ -128,7 +128,7 @@ lrwxrwxrwx root root 0 Fri, 2015\-03\-27 20:24:26 bin/bzcmp \-> bzdif \-rwxr\-xr\-x root root 2140 Fri, 2015\-03\-27 20:24:22 bin/bzdiff \&... -$ borg list /path/to/repo::archiveA \-\-list\-format="{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}" +$ borg list /path/to/repo::archiveA \-\-format="{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}" drwxrwxr\-x user user 0 Sun, 2015\-02\-01 11:00:00 . drwxrwxr\-x user user 0 Sun, 2015\-02\-01 11:00:00 code drwxrwxr\-x user user 0 Sun, 2015\-02\-01 11:00:00 code/myproject diff --git a/docs/man/borg-mount.1 b/docs/man/borg-mount.1 index d10ca21ae6..1bc624f40d 100644 --- a/docs/man/borg-mount.1 +++ b/docs/man/borg-mount.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-MOUNT 1 "2017-10-07" "" "borg backup tool" +.TH BORG-MOUNT 1 "2017-10-22" "" "borg backup tool" .SH NAME borg-mount \- Mount archive or an entire repository as a FUSE filesystem . diff --git a/docs/man/borg-patterns.1 b/docs/man/borg-patterns.1 index 3e5a718e40..531ad5556c 100644 --- a/docs/man/borg-patterns.1 +++ b/docs/man/borg-patterns.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-PATTERNS 1 "2017-10-07" "" "borg backup tool" +.TH BORG-PATTERNS 1 "2017-10-22" "" "borg backup tool" .SH NAME borg-patterns \- Details regarding patterns . diff --git a/docs/man/borg-placeholders.1 b/docs/man/borg-placeholders.1 index f6177d915b..31b5fb355d 100644 --- a/docs/man/borg-placeholders.1 +++ b/docs/man/borg-placeholders.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-PLACEHOLDERS 1 "2017-10-07" "" "borg backup tool" +.TH BORG-PLACEHOLDERS 1 "2017-10-22" "" "borg backup tool" .SH NAME borg-placeholders \- Details regarding placeholders . diff --git a/docs/man/borg-prune.1 b/docs/man/borg-prune.1 index 96cd24419f..e2ffdb2540 100644 --- a/docs/man/borg-prune.1 +++ b/docs/man/borg-prune.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-PRUNE 1 "2017-10-07" "" "borg backup tool" +.TH BORG-PRUNE 1 "2017-10-22" "" "borg backup tool" .SH NAME borg-prune \- Prune repository archives according to specified rules . diff --git a/docs/man/borg-recreate.1 b/docs/man/borg-recreate.1 index 584d56f411..0768cd7485 100644 --- a/docs/man/borg-recreate.1 +++ b/docs/man/borg-recreate.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-RECREATE 1 "2017-10-07" "" "borg backup tool" +.TH BORG-RECREATE 1 "2017-10-22" "" "borg backup tool" .SH NAME borg-recreate \- Re-create archives . diff --git a/docs/man/borg-rename.1 b/docs/man/borg-rename.1 index 1050795214..c628299f18 100644 --- a/docs/man/borg-rename.1 +++ b/docs/man/borg-rename.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-RENAME 1 "2017-10-07" "" "borg backup tool" +.TH BORG-RENAME 1 "2017-10-22" "" "borg backup tool" .SH NAME borg-rename \- Rename an existing archive . diff --git a/docs/man/borg-serve.1 b/docs/man/borg-serve.1 index a5cd6c9aa1..3a567ee840 100644 --- a/docs/man/borg-serve.1 +++ b/docs/man/borg-serve.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-SERVE 1 "2017-10-07" "" "borg backup tool" +.TH BORG-SERVE 1 "2017-10-22" "" "borg backup tool" .SH NAME borg-serve \- Start in server mode. This command is usually not used manually. . diff --git a/docs/man/borg-umount.1 b/docs/man/borg-umount.1 index 5c53d501ac..932a407500 100644 --- a/docs/man/borg-umount.1 +++ b/docs/man/borg-umount.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-UMOUNT 1 "2017-10-07" "" "borg backup tool" +.TH BORG-UMOUNT 1 "2017-10-22" "" "borg backup tool" .SH NAME borg-umount \- un-mount the FUSE filesystem . diff --git a/docs/man/borg-upgrade.1 b/docs/man/borg-upgrade.1 index a9cd99cbce..0d5ef5905b 100644 --- a/docs/man/borg-upgrade.1 +++ b/docs/man/borg-upgrade.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-UPGRADE 1 "2017-10-07" "" "borg backup tool" +.TH BORG-UPGRADE 1 "2017-10-22" "" "borg backup tool" .SH NAME borg-upgrade \- upgrade a repository from a previous version . diff --git a/docs/man/borg-with-lock.1 b/docs/man/borg-with-lock.1 index 0f84a7ff7e..27321d56a8 100644 --- a/docs/man/borg-with-lock.1 +++ b/docs/man/borg-with-lock.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-WITH-LOCK 1 "2017-10-07" "" "borg backup tool" +.TH BORG-WITH-LOCK 1 "2017-10-22" "" "borg backup tool" .SH NAME borg-with-lock \- run a user specified command with the repository lock held . From 5ce4fa970413e3cedbc36e2f728863eaa69dad21 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 22 Oct 2017 17:34:02 +0200 Subject: [PATCH 100/798] arg parsing: fix fallback function, refactor, fixes #3205 This is a fixup for #3155, which was broken on at least python <= 3.4.2. Also clarify when to use which *func in set_defaults. --- src/borg/archiver.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index ff78a360d4..84c913a5e4 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -169,6 +169,18 @@ def parse_storage_quota(storage_quota): return parsed +def get_func(args): + # This works around http://bugs.python.org/issue9351 + # func is used at the leaf parsers of the argparse parser tree, + # fallback_func at next level towards the root, + # fallback2_func at the 2nd next level (which is root in our case). + for name in 'func', 'fallback_func', 'fallback2_func': + func = getattr(args, name, None) + if func is not None: + return func + raise Exception('expected func attributes not found') + + class Archiver: def __init__(self, lock_wait=None, prog=None): @@ -2429,7 +2441,7 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): parser = argparse.ArgumentParser(prog=self.prog, description='Borg - Deduplicated Backups', add_help=False) - parser.set_defaults(func=functools.partial(self.do_maincommand_help, parser)) + parser.set_defaults(fallback2_func=functools.partial(self.do_maincommand_help, parser)) parser.common_options = self.CommonOptions(define_common_options, suffix_precedence=('_maincommand', '_midcommand', '_subcommand')) parser.add_argument('-V', '--version', action='version', version='%(prog)s ' + __version__, @@ -3884,8 +3896,7 @@ def parse_args(self, args=None): parser = self.build_parser() args = parser.parse_args(args or ['-h']) parser.common_options.resolve(args) - # This works around http://bugs.python.org/issue9351 - func = getattr(args, 'func', None) or getattr(args, 'fallback_func') + func = get_func(args) if func == self.do_create and not args.paths: # need at least 1 path but args.paths may also be populated from patterns parser.error('Need at least one PATH argument.') @@ -3921,8 +3932,7 @@ def _setup_topic_debugging(self, args): def run(self, args): os.umask(args.umask) # early, before opening files self.lock_wait = args.lock_wait - # This works around http://bugs.python.org/issue9351 - func = getattr(args, 'func', None) or getattr(args, 'fallback_func') + func = get_func(args) # do not use loggers before this! is_serve = func == self.do_serve setup_logging(level=args.log_level, is_serve=is_serve, json=args.log_json) From 5b7d4392dca4cc704e1df664d461f73787ecb24b Mon Sep 17 00:00:00 2001 From: Milkey Mouse Date: Wed, 25 Oct 2017 19:42:02 -0700 Subject: [PATCH 101/798] Clarify using a blank passphrase in keyfile mode --- docs/faq.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 6726e15bc1..442df6f66e 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -304,7 +304,7 @@ Setting ``BORG_PASSPHRASE`` .. _password_env: .. note:: Be careful how you set the environment; using the ``env`` command, a ``system()`` call or using inline shell scripts - (e.g. ``BORG_PASSPHRASE=hunter12 borg ...``) + (e.g. ``BORG_PASSPHRASE=hunter2 borg ...``) might expose the credentials in the process list directly and they will be readable to all users on a system. Using ``export`` in a shell script file should be safe, however, as @@ -328,7 +328,8 @@ Using ``BORG_PASSCOMMAND`` with a properly permissioned file Using keyfile-based encryption with a blank passphrase It is possible to encrypt your repository in ``keyfile`` mode instead of the default - ``repokey`` mode and use a blank passphrase for the key file. See :ref:`encrypted_repos` + ``repokey`` mode and use a blank passphrase for the key file (simply press Enter twice + when ``borg init`` asks for the password). See :ref:`encrypted_repos` for more details. Using ``BORG_PASSCOMMAND`` with macOS Keychain From 94fa3fae3f5a23fe513e14ea61e3176962d898b6 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 29 Oct 2017 11:39:13 +0100 Subject: [PATCH 102/798] flake8: add some ignores (cherry picked from commit bb1c7a7f57ddbd3e046524a0de8c161ba807a1fd) --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 93f009af50..8e9414ac6d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,7 +5,7 @@ python_files = testsuite/*.py # please note that the values are adjusted so that they do not cause failures # with existing code. if you want to change them, you should first fix all # flake8 failures that appear with your change. -ignore = E122,E123,E125,E126,E127,E128,E226,E402,F401,F405,F811 +ignore = E122,E123,E125,E126,E127,E128,E226,E402,E722,E741,F401,F405,F811 # line length long term target: 120 max-line-length = 255 exclude = build,dist,.git,.idea,.cache,.tox,docs/conf.py From a43e2c30430cf8e79d5cca49f26885128f49dad2 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 2 Nov 2017 16:03:55 +0100 Subject: [PATCH 103/798] remove evil trailing comma, fixes #3244 value type of compatMap is a tuple of strings. due to the trailing comma, this was a 1-tuple of a tuple of strings. (cherry picked from commit 117d30d17126f2118e43dec2f08d502d3369f6c8) --- src/borg/remote.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/borg/remote.py b/src/borg/remote.py index 7675864f7a..219f8c5c32 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -603,7 +603,7 @@ def do_open(): # emit this msg in the same way as the 'Remote: ...' lines that show the remote TypeError sys.stderr.write(msg) self.server_version = parse_version('1.0.6') - compatMap['open'] = ('path', 'create', 'lock_wait', 'lock', ), + compatMap['open'] = ('path', 'create', 'lock_wait', 'lock', ) # try again with corrected version and compatMap do_open() except Exception: From 338f56b97d2844fe5d57c2869453681bdf2df2a5 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 24 Oct 2017 01:25:53 +0200 Subject: [PATCH 104/798] don't do stat() when not recursing into excluded dir, fixes #3209 also: fix exception handling for the stat() calls just moving all these lines into the "try"-block below (like it was in 1.0). (cherry picked from commit 0c410e84fe9d86b24a6022c13df24005283e70c8) --- src/borg/archiver.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 84c913a5e4..2e7856b048 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -555,20 +555,27 @@ def _process(self, archive, cache, matcher, exclude_caches, exclude_if_present, This should only raise on critical errors. Per-item errors must be handled within this method. """ - if st is None: - with backup_io('stat'): - st = os.stat(path, follow_symlinks=False) - - recurse_excluded_dir = False - if not matcher.match(path): - self.print_file_status('x', path) - - if stat.S_ISDIR(st.st_mode) and matcher.recurse_dir: - recurse_excluded_dir = True + try: + recurse_excluded_dir = False + if matcher.match(path): + if st is None: + with backup_io('stat'): + st = os.stat(path, follow_symlinks=False) else: - return + self.print_file_status('x', path) + # get out here as quickly as possible: + # we only need to continue if we shall recurse into an excluded directory. + # if we shall not recurse, then do not even touch (stat()) the item, it + # could trigger an error, e.g. if access is forbidden, see #3209. + if not matcher.recurse_dir: + return + if st is None: + with backup_io('stat'): + st = os.stat(path, follow_symlinks=False) + recurse_excluded_dir = stat.S_ISDIR(st.st_mode) + if not recurse_excluded_dir: + return - try: if (st.st_ino, st.st_dev) in skip_inodes: return # if restrict_dev is given, we do not want to recurse into a new filesystem, From 0943f2235e66726e714df9407ddd1991ee144065 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 24 Oct 2017 02:38:03 +0200 Subject: [PATCH 105/798] mention "!" (exclude-norecurse) type in patterns help (cherry picked from commit 36f2a4e1dcac69f58e5cfc8141a65b5006f32cf3) --- src/borg/archiver.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 2e7856b048..d978ec1078 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -2026,10 +2026,12 @@ def do_break_lock(self, args, repository): may specify the backup roots (starting points) and patterns for inclusion/exclusion. A root path starts with the prefix `R`, followed by a path (a plain path, not a file pattern). An include rule starts with the prefix +, an exclude rule starts - with the prefix -, both followed by a pattern. + with the prefix -, an exclude-norecurse rule starts with !, all followed by a pattern. Inclusion patterns are useful to include paths that are contained in an excluded path. The first matching pattern is used so if an include pattern matches before - an exclude pattern, the file is backed up. + an exclude pattern, the file is backed up. If an exclude-norecurse pattern matches + a directory, it won't recurse into it and won't discover any potential matches for + include rules below that directory. Note that the default pattern style for ``--pattern`` and ``--patterns-from`` is shell style (`sh:`), so those patterns behave similar to rsync include/exclude From 60d2b9c1931be9561cdbff02dd20af37ea3bb45a Mon Sep 17 00:00:00 2001 From: Narendra Vardi Date: Mon, 16 Oct 2017 00:07:51 +0530 Subject: [PATCH 106/798] add BORG_CONFIG_DIR env (#3083) (cherry picked from commit 56fed4f96490cedc81f764750f63547b0aabde9b) --- src/borg/helpers.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/borg/helpers.py b/src/borg/helpers.py index f1721c52bc..82e77a0eec 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -482,8 +482,7 @@ def get_home_dir(): def get_keys_dir(): """Determine where to repository keys and cache""" - xdg_config = os.environ.get('XDG_CONFIG_HOME', os.path.join(get_home_dir(), '.config')) - keys_dir = os.environ.get('BORG_KEYS_DIR', os.path.join(xdg_config, 'borg', 'keys')) + keys_dir = os.environ.get('BORG_KEYS_DIR', os.path.join(get_config_dir(), 'keys')) if not os.path.exists(keys_dir): os.makedirs(keys_dir) os.chmod(keys_dir, stat.S_IRWXU) @@ -492,8 +491,7 @@ def get_keys_dir(): def get_security_dir(repository_id=None): """Determine where to store local security information.""" - xdg_config = os.environ.get('XDG_CONFIG_HOME', os.path.join(get_home_dir(), '.config')) - security_dir = os.environ.get('BORG_SECURITY_DIR', os.path.join(xdg_config, 'borg', 'security')) + security_dir = os.environ.get('BORG_SECURITY_DIR', os.path.join(get_config_dir(), 'security')) if repository_id: security_dir = os.path.join(security_dir, repository_id) if not os.path.exists(security_dir): @@ -519,6 +517,16 @@ def get_cache_dir(): return cache_dir +def get_config_dir(): + """Determine where to store whole config""" + xdg_config = os.environ.get('XDG_CONFIG_HOME', os.path.join(get_home_dir(), '.config')) + config_dir = os.environ.get('BORG_CONFIG_DIR', os.path.join(xdg_config, 'borg')) + if not os.path.exists(config_dir): + os.makedirs(config_dir) + os.chmod(config_dir, stat.S_IRWXU) + return config_dir + + def to_localtime(ts): """Convert datetime object from UTC to local time zone""" return datetime(*time.localtime((ts - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds())[:6]) From 67e0fbb27310ca0453b19a3c35742695c7cb2213 Mon Sep 17 00:00:00 2001 From: Narendra Vardi Date: Wed, 25 Oct 2017 23:07:45 +0530 Subject: [PATCH 107/798] added unittest for get_config_dir function (cherry picked from commit a3f1e6e2503b7ad658aa53af9c2c0e82a0138b1c) --- src/borg/testsuite/helpers.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/borg/testsuite/helpers.py b/src/borg/testsuite/helpers.py index 2a4160d2c0..15c029f5c5 100644 --- a/src/borg/testsuite/helpers.py +++ b/src/borg/testsuite/helpers.py @@ -18,7 +18,7 @@ from ..helpers import partial_format, format_file_size, parse_file_size, format_timedelta, format_line, PlaceholderError, replace_placeholders from ..helpers import make_path_safe, clean_lines from ..helpers import interval, prune_within, prune_split -from ..helpers import get_cache_dir, get_keys_dir, get_security_dir +from ..helpers import get_cache_dir, get_keys_dir, get_security_dir, get_config_dir from ..helpers import is_slow_msgpack from ..helpers import yes, TRUISH, FALSISH, DEFAULTISH from ..helpers import StableDict, int_to_bigint, bigint_to_int, bin_to_hex @@ -447,6 +447,17 @@ def test(self): self.assert_equal(parse_timestamp('2015-04-19T20:25:00'), datetime(2015, 4, 19, 20, 25, 0, 0, timezone.utc)) +def test_get_config_dir(monkeypatch): + """test that get_config_dir respects environment""" + monkeypatch.delenv('BORG_CONFIG_DIR', raising=False) + monkeypatch.delenv('XDG_CONFIG_HOME', raising=False) + assert get_config_dir() == os.path.join(os.path.expanduser('~'), '.config', 'borg') + monkeypatch.setenv('XDG_CONFIG_HOME', '/var/tmp/.config') + assert get_config_dir() == os.path.join('/var/tmp/.config', 'borg') + monkeypatch.setenv('BORG_CONFIG_DIR', '/var/tmp') + assert get_config_dir() == '/var/tmp' + + def test_get_cache_dir(monkeypatch): """test that get_cache_dir respects environment""" monkeypatch.delenv('BORG_CACHE_DIR', raising=False) From 00152e8a61f68e111158f62c2bea8eba8c7a58f8 Mon Sep 17 00:00:00 2001 From: narendravardi Date: Sun, 29 Oct 2017 21:52:33 +0530 Subject: [PATCH 108/798] Added documentation for BORG_CONFIG_DIR (cherry picked from commit 764a62f42c41ef85fb916773e085f67a7c77eda6) --- docs/usage_general.rst.inc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/usage_general.rst.inc b/docs/usage_general.rst.inc index 3c61914388..c87a8b8b47 100644 --- a/docs/usage_general.rst.inc +++ b/docs/usage_general.rst.inc @@ -197,6 +197,8 @@ Directories and files: BORG_CACHE_DIR Default to '~/.cache/borg'. This directory contains the local cache and might need a lot of space for dealing with big repositories). + BORG_CONFIG_DIR + Default to '~/.config/borg'. This directory contains the whole config directories. Building: BORG_OPENSSL_PREFIX From 8df6aa5bfc819137cb14220e601db5cd431549d7 Mon Sep 17 00:00:00 2001 From: Kevin Date: Sat, 4 Nov 2017 12:02:03 -0400 Subject: [PATCH 109/798] Added ZSH Shell Completion Added ZSH Shell Completion from upstream. https://github.com/mrkmg/borgbackup-zsh-completion --- scripts/shell_completions/zsh/_borg | 390 ++++++++++++++++++++++++++++ 1 file changed, 390 insertions(+) create mode 100644 scripts/shell_completions/zsh/_borg diff --git a/scripts/shell_completions/zsh/_borg b/scripts/shell_completions/zsh/_borg new file mode 100644 index 0000000000..2abb561bc1 --- /dev/null +++ b/scripts/shell_completions/zsh/_borg @@ -0,0 +1,390 @@ +#compdef borg + +# ------- +# +# For borg 1.1.1 +# +# To Install: +# +# Copy this file to /usr/share/zsh/site-functions/ +# +# ------- +# borgbackup ZSH completion +# Kevin Gravier 2017 +# +# The MIT License (MIT) + +# Copyright (c) 2017 Kevin + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + + +_borg() { + typeset -A opt_args + local -a borg_possible_commands borg_possible_key_commands commands keyCommands borg_standard_options + local command keyCommand item subItem + + borg_standard_options=({-h,--help}'[show this help message and exit]' + --critical'[work on log level CRITICAL]' + --error'[work on log level ERROR]' + --warning'[work on log level WARNING (default)]' + {--info,-v,--verbose}'[work on log level INFO]' + --debug'[work on log level DEBUG]' + --default-topic'[enable TOPIC debugging (can be specified multiple times).]:TOPIC' + {-p,--progress}'[show progress information]' + --log-json'[Output one JSON object per log line instead of formatted text.]' + --lock-wait'[wait at most SECONDS for acquiring a repository/cache lock (default: 1).]:SECONDS' + --show-version'[show/log the borg version]' + --show-rc'[show/log the return code (rc)]' + --umask'[set umask to M (local and remote, default: 0077)]:umask' + --remote-path'[set remote path to executable (default: "borg")]:_files' + --remote-ratelimit'[set remote network upload rate limit in kiByte/s (default: 0=unlimited)]:RATE' + --consider-part-files'[treat part files like normal files (e.g. to list/extract them)]' + --debug-profile'[Write execution profile in Borg format into FILE.]:_files') + + borg_possible_commands=(serve init check key change-passphrase create extract export-tar diff rename delete list mount umount info break-lock prune upgrade with-lock benchmark) + borg_possible_key_commands=(change-passphrase import export) + command="" + keyCommand="" + + for item in $words; do + (( ${borg_possible_commands[(I)$item]} )) && command=$item + (( ${borg_possible_key_commands[(I)$item]} )) && keyCommand=$item + done + + case $command in + (serve) + _arguments \ + --restrict-to-path'[restrict repository access to PATH]:_files'\ + --restrict-to-repository'[restrict repository access]:_files'\ + --append-only'[only allow appending to repository segment files]'\ + --storage-quota'[Override storage quota of the repository]:QUOTA'\ + $borg_standard_options + ;; + (init) + _arguments \ + '2:repo:_files'\ + {-e,--encryption}'[select encryption key mode]:MODE'\ + --append-only'[only allow appending to repository segment files]'\ + --storage-quota'[Override storage quota of the repository]:QUOTA'\ + $borg_standard_options + ;; + (check) + _arguments \ + '2:archives:__borg_archive'\ + --repository-only'[only perform repository checks]'\ + --archives-only'[only perform archives checks]'\ + --verify-data'[perform cryptographic archive data integrity verification]'\ + --repair'[attempt to repair any inconsistencies found]'\ + --save-space'[work slower, but using less space]'\ + {-P,--prefix}'[only consider archive names starting with this prefix.]:PREFIX'\ + {-a,--glob-archives}'[only consider archive names matching the glob]:GLOB'\ + --sort-keys'[Comma-separated list of sorting keys]:KEYS'\ + --first'[consider first N archives after other filters were applied]:N'\ + --last'[consider last N archives after other filters were applied]:N'\ + $borg_standard_options + ;; + (key) + case $keyCommand in + (change-passphrase) + _arguments \ + '2:subCommand:(change-passphrase import export)'\ + '3:archives:__borg_archive'\ + $borg_standard_options + ;; + (import) + _arguments \ + '2:subCommand:(change-passphrase import export)'\ + '3:archives:__borg_archive'\ + '4:path:_files'\ + --paper'[interactively import from a backup done with --paper]'\ + $borg_standard_options + ;; + (export) + _arguments \ + '2:subCommand:(change-passphrase import export)'\ + '3:archives:__borg_archive'\ + '4:path:_files'\ + --paper'[Create an export suitable for printing and later type-in]'\ + --qr-html'[Create an html file suitable for printing and later type-in or qr scan]'\ + $borg_standard_options + ;; + *) + _arguments \ + '2:subCommand:(change-passphrase import export)'\ + $borg_standard_options + ;; + esac + ;; + (change-passphrase) + _arguments \ + '2:archives:__borg_archive'\ + $borg_standard_options + ;; + (create) + _arguments \ + '2:archives:__borg_archive'\ + '3:path:_files'\ + {-n,--dry-run}'[do not create a backup archive]'\ + {-s,--stats}'[print statistics for the created archive]'\ + --list'[output verbose list of items (files, dirs, ...)]'\ + --filter'[only display items with the given status characters]:STATUSCHARS'\ + --json'[output stats as JSON. Implies --stats.]'\ + --no-cache-sync'[experimental: do not synchronize the cache. Implies not using the files cache.]'\ + {-x,--one-file-system}'[stay in the same file system ]'\ + --numeric-owner'[only store numeric user and group identifiers]'\ + --noatime'[do not store atime into archive]'\ + --noctime'[do not store ctime into archive]'\ + --nobsdflags'[do not read and store bsdflags (e.g. NODUMP, IMMUTABLE) into archive]'\ + --ignore-inode'[gnore inode data in the file metadata cache used to detect unchanged files.]'\ + --files-cache'[operate files cache in MODE. default: ctime,size,inode]:MODE'\ + --read-special'[open and read block and char device files as well as FIFOs as if they were regular files.]'\ + {-e,--exclude}'[exclude paths matching PATTERN]:PATTERN'\ + --exclude-from'[read exclude patterns from EXCLUDEFILE, one per line]:_files'\ + --pattern'[experimental: include/exclude paths matching PATTERN]:PATTERN'\ + --patterns-from'[experimental: read include/exclude patterns from PATTERNFILE, one per line]:_files'\ + --exclude-caches'[exclude directories that contain a CACHEDIR.TAG file ]'\ + --exclude-if-present'[exclude directories that are tagged by containing a filesystem object with the given NAME]:NAME'\ + {--keep-exclude-tags,--keep-tag-files}'[if tag objects are specified with --exclude-if-present, don’t omit the tag objects themselves]'\ + --exclude-nodump'[exclude files flagged NODUMP]'\ + --comment'[add a comment text to the archive]:COMMENT'\ + --timestamp'[manually specify the archive creation date/time]:TIMESTAMP'\ + {-c,--checkpoint-interval}'[write checkpoint every SECONDS seconds]:SECONDS'\ + --chunker-params'[pecify the chunker parameters]:PARAMS'\ + {-C,--compression}'[select compression algorithm]:COMPRESSION'\ + $borg_standard_options + ;; + (extract) + _arguments \ + '2:archives:__borg_archive'\ + '3:path:_files'\ + --list'[output verbose list of items (files, dirs, ...)]'\ + {-n,--dry-run}'[do not actually change any files]'\ + --numeric-owner'[only obey numeric user and group identifiers]'\ + --nobsdflags'[do not extract/set bsdflags (e.g. NODUMP, IMMUTABLE)]'\ + --stdout'[write all extracted data to stdout]'\ + --sparse'[create holes in output sparse file from all-zero chunks]'\ + {-e,--exclude}'[exclude paths matching PATTERN]:PATTERN'\ + --exclude-from'[read exclude patterns from EXCLUDEFILE, one per line]:_files'\ + --pattern'[experimental: include/exclude paths matching PATTERN]:PATTERN'\ + --patterns-from'[experimental: read include/exclude patterns from PATTERNFILE, one per line]:_files'\ + --strip-components'[Remove the specified number of leading path elements. Paths with fewer elements will be silently skipped.]:NUMBER'\ + $borg_standard_options + ;; + (export-tar) + _arguments \ + '2:archives:__borg_archive'\ + '3:tar:_files'\ + '4:path:_files'\ + --tar-filter'[filter program to pipe data through]'\ + --list'[output verbose list of items (files, dirs, ...)]'\ + {-e,--exclude}'[exclude paths matching PATTERN]:PATTERN'\ + --exclude-from'[read exclude patterns from EXCLUDEFILE, one per line]:_files'\ + --pattern'[experimental: include/exclude paths matching PATTERN]:PATTERN'\ + --patterns-from'[experimental: read include/exclude patterns from PATTERNFILE, one per line]:_files'\ + --strip-components'[Remove the specified number of leading path elements. Paths with fewer elements will be silently skipped.]:NUMBER'\ + $borg_standard_options + ;; + (diff) + _arguments \ + '2:archives:__borg_archive'\ + '3:archives:__borg_archive'\ + '*:path:_files'\ + --numeric-owner'[only obey numeric user and group identifiers]'\ + --same-chunker-params'[Override check of chunker parameters.]'\ + --sort'[Sort the output lines by file path.]'\ + {-e,--exclude}'[exclude paths matching PATTERN]:PATTERN'\ + --exclude-from'[read exclude patterns from EXCLUDEFILE, one per line]:_files'\ + --pattern'[experimental: include/exclude paths matching PATTERN]:PATTERN'\ + --patterns-from'[experimental: read include/exclude patterns from PATTERNFILE, one per line]:_files'\ + --exclude-caches'[exclude directories that contain a CACHEDIR.TAG file ]'\ + --exclude-if-present'[exclude directories that are tagged by containing a filesystem object with the given NAME]:NAME'\ + {--keep-exclude-tags,--keep-tag-files}'[if tag objects are specified with --exclude-if-present, don’t omit the tag objects themselves]'\ + $borg_standard_options + + ;; + (rename) + _arguments \ + '2:archives:__borg_archive'\ + '3:name:NAME'\ + $borg_standard_options + ;; + (delete) + _arguments \ + '2:archives:__borg_archive'\ + '*:archives:archives'\ + {-s,--stats}'[print statistics for the deleted archive]'\ + --cache-only'[delete only the local cache for the given repository]'\ + --force'[force deletion of corrupted archives, use --force --force in case --force does not work.]'\ + --save-space'[work slower, but using less space]'\ + {-P,--prefix}'[only consider archive names starting with this prefix.]:PREFIX'\ + {-a,--glob-archives}'[only consider archive names matching the glob]:GLOB'\ + --sort-keys'[Comma-separated list of sorting keys]:KEYS'\ + --first'[consider first N archives after other filters were applied]:N'\ + --last'[consider last N archives after other filters were applied]:N'\ + $borg_standard_options + ;; + (list) + _arguments \ + '2:archives:__borg_archive'\ + '*:path:_files'\ + --short'[only print file/directory names, nothing else]'\ + {--format,--list-format}'[specify format for file listing]:FORMAT'\ + --json'[Only valid for listing repository contents. Format output as JSON.]'\ + --json-lines'[Only valid for listing archive contents. Format output as JSON Lines. ]'\ + {-P,--prefix}'[only consider archive names starting with this prefix.]:PREFIX'\ + {-a,--glob-archives}'[only consider archive names matching the glob]:GLOB'\ + --sort-keys'[Comma-separated list of sorting keys]:KEYS'\ + --first'[consider first N archives after other filters were applied]:N'\ + --last'[consider last N archives after other filters were applied]:N'\ + {-e,--exclude}'[exclude paths matching PATTERN]:PATTERN'\ + --exclude-from'[read exclude patterns from EXCLUDEFILE, one per line]:_files'\ + --pattern'[experimental: include/exclude paths matching PATTERN]:PATTERN'\ + --patterns-from'[experimental: read include/exclude patterns from PATTERNFILE, one per line]:_files'\ + --exclude-caches'[exclude directories that contain a CACHEDIR.TAG file ]'\ + --exclude-if-present'[exclude directories that are tagged by containing a filesystem object with the given NAME]:NAME'\ + {--keep-exclude-tags,--keep-tag-files}'[if tag objects are specified with --exclude-if-present, don’t omit the tag objects themselves]'\ + $borg_standard_options + ;; + (mount) + _arguments \ + '2:archives:__borg_archive'\ + '3:mountpoint:_files'\ + {-f,--foreground}'[stay in foreground, do not daemonize]'\ + -o'[Extra mount options]:OPTS'\ + {-P,--prefix}'[only consider archive names starting with this prefix.]:PREFIX'\ + {-a,--glob-archives}'[only consider archive names matching the glob]:GLOB'\ + --sort-keys'[Comma-separated list of sorting keys]:KEYS'\ + --first'[consider first N archives after other filters were applied]:N'\ + --last'[consider last N archives after other filters were applied]:N'\ + $borg_standard_options + ;; + (umount) + _arguments \ + '2:mountpoint:_files'\ + $borg_standard_options + ;; + (info) + _arguments \ + '2:archives:__borg_archive'\ + --json'[format output as JSON]'\ + {-P,--prefix}'[only consider archive names starting with this prefix.]:PREFIX'\ + {-a,--glob-archives}'[only consider archive names matching the glob]:GLOB'\ + --sort-keys'[Comma-separated list of sorting keys]:KEYS'\ + --first'[consider first N archives after other filters were applied]:N'\ + --last'[consider last N archives after other filters were applied]:N'\ + {-e,--exclude}'[exclude paths matching PATTERN]:PATTERN'\ + $borg_standard_options + ;; + (break-lock) + _arguments \ + '2:archives:__borg_archive'\ + $borg_standard_options + ;; + (prune) + _arguments \ + '2:archives:__borg_archive'\ + {-n,--dry-run}'[do not change repository]'\ + --force'[]'\ + {-s,--stats}'[]'\ + --list'[]'\ + --keep-within'[keep all archives within this time interval]:INERVAL'\ + {--keep-last,--keep-secondly}'[number of secondly archives to keep]:N'\ + --keep-minutely'[number of minutely archives to keep]:N'\ + {-H,--keep-hourly}'[number of hourly archives to keep]:N'\ + {-d,--keep-daily}'[number of daily archives to keep]:N'\ + {-w,--keep-weekly}'[number of weekly archives to keep]:N'\ + {-m,--keep-monthly}'[number of monthly archives to keep]:N'\ + {-y,--keep-yearly}'[number of yearly archives to keep]:N'\ + --save-space'[work slower, but using less space]'\ + $borg_standard_options + ;; + (upgrade) + _arguments \ + '2:archives:__borg_archive'\ + {-n,--dry-run}'[do not change repository]'\ + --inplace'[rewrite repository in place, with no chance of going back to older versions of the repository.]'\ + --force'[Force upgrade]'\ + --tam'[Enable manifest authentication (in key and cache) (Borg 1.0.9 and later).]'\ + --disable-tam'[Disable manifest authentication (in key and cache).]'\ + $borg_standard_options + ;; + (with-lock) + _arguments \ + '(-)2:archives:__borg_archive'\ + $borg_standard_options + #'3:command:_command_names -e'\ + #'4:arguments:_normal' + #TODO Debug this, getting "_tags:comptags:36: nesting level too deep" error + ;; + (benchmark) + _arguments \ + '2:type:(crud)'\ + '3:repo:_files'\ + '4:path:_files'\ + $borg_standard_options + ;; + *) + commands=( + 'serve:start repository server process' + 'init:initialize empty repository' + 'check:verify repository' + 'key:manage repository key' + 'change-passphrase:change repository passphrase' + 'create:create backup' + 'extract:extract archive contents' + 'export-tar:create tarball from archive' + 'diff:find differences in archive contents' + 'rename:rename archive' + 'delete:delete archive' + 'list:list archive or repository contents' + 'mount:mount repository' + 'umount:umount repository' + 'info:show repository or archive information' + 'break-lock:break repository and cache locks' + 'prune:prune archives' + 'upgrade:upgrade repository format' + 'with-lock:run user command with lock held' + 'benchmark:benchmark command' + ) + + _describe 'values' commands + _arguments $borg_standard_options + + ;; + esac +} + +__borg_archive() { + local -a items + if (($+BORG_REPO)); then + items=("${(@f)$(borg list $cpath --short 2>/dev/null | sed "s#^#::#")}") + else + local cpath + cpath=$words[-1] + cpath=${cpath%::*} + items=("${(@f)$(borg list $cpath --short 2>/dev/null | sed "s#^#${cpath}::#")}") + fi + if [[ $items[1] == "" ]]; then + _files -/ + else + _wanted archives expl 'archive' compadd $items + fi +} From 5d71ae2f66e04f1bb53d2bd92517bd66942b28af Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 4 Nov 2017 15:18:55 +0100 Subject: [PATCH 110/798] set bsdflags last (include immutable flag), fixes #3263 (cherry picked from commit 2c6f9634bc970eefd46fdc66642b8998f3c3fa1b) --- src/borg/archive.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index 577fa8e645..b00c5be1d4 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -692,11 +692,6 @@ def restore_attrs(self, path, item, symlink=False, fd=None): # some systems don't support calling utime on a symlink pass acl_set(path, item, self.numeric_owner) - if not self.nobsdflags and 'bsdflags' in item: - try: - set_flags(path, item.bsdflags, fd=fd) - except OSError: - pass # chown removes Linux capabilities, so set the extended attributes at the end, after chown, since they include # the Linux capabilities in the "security.capability" attribute. xattrs = item.get('xattrs', {}) @@ -719,6 +714,12 @@ def restore_attrs(self, path, item, symlink=False, fd=None): set_ec(EXIT_WARNING) else: raise + # bsdflags include the immutable flag and need to be set last: + if not self.nobsdflags and 'bsdflags' in item: + try: + set_flags(path, item.bsdflags, fd=fd) + except OSError: + pass def set_meta(self, key, value): metadata = self._load_meta(self.id) From fe74b54890350fb3725202cf359814f50b24d685 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 29 Oct 2017 10:53:12 +0100 Subject: [PATCH 111/798] get rid of chunks_healthy when rechunking, fixes #3218 (cherry picked from commit 7211bb2211606e97eb57751a4f61a968862de0e8) --- src/borg/archive.py | 4 ++++ src/borg/archiver.py | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/src/borg/archive.py b/src/borg/archive.py index 577fa8e645..1c8a1332e8 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -917,6 +917,10 @@ def chunk_processor(data): return chunk_entry item.chunks = [] + # if we rechunkify, we'll get a fundamentally different chunks list, thus we need + # to get rid of .chunks_healthy, as it might not correspond to .chunks any more. + if getattr(self, 'recreate_rechunkify', False) and 'chunks_healthy' in item: + del item.chunks_healthy from_chunk = 0 part_number = 1 for data in chunk_iter: diff --git a/src/borg/archiver.py b/src/borg/archiver.py index d978ec1078..588350a8e1 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -3563,6 +3563,17 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): deduplicated size of the archives using the previous chunker params. When recompressing expect approx. (throughput / checkpoint-interval) in space usage, assuming all chunks are recompressed. + + If you recently ran borg check --repair and it had to fix lost chunks with all-zero + replacement chunks, please first run another backup for the same data and re-run + borg check --repair afterwards to heal any archives that had lost chunks which are + still generated from the input data. + + Important: running borg recreate to re-chunk will remove the chunks_healthy + metadata of all items with replacement chunks, so healing will not be possible + any more after re-chunking (it is also unlikely it would ever work: due to the + change of chunking parameters, the missing chunk likely will never be seen again + even if you still have the data that produced it). """) subparser = subparsers.add_parser('recreate', parents=[common_parser], add_help=False, description=self.do_recreate.__doc__, From b07899115756942c98906070302f473cd41edb9b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 29 Oct 2017 11:25:11 +0100 Subject: [PATCH 112/798] get rid of already existing invalid chunks_healthy metadata, see #3218 (cherry picked from commit 90186ad12b7f67ebca56250cafbe7d8b3f6ac628) --- src/borg/archive.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index 1c8a1332e8..e0f6ddb3b8 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -1378,7 +1378,12 @@ def replacement_chunk(size): has_chunks_healthy = 'chunks_healthy' in item chunks_current = item.chunks chunks_healthy = item.chunks_healthy if has_chunks_healthy else chunks_current - assert len(chunks_current) == len(chunks_healthy) + if has_chunks_healthy and len(chunks_current) != len(chunks_healthy): + # should never happen, but there was issue #3218. + logger.warning('{}: Invalid chunks_healthy metadata removed!'.format(item.path)) + del item.chunks_healthy + has_chunks_healthy = False + chunks_healthy = chunks_current for chunk_current, chunk_healthy in zip(chunks_current, chunks_healthy): chunk_id, size, csize = chunk_healthy if chunk_id not in self.chunks: From 28a25b354b2b741d0fea28ba11c26ed5427f3d72 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 29 Oct 2017 14:49:42 +0100 Subject: [PATCH 113/798] recreate: move chunks_healthy when excluding hardlink master, fixes #3228 (cherry picked from commit 7aafcc517a30627d5b76180cc0cc604c7b04cb34) --- src/borg/archive.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index e0f6ddb3b8..78a7f07d05 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -1639,15 +1639,17 @@ def item_is_hardlink_master(item): if not matcher.match(item.path): self.print_file_status('x', item.path) if item_is_hardlink_master(item): - hardlink_masters[item.path] = (item.get('chunks'), None) + hardlink_masters[item.path] = (item.get('chunks'), item.get('chunks_healthy'), None) continue if target_is_subset and hardlinkable(item.mode) and item.get('source') in hardlink_masters: # master of this hard link is outside the target subset - chunks, new_source = hardlink_masters[item.source] + chunks, chunks_healthy, new_source = hardlink_masters[item.source] if new_source is None: # First item to use this master, move the chunks item.chunks = chunks - hardlink_masters[item.source] = (None, item.path) + if chunks_healthy is not None: + item.chunks_healthy = chunks_healthy + hardlink_masters[item.source] = (None, None, item.path) del item.source else: # Master was already moved, only update this item's source From 9325c4e9c1b98633212d2e5f19c79d011208d840 Mon Sep 17 00:00:00 2001 From: SanskritFritz Date: Sat, 4 Nov 2017 15:56:53 +0100 Subject: [PATCH 114/798] Added bash and fish shell auto-completions. --- scripts/shell_completions/bash/borg | 165 ++++++++++++ scripts/shell_completions/fish/borg.fish | 319 +++++++++++++++++++++++ 2 files changed, 484 insertions(+) create mode 100644 scripts/shell_completions/bash/borg create mode 100644 scripts/shell_completions/fish/borg.fish diff --git a/scripts/shell_completions/bash/borg b/scripts/shell_completions/bash/borg new file mode 100644 index 0000000000..b671f1bc7d --- /dev/null +++ b/scripts/shell_completions/bash/borg @@ -0,0 +1,165 @@ +#!/bin/bash + +# Completions for borg version 1.1.1 +# https://www.borgbackup.org/ +# Note: +# Listing archives works on password protected repositories only if $BORG_PASSPHRASE is set. +# Install: +# Copy this file to /usr/share/bash-completion/completions/ or /etc/bash_completion.d/ + +_borg() +{ + COMPREPLY=() + local cur="${COMP_WORDS[COMP_CWORD]}" + local prev="${COMP_WORDS[COMP_CWORD-1]}" + local prevprev="${COMP_WORDS[COMP_CWORD-2]}" + local common_opts="-h --help --version --critical --error --warning --info -v --verbose --debug --debug-topic -p --progress --log-json --lock-wait --show-version --show-rc --umask --remote-path --remote-ratelimit --consider-part-files --debug-profile" + local opts="${common_opts}" + + # Commands + if [[ ${COMP_CWORD} == 1 ]] ; then + local borg_commands="init create extract check rename list diff delete prune info mount umount key serve upgrade recreate export-tar with-lock break-lock benchmark" + COMPREPLY=( $(compgen -W "${borg_commands}" -- ${cur}) ) + return 0 + fi + + case "${prev}" in + 'key') + COMPREPLY=( $(compgen -W "import export change-passphrase" -- ${cur}) ) + return 0 + ;; + 'benchmark') + COMPREPLY=( $(compgen -W "crud" -- ${cur}) ) + return 0 + ;; + '--encryption' | '-e') + local encryption_modes="none keyfile keyfile-blake2 repokey repokey-blake2 authenticated authenticated-blake2" + COMPREPLY=( $(compgen -W "${encryption_modes}" -- ${cur}) ) + return 0 + ;; + '--files-cache') + local files_cache_mode="ctime,size,inode mtime,size,inode ctime,size mtime,size rechunk,ctime rechunk,mtime disabled" + COMPREPLY=( $(compgen -W "${files_cache_mode}" -- ${cur}) ) + return 0 + ;; + '--compression') + local compression_methods="none auto lz4 zlib,1 zlib,2 zlib,3 zlib,4 zlib,5 zlib,6 zlib,7 zlib,8 zlib,9 lzma,0 lzma,1 lzma,2 lzma,3 lzma,4 lzma,5 lzma,6 lzma,7 lzma,8 lzma,9" + COMPREPLY=( $(compgen -W "${compression_methods}" -- ${cur}) ) + return 0 + ;; + '--sort-by') + local sort_keys="timestamp name id" + COMPREPLY=( $(compgen -W "${sort_keys}" -- ${cur}) ) + return 0 + ;; + '-o') + local fuse_options="allow_other allow_root" + # FIXME there are lot more options, but not all are applicable + COMPREPLY=( $(compgen -W "${fuse_options}" -- ${cur}) ) + return 0 + ;; + '--recompress') + local recompress_when="if-different always" + COMPREPLY=( $(compgen -W "${recompress_when}" -- ${cur}) ) + return 0 + ;; + esac + + if [[ ${cur} == -* ]] ; then + case "${COMP_LINE}" in + *' init '*) + local opts="-e --encryption --append-only --storage-quota ${common_opts}" + ;; + *' create '*) + local opts="-n --dry-run -s --stats --list --filter --json --no-cache-sync --no-files-cache -e --exclude --exclude-from --pattern --patterns-from --exclude-caches --exclude-if-present --keep-exclude-tags --keep-tag-files --exclude-nodump -x --one-file-system --numeric-owner --noatime --noctime --nobsdflags --ignore-inode --mtime --files-cache --read-special --comment --timestamp -c --checkpoint-interval --chunker-params -C --compression ${common_opts}" + ;; + *' extract '*) + local opts="--list -n --dry-run --numeric-owner --nobsdflags --stdout --sparse -e --exclude --exclude-from --pattern --patterns-from --strip-components ${common_opts}" + ;; + *' check '*) + local opts="--repository-only --archives-only --verify-data --repair --save-space -P --prefix -a --glob-archives --sort-by --first --last ${common_opts}" + ;; + # rename + # no specific options + *" list "*) + local opts="--short --list-format --format --json --json-lines -P --prefix -a --glob-archives --sort-by --first --last -e --exclude --exclude-from --pattern --patterns-from --exclude-caches --exclude-if-present --keep-exclude-tags --keep-tag-files ${common_opts}" + ;; + *' diff '*) + local opts="--numeric-owner --same-chunker-params --sort -e --exclude --exclude-from --pattern --patterns-from --exclude-caches --exclude-if-present --keep-exclude-tags --keep-tag-files ${common_opts}" + ;; + *' delete '*) + local opts="-s --stats --cache-only --force --save-space -P --prefix -a --glob-archives --sort-by --first --last ${common_opts}" + ;; + *' prune '*) + local opts="-n --dry-run --force -s --stats --list --keep-within --keep-last --keep-secondly --keep-minutely -H --keep-hourly -d --keep-daily -w --keep-weekly -m --keep-monthly -y --keep-yearly --save-space -P --prefix -a --glob-archives ${common_opts}" + ;; + *' info '*) + local opts="--json -P --prefix -a --glob-archives --sort-by --first --last ${common_opts}" + ;; + *' mount '*) + local opts="-f --foreground -o -P --prefix -a --glob-archives --sort-by --first --last ${common_opts}" + ;; + # umount + # no specific options + # key change-passphrase + # no specific options + *' export '*) + local opts="--paper --qr-html ${common_opts}" + ;; + *' import '*) + local opts="--paper ${common_opts}" + ;; + *' upgrade '*) + local opts="-n --dry-run --inplace --force --tam --disable-tam ${common_opts}" + ;; + *' recreate '*) + local opts="--list --filter -n dry-run -s stats -e exclude --exclude-from --pattern --patterns-from --exclude-caches --exclude-if-present --keep-exclude-tags --keep-tag-files --target -c checkpoint-interval --comment --timestamp --timestamp -C compression --recompress --chunker-params ${common_opts}" + ;; + *' export-tar '*) + local opts="--tar-filter --list -e exclude --exclude-from --pattern --patterns-from --strip-components ${common_opts}" + ;; + *' serve '*) + local opts="--restrict-to-path --restrict-to-repository --append-only --storage-quota ${common_opts}" + ;; + # with-lock + # no specific options + # break-lock + # no specific options + # benchmark crud + # no specific options + esac + + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + return 0 + fi + + # Listing archives. + # Since "::" is treated as separate word in bash, + # it is $cur when the cursor is right behind it + # and $prev if the user has started to type an archive name. + local repository_name="" # If set, we'll list the archives + local typed_word="" + if [[ ${cur} == "::" ]] ; then + repository_name=${prev} + fi + if [[ ${prev} == "::" ]] ; then + repository_name=${prevprev} + typed_word=${cur} + fi + if [[ ${repository_name} != "" ]] ; then + if [[ ${COMP_LINE} == *" ::"* ]] ; then + # There is a space before the "::" + # which means that no repository name was typed, + # so probably $BORG_REPO is set. + repository_name="" + fi + local archive_list=$(borg list --short "${repository_name}" 2>/dev/null) + COMPREPLY=( $(compgen -W "${archive_list}" -- "${typed_word}" ) ) + return 0 + fi + + COMPREPLY=( $(compgen -f ${cur}) ) + return 0 +} + +complete -F _borg borg diff --git a/scripts/shell_completions/fish/borg.fish b/scripts/shell_completions/fish/borg.fish new file mode 100644 index 0000000000..d17b36470e --- /dev/null +++ b/scripts/shell_completions/fish/borg.fish @@ -0,0 +1,319 @@ +# Completions for borg version 1.1.1 +# https://www.borgbackup.org/ +# Note: +# The list_archives function works on password protected repositories only if $BORG_PASSPHRASE is set. +# Install: +# Copy this file to /usr/share/fish/vendor_completions.d/ + +# Commands + +complete -c borg -f -n __fish_is_first_token -a 'init' -d 'Initialize an empty repository' +complete -c borg -f -n __fish_is_first_token -a 'create' -d 'Create new archive' +complete -c borg -f -n __fish_is_first_token -a 'extract' -d 'Extract archive contents' +complete -c borg -f -n __fish_is_first_token -a 'check' -d 'Check repository consistency' +complete -c borg -f -n __fish_is_first_token -a 'rename' -d 'Rename an existing archive' +complete -c borg -f -n __fish_is_first_token -a 'list' -d 'List archive or repository contents' +complete -c borg -f -n __fish_is_first_token -a 'diff' -d 'Find differences between archives' +complete -c borg -f -n __fish_is_first_token -a 'delete' -d 'Delete a repository or archive' +complete -c borg -f -n __fish_is_first_token -a 'prune' -d 'Prune repository archives' +complete -c borg -f -n __fish_is_first_token -a 'info' -d 'Show archive details' +complete -c borg -f -n __fish_is_first_token -a 'mount' -d 'Mount archive or a repository' +complete -c borg -f -n __fish_is_first_token -a 'umount' -d 'Un-mount the mounted archive' + +function __fish_borg_seen_key + if __fish_seen_subcommand_from key + and not __fish_seen_subcommand_from import export change-passphrase + return 0 + end + return 1 +end +complete -c borg -f -n __fish_is_first_token -a 'key' -d 'Manage a repository key' +complete -c borg -f -n __fish_borg_seen_key -a 'import' -d 'Import a repository key' +complete -c borg -f -n __fish_borg_seen_key -a 'export' -d 'Export a repository key' +complete -c borg -f -n __fish_borg_seen_key -a 'change-passphrase' -d 'Change key file passphrase' + +complete -c borg -f -n __fish_is_first_token -a 'serve' -d 'Start in server mode' +complete -c borg -f -n __fish_is_first_token -a 'upgrade' -d 'Upgrade a repository' +complete -c borg -f -n __fish_is_first_token -a 'recreate' -d 'Recreate contents of existing archives' +complete -c borg -f -n __fish_is_first_token -a 'export-tar' -d 'Create tarball from an archive' +complete -c borg -f -n __fish_is_first_token -a 'with-lock' -d 'Run a command while repository lock held' +complete -c borg -f -n __fish_is_first_token -a 'break-lock' -d 'Break the repository lock' + +function __fish_borg_seen_benchmark + if __fish_seen_subcommand_from benchmark + and not __fish_seen_subcommand_from crud + return 0 + end + return 1 +end +complete -c borg -f -n __fish_is_first_token -a 'benchmark' -d 'Benchmark borg operations' +complete -c borg -f -n __fish_borg_seen_benchmark -a 'crud' -d 'Benchmark borg CRUD operations' + + +# Common options +complete -c borg -f -s h -l 'help' -d 'Show help information' +complete -c borg -f -l 'version' -d 'Show version information' +complete -c borg -f -l 'critical' -d 'Log level CRITICAL' +complete -c borg -f -l 'error' -d 'Log level ERROR' +complete -c borg -f -l 'warning' -d 'Log level WARNING (default)' +complete -c borg -f -l 'info' -d 'Log level INFO' +complete -c borg -f -s v -l 'verbose' -d 'Log level INFO' +complete -c borg -f -l 'debug' -d 'Log level DEBUG' +complete -c borg -f -l 'debug-topic' -d 'Enable TOPIC debugging' +complete -c borg -f -s p -l 'progress' -d 'Show progress information' +complete -c borg -f -l 'log-json' -d 'Output one JSON object per log line' +complete -c borg -f -l 'lock-wait' -d 'Wait for lock max N seconds [1]' +complete -c borg -f -l 'show-version' -d 'Log version information' +complete -c borg -f -l 'show-rc' -d 'Log the return code' +complete -c borg -f -l 'umask' -d 'Set umask to M [0077]' +complete -c borg -l 'remote-path' -d 'Use PATH as remote borg executable' +complete -c borg -f -l 'remote-ratelimit' -d 'Set remote network upload RATE limit' +complete -c borg -f -l 'consider-part-files' -d 'Treat part files like normal files' +complete -c borg -l 'debug-profile' -d 'Write execution profile into FILE' + +# borg init options +set -l encryption_modes "none keyfile keyfile-blake2 repokey repokey-blake2 authenticated authenticated-blake2" +complete -c borg -f -s e -l 'encryption' -d 'Encryption key MODE' -a "$encryption_modes" -n "__fish_seen_subcommand_from init" +complete -c borg -f -l 'append-only' -d 'Create an append-only mode repository' -n "__fish_seen_subcommand_from init" +complete -c borg -f -l 'storage-quota' -d 'Set storage QUOTA of the repository' -n "__fish_seen_subcommand_from init" + +# borg create options +complete -c borg -f -s n -l 'dry-run' -d 'Do not change the repository' -n "__fish_seen_subcommand_from create" +complete -c borg -f -s s -l 'stats' -d 'Print verbose statistics' -n "__fish_seen_subcommand_from create" +complete -c borg -f -l 'list' -d 'Print verbose list of items' -n "__fish_seen_subcommand_from create" +complete -c borg -f -l 'filter' -d 'Only items with given STATUSCHARS' -n "__fish_seen_subcommand_from create" +complete -c borg -f -l 'json' -d 'Print verbose stats as json' -n "__fish_seen_subcommand_from create" +complete -c borg -f -l 'no-cache-sync' -d 'Do not synchronize the cache' -n "__fish_seen_subcommand_from create" +complete -c borg -f -l 'no-files-cache' -d 'Do not load/update metadata cache' -n "__fish_seen_subcommand_from create" +# Exclusion options +complete -c borg -s e -l 'exclude' -d 'Exclude paths matching PATTERN' -n "__fish_seen_subcommand_from create" +complete -c borg -l 'exclude-from' -d 'Read exclude patterns from EXCLUDEFILE' -n "__fish_seen_subcommand_from create" +complete -c borg -f -l 'pattern' -d 'Include/exclude paths matching PATTERN' -n "__fish_seen_subcommand_from create" +complete -c borg -l 'patterns-from' -d 'Include/exclude paths from PATTERNFILE' -n "__fish_seen_subcommand_from create" +complete -c borg -f -l 'exclude-caches' -d 'Exclude directories tagged as cache' -n "__fish_seen_subcommand_from create" +complete -c borg -l 'exclude-if-present' -d 'Exclude directories that contain FILENAME' -n "__fish_seen_subcommand_from create" +complete -c borg -f -l 'keep-exclude-tags' -d 'Keep tag files of excluded directories' -n "__fish_seen_subcommand_from create" +complete -c borg -f -l 'keep-tag-files' -d 'Keep tag files of excluded directories' -n "__fish_seen_subcommand_from create" +complete -c borg -f -l 'exclude-nodump' -d 'Exclude files flagged nodump' -n "__fish_seen_subcommand_from create" +# Filesytem options +complete -c borg -f -s x -l 'one-file-system' -d 'Stay in the same file system' -n "__fish_seen_subcommand_from create" +complete -c borg -f -l 'numeric-owner' -d 'Only store numeric user:group identifiers' -n "__fish_seen_subcommand_from create" +complete -c borg -f -l 'noatime' -d 'Do not store atime' -n "__fish_seen_subcommand_from create" +complete -c borg -f -l 'noctime' -d 'Do not store ctime' -n "__fish_seen_subcommand_from create" +complete -c borg -f -l 'nobsdflags' -d 'Do not store bsdflags' -n "__fish_seen_subcommand_from create" +complete -c borg -f -l 'ignore-inode' -d 'Ignore inode data in file metadata cache' -n "__fish_seen_subcommand_from create" +set -l files_cache_mode "ctime,size,inode mtime,size,inode ctime,size mtime,size rechunk,ctime rechunk,mtime disabled" +complete -c borg -f -l 'files-cache' -d 'Operate files cache in MODE' -a "$files_cache_mode" -n "__fish_seen_subcommand_from create" +complete -c borg -f -l 'read-special' -d 'Open device files like regular files' -n "__fish_seen_subcommand_from create" +# Archive options +complete -c borg -f -l 'comment' -d 'Add COMMENT to the archive' -n "__fish_seen_subcommand_from create" +complete -c borg -f -l 'timestamp' -d 'Set creation TIME (yyyy-mm-ddThh:mm:ss)' -n "__fish_seen_subcommand_from create" +complete -c borg -l 'timestamp' -d 'Set creation time by reference FILE' -n "__fish_seen_subcommand_from create" +complete -c borg -f -s c -l 'checkpoint-interval' -d 'Write checkpoint every N seconds [1800]' -n "__fish_seen_subcommand_from create" +complete -c borg -f -l 'chunker-params' -d 'Chunker PARAMETERS [19,23,21,4095]' -n "__fish_seen_subcommand_from create" +set -l compression_methods "none auto lz4 zlib,1 zlib,2 zlib,3 zlib,4 zlib,5 zlib,6 zlib,7 zlib,8 zlib,9 lzma,0 lzma,1 lzma,2 lzma,3 lzma,4 lzma,5 lzma,6 lzma,7 lzma,8 lzma,9" +complete -c borg -f -s C -l 'compression' -d 'Select compression ALGORITHM,LEVEL [lz4]' -a "$compression_methods" -n "__fish_seen_subcommand_from create" + +# borg extract options +complete -c borg -f -l 'list' -d 'Print verbose list of items' -n "__fish_seen_subcommand_from extract" +complete -c borg -f -s n -l 'dry-run' -d 'Do not actually extract any files' -n "__fish_seen_subcommand_from extract" +complete -c borg -f -l 'numeric-owner' -d 'Only obey numeric user:group identifiers' -n "__fish_seen_subcommand_from extract" +complete -c borg -f -l 'nobsdflags' -d 'Do not extract bsdflags' -n "__fish_seen_subcommand_from extract" +complete -c borg -f -l 'stdout' -d 'Write all extracted data to stdout' -n "__fish_seen_subcommand_from extract" +complete -c borg -f -l 'sparse' -d 'Create holes in sparse file' -n "__fish_seen_subcommand_from extract" +# Exclusion options +complete -c borg -s e -l 'exclude' -d 'Exclude paths matching PATTERN' -n "__fish_seen_subcommand_from extract" +complete -c borg -l 'exclude-from' -d 'Read exclude patterns from EXCLUDEFILE' -n "__fish_seen_subcommand_from extract" +complete -c borg -l 'pattern' -d 'Include/exclude paths matching PATTERN' -n "__fish_seen_subcommand_from extract" +complete -c borg -l 'patterns-from' -d 'Include/exclude paths from PATTERNFILE' -n "__fish_seen_subcommand_from extract" +complete -c borg -f -l 'strip-components' -d 'Remove NUMBER of leading path elements' -n "__fish_seen_subcommand_from extract" + +# borg check options +complete -c borg -f -l 'repository-only' -d 'Only perform repository checks' -n "__fish_seen_subcommand_from check" +complete -c borg -f -l 'archives-only' -d 'Only perform archives checks' -n "__fish_seen_subcommand_from check" +complete -c borg -f -l 'verify-data' -d 'Cryptographic integrity verification' -n "__fish_seen_subcommand_from check" +complete -c borg -f -l 'repair' -d 'Attempt to repair found inconsistencies' -n "__fish_seen_subcommand_from check" +complete -c borg -f -l 'save-space' -d 'Work slower but using less space' -n "__fish_seen_subcommand_from check" +# Archive filters +complete -c borg -f -s P -l 'prefix' -d 'Only archive names starting with PREFIX' -n "__fish_seen_subcommand_from check" +complete -c borg -f -s a -l 'glob-archives' -d 'Only archive names matching GLOB' -n "__fish_seen_subcommand_from check" +set -l sort_keys "timestamp name id" +complete -c borg -f -l 'sort-by' -d 'Sorting KEYS [timestamp]' -a "$sort_keys" -n "__fish_seen_subcommand_from check" +complete -c borg -f -l 'first' -d 'Only first N archives' -n "__fish_seen_subcommand_from check" +complete -c borg -f -l 'last' -d 'Only last N archives' -n "__fish_seen_subcommand_from check" + +# borg rename +# no specific options + +# borg list options +complete -c borg -f -l 'short' -d 'Only print file/directory names' -n "__fish_seen_subcommand_from list" +complete -c borg -f -l 'list-format' -d 'Specify FORMAT for file listing' -n "__fish_seen_subcommand_from list" +complete -c borg -f -l 'format' -d 'Specify FORMAT for file listing' -n "__fish_seen_subcommand_from list" +complete -c borg -f -l 'json' -d 'List contents in json format' -n "__fish_seen_subcommand_from list" +complete -c borg -f -l 'json-lines' -d 'List contents in json lines format' -n "__fish_seen_subcommand_from list" +# Archive filters +complete -c borg -f -s P -l 'prefix' -d 'Only archive names starting with PREFIX' -n "__fish_seen_subcommand_from list" +complete -c borg -f -s a -l 'glob-archives' -d 'Only archive names matching GLOB' -n "__fish_seen_subcommand_from list" +complete -c borg -f -l 'sort-by' -d 'Sorting KEYS [timestamp]' -a "$sort_keys" -n "__fish_seen_subcommand_from list" +complete -c borg -f -l 'first' -d 'Only first N archives' -n "__fish_seen_subcommand_from list" +complete -c borg -f -l 'last' -d 'Only last N archives' -n "__fish_seen_subcommand_from list" +# Exclusion options +complete -c borg -s e -l 'exclude' -d 'Exclude paths matching PATTERN' -n "__fish_seen_subcommand_from list" +complete -c borg -l 'exclude-from' -d 'Read exclude patterns from EXCLUDEFILE' -n "__fish_seen_subcommand_from list" +complete -c borg -f -l 'pattern' -d 'Include/exclude paths matching PATTERN' -n "__fish_seen_subcommand_from list" +complete -c borg -l 'patterns-from' -d 'Include/exclude paths from PATTERNFILE' -n "__fish_seen_subcommand_from list" +complete -c borg -f -l 'exclude-caches' -d 'Exclude directories tagged as cache' -n "__fish_seen_subcommand_from list" +complete -c borg -l 'exclude-if-present' -d 'Exclude directories that contain FILENAME' -n "__fish_seen_subcommand_from list" +# FIXME are these options really part of borg list? +complete -c borg -f -l 'keep-exclude-tags' -d 'Keep tag files of excluded directories' -n "__fish_seen_subcommand_from list" +complete -c borg -f -l 'keep-tag-files' -d 'Keep tag files of excluded directories' -n "__fish_seen_subcommand_from list" + +# borg diff options +complete -c borg -f -l 'numeric-owner' -d 'Only consider numeric user:group' -n "__fish_seen_subcommand_from diff" +complete -c borg -f -l 'same-chunker-params' -d 'Override check of chunker parameters' -n "__fish_seen_subcommand_from diff" +complete -c borg -f -l 'sort' -d 'Sort the output lines by file path' -n "__fish_seen_subcommand_from diff" +# Exclusion options +complete -c borg -s e -l 'exclude' -d 'Exclude paths matching PATTERN' -n "__fish_seen_subcommand_from diff" +complete -c borg -l 'exclude-from' -d 'Read exclude patterns from EXCLUDEFILE' -n "__fish_seen_subcommand_from diff" +complete -c borg -f -l 'pattern' -d 'Include/exclude paths matching PATTERN' -n "__fish_seen_subcommand_from diff" +complete -c borg -l 'patterns-from' -d 'Include/exclude paths from PATTERNFILE' -n "__fish_seen_subcommand_from diff" +complete -c borg -f -l 'exclude-caches' -d 'Exclude directories tagged as cache' -n "__fish_seen_subcommand_from diff" +complete -c borg -l 'exclude-if-present' -d 'Exclude directories that contain FILENAME' -n "__fish_seen_subcommand_from diff" +# FIXME are these options really part of borg diff? +complete -c borg -f -l 'keep-exclude-tags' -d 'Keep tag files of excluded directories' -n "__fish_seen_subcommand_from diff" +complete -c borg -f -l 'keep-tag-files' -d 'Keep tag files of excluded directories' -n "__fish_seen_subcommand_from diff" + +# borg delete options +complete -c borg -f -s s -l 'stats' -d 'Print verbose statistics' -n "__fish_seen_subcommand_from delete" +complete -c borg -f -l 'cache-only' -d "Delete only the local cache" -n "__fish_seen_subcommand_from delete" +complete -c borg -f -l 'force' -d 'Force deletion of corrupted archives' -n "__fish_seen_subcommand_from delete" +complete -c borg -f -l 'save-space' -d 'Work slower but using less space' -n "__fish_seen_subcommand_from delete" +# Archive filters +complete -c borg -f -s P -l 'prefix' -d 'Only archive names starting with PREFIX' -n "__fish_seen_subcommand_from delete" +complete -c borg -f -s a -l 'glob-archives' -d 'Only archive names matching GLOB' -n "__fish_seen_subcommand_from delete" +complete -c borg -f -l 'sort-by' -d 'Sorting KEYS [timestamp]' -a "$sort_keys" -n "__fish_seen_subcommand_from delete" +complete -c borg -f -l 'first' -d 'Only first N archives' -n "__fish_seen_subcommand_from delete" +complete -c borg -f -l 'last' -d 'Only last N archives' -n "__fish_seen_subcommand_from delete" + +# borg prune options +complete -c borg -f -s n -l 'dry-run' -d 'Do not change the repository' -n "__fish_seen_subcommand_from prune" +complete -c borg -f -l 'force' -d 'Force pruning of corrupted archives' -n "__fish_seen_subcommand_from prune" +complete -c borg -f -s s -l 'stats' -d 'Print verbose statistics' -n "__fish_seen_subcommand_from prune" +complete -c borg -f -l 'list' -d 'Print verbose list of items' -n "__fish_seen_subcommand_from prune" +complete -c borg -f -l 'keep-within' -d 'Keep archives within time INTERVAL' -n "__fish_seen_subcommand_from prune" +complete -c borg -f -l 'keep-last' -d 'NUMBER of secondly archives to keep' -n "__fish_seen_subcommand_from prune" +complete -c borg -f -l 'keep-secondly' -d 'NUMBER of secondly archives to keep' -n "__fish_seen_subcommand_from prune" +complete -c borg -f -l 'keep-minutely' -d 'NUMBER of minutely archives to keep' -n "__fish_seen_subcommand_from prune" +complete -c borg -f -s H -l 'keep-hourly' -d 'NUMBER of hourly archives to keep' -n "__fish_seen_subcommand_from prune" +complete -c borg -f -s d -l 'keep-daily' -d 'NUMBER of daily archives to keep' -n "__fish_seen_subcommand_from prune" +complete -c borg -f -s w -l 'keep-weekly' -d 'NUMBER of weekly archives to keep' -n "__fish_seen_subcommand_from prune" +complete -c borg -f -s m -l 'keep-monthly' -d 'NUMBER of monthly archives to keep' -n "__fish_seen_subcommand_from prune" +complete -c borg -f -s y -l 'keep-yearly' -d 'NUMBER of yearly archives to keep' -n "__fish_seen_subcommand_from prune" +complete -c borg -f -l 'save-space' -d 'Work slower but using less space' -n "__fish_seen_subcommand_from prune" +# Archive filters +complete -c borg -f -s P -l 'prefix' -d 'Only archive names starting with PREFIX' -n "__fish_seen_subcommand_from prune" +complete -c borg -f -s a -l 'glob-archives' -d 'Only archive names matching GLOB' -n "__fish_seen_subcommand_from prune" + +# borg info options +complete -c borg -f -l 'json' -d 'Format output in json format' -n "__fish_seen_subcommand_from info" +# Archive filters +complete -c borg -f -s P -l 'prefix' -d 'Only archive names starting with PREFIX' -n "__fish_seen_subcommand_from info" +complete -c borg -f -s a -l 'glob-archives' -d 'Only archive names matching GLOB' -n "__fish_seen_subcommand_from info" +complete -c borg -f -l 'sort-by' -d 'Sorting KEYS [timestamp]' -a "$sort_keys" -n "__fish_seen_subcommand_from info" +complete -c borg -f -l 'first' -d 'Only first N archives' -n "__fish_seen_subcommand_from info" +complete -c borg -f -l 'last' -d 'Only last N archives' -n "__fish_seen_subcommand_from info" + +# borg mount options +complete -c borg -f -s f -l 'foreground' -d 'Stay in foreground, do not daemonize' -n "__fish_seen_subcommand_from mount" +set -l fuse_options "allow_other allow_root" # FIXME there are lot more options, but not all are applicable +complete -c borg -f -s o -d 'Fuse mount OPTIONS' -a "$fuse_options" -n "__fish_seen_subcommand_from mount" +# Archive filters +complete -c borg -f -s P -l 'prefix' -d 'Only archive names starting with PREFIX' -n "__fish_seen_subcommand_from mount" +complete -c borg -f -s a -l 'glob-archives' -d 'Only archive names matching GLOB' -n "__fish_seen_subcommand_from mount" +complete -c borg -f -l 'sort-by' -d 'Sorting KEYS [timestamp]' -a "$sort_keys" -n "__fish_seen_subcommand_from mount" +complete -c borg -f -l 'first' -d 'Only first N archives' -n "__fish_seen_subcommand_from mount" +complete -c borg -f -l 'last' -d 'Only last N archives' -n "__fish_seen_subcommand_from mount" + +# borg umount +# no specific options + +# borg key change-passphrase +# no specific options + +# borg key export +complete -c borg -f -l 'paper' -d 'Create an export for printing' -n "__fish_seen_subcommand_from export" +complete -c borg -f -l 'qr-html' -d 'Create an html file for printing and qr' -n "__fish_seen_subcommand_from export" + +# borg key import +complete -c borg -f -l 'paper' -d 'Import from a backup done with --paper' -n "__fish_seen_subcommand_from import" + +# borg upgrade +complete -c borg -f -s n -l 'dry-run' -d 'Do not change the repository' -n "__fish_seen_subcommand_from upgrade" +complete -c borg -f -l 'inplace' -d 'Rewrite repository in place' -n "__fish_seen_subcommand_from upgrade" +complete -c borg -f -l 'force' -d 'Force upgrade' -n "__fish_seen_subcommand_from upgrade" +complete -c borg -f -l 'tam' -d 'Enable manifest authentication' -n "__fish_seen_subcommand_from upgrade" +complete -c borg -f -l 'disable-tam' -d 'Disable manifest authentication' -n "__fish_seen_subcommand_from upgrade" + +# borg recreate +complete -c borg -f -l 'list' -d 'Print verbose list of items' -n "__fish_seen_subcommand_from recreate" +complete -c borg -f -l 'filter' -d 'Only items with given STATUSCHARS' -n "__fish_seen_subcommand_from recreate" +complete -c borg -f -s n -l 'dry-run' -d 'Do not change the repository' -n "__fish_seen_subcommand_from recreate" +complete -c borg -f -s s -l 'stats' -d 'Print verbose statistics' -n "__fish_seen_subcommand_from recreate" +# Exclusion options +complete -c borg -s e -l 'exclude' -d 'Exclude paths matching PATTERN' -n "__fish_seen_subcommand_from recreate" +complete -c borg -l 'exclude-from' -d 'Read exclude patterns from EXCLUDEFILE' -n "__fish_seen_subcommand_from recreate" +complete -c borg -f -l 'pattern' -d 'Include/exclude paths matching PATTERN' -n "__fish_seen_subcommand_from recreate" +complete -c borg -l 'patterns-from' -d 'Include/exclude paths from PATTERNFILE' -n "__fish_seen_subcommand_from recreate" +complete -c borg -f -l 'exclude-caches' -d 'Exclude directories tagged as cache' -n "__fish_seen_subcommand_from recreate" +complete -c borg -l 'exclude-if-present' -d 'Exclude directories that contain FILENAME' -n "__fish_seen_subcommand_from recreate" +complete -c borg -f -l 'keep-exclude-tags' -d 'Keep tag files of excluded directories' -n "__fish_seen_subcommand_from recreate" +complete -c borg -f -l 'keep-tag-files' -d 'Keep tag files of excluded directories' -n "__fish_seen_subcommand_from recreate" +# Archive options +complete -c borg -f -l 'target' -d "Create a new ARCHIVE" -n "__fish_seen_subcommand_from recreate" +complete -c borg -f -s c -l 'checkpoint-interval' -d 'Write checkpoint every N seconds [1800]' -n "__fish_seen_subcommand_from recreate" +complete -c borg -f -l 'comment' -d 'Add COMMENT to the archive' -n "__fish_seen_subcommand_from recreate" +complete -c borg -f -l 'timestamp' -d 'Set creation TIME (yyyy-mm-ddThh:mm:ss)' -n "__fish_seen_subcommand_from recreate" +complete -c borg -l 'timestamp' -d 'Set creation time using reference FILE' -n "__fish_seen_subcommand_from recreate" +complete -c borg -f -s C -l 'compression' -d 'Select compression ALGORITHM,LEVEL [lz4]' -a "$compression_methods" -n "__fish_seen_subcommand_from recreate" +set -l recompress_when "if-different always" +complete -c borg -f -l 'recompress' -d 'Recompress chunks CONDITION' -a "$recompress_when" -n "__fish_seen_subcommand_from recreate" +complete -c borg -f -l 'chunker-params' -d 'Chunker PARAMETERS [19,23,21,4095]' -n "__fish_seen_subcommand_from recreate" + +# borg export-tar options +complete -c borg -l 'tar-filter' -d 'Filter program to pipe data through' -n "__fish_seen_subcommand_from export-tar" +complete -c borg -f -l 'list' -d 'Print verbose list of items' -n "__fish_seen_subcommand_from export-tar" +# Exclusion options +complete -c borg -s e -l 'exclude' -d 'Exclude paths matching PATTERN' -n "__fish_seen_subcommand_from recreate" +complete -c borg -l 'exclude-from' -d 'Read exclude patterns from EXCLUDEFILE' -n "__fish_seen_subcommand_from recreate" +complete -c borg -f -l 'pattern' -d 'Include/exclude paths matching PATTERN' -n "__fish_seen_subcommand_from recreate" +complete -c borg -l 'patterns-from' -d 'Include/exclude paths from PATTERNFILE' -n "__fish_seen_subcommand_from recreate" +complete -c borg -f -l 'strip-components' -d 'Remove NUMBER of leading path elements' -n "__fish_seen_subcommand_from recreate" + +# borg serve +complete -c borg -l 'restrict-to-path' -d 'Restrict repository access to PATH' -n "__fish_seen_subcommand_from serve" +complete -c borg -l 'restrict-to-repository' -d 'Restrict repository access at PATH' -n "__fish_seen_subcommand_from serve" +complete -c borg -f -l 'append-only' -d 'Only allow appending to repository' -n "__fish_seen_subcommand_from serve" +complete -c borg -f -l 'storage-quota' -d 'Override storage QUOTA of the repository' -n "__fish_seen_subcommand_from serve" + +# borg with-lock +# no specific options + +# borg break-lock +# no specific options + +# borg benchmark +# no specific options + + +# List archives + +function __fish_borg_is_repository + return (string match --quiet --regex '.*::' '"'(commandline --current-token)'"') +end + +function __fish_borg_list_archives + set -l repository_name (string replace --regex '::.*' '' (commandline --current-token)) + borg list --format="$repository_name::{archive}{NEWLINE}" "$repository_name" ^/dev/null +end + +complete -c borg -f -n __fish_borg_is_repository -a '(__fish_borg_list_archives)' From e4d85294c4d137951b672aa7f142265dbc8091b1 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 4 Nov 2017 21:45:04 +0100 Subject: [PATCH 115/798] vagrant: openindiana updates --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 206ca54988..2eb9a462d8 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -151,7 +151,7 @@ end def packages_openindiana return <<-EOF #pkg update # XXX needs separate provisioning step + reboot - pkg install python-34 clang-3.4 lz4 git + pkg install python-34 clang-40 lz4 git python3 -m ensurepip pip3 install -U setuptools pip wheel virtualenv EOF From 095a0b61da7d4629617db29d54333e8a9404b5c3 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 4 Nov 2017 15:59:33 +0100 Subject: [PATCH 116/798] borg list: remove tag-file options, fixes #3226 they are not used for borg list. (cherry picked from commit 702afa9e4883f92feb00dcecee54681a60792519) --- src/borg/archiver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 588350a8e1..995798b5db 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -3233,7 +3233,7 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): subparser.add_argument('paths', metavar='PATH', nargs='*', type=str, help='paths to list; patterns are supported') define_archive_filters_group(subparser) - define_exclusion_group(subparser, tag_files=True) + define_exclusion_group(subparser) mount_epilog = process_epilog(""" This command mounts an archive as a FUSE filesystem. This can be useful for From bdb3c0a38a1d5b9f4938cee922e060ffbf721140 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 4 Nov 2017 22:46:28 +0100 Subject: [PATCH 117/798] borg diff: remove tag-file options, fixes #3226 they are not used for borg diff. (cherry picked from commit 67f6aaa52b219df94a4c84abf0419def3b25ebc5) --- src/borg/archiver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 588350a8e1..57c5e3ae0f 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -3137,7 +3137,7 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): help='ARCHIVE2 name (no repository location allowed)') subparser.add_argument('paths', metavar='PATH', nargs='*', type=str, help='paths of items inside the archives to compare; patterns are supported') - define_exclusion_group(subparser, tag_files=True) + define_exclusion_group(subparser) rename_epilog = process_epilog(""" This command renames an archive in the repository. From 936498065c60a46a159adead70cb4bfe1054b743 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 3 Nov 2017 16:51:12 +0100 Subject: [PATCH 118/798] borg list: fix broken pipe handling, fixes #3245 always use sys.stdout.write as in main() we have replaced that with a handler that deals with BrokenPipe exceptions. all the encode / decode dance was just to remove surrogates, but that is not needed as format_item already does remove surrogates. (cherry picked from commit 726ef118bc3a86b1d2c91101149a3fc1149dec27) --- src/borg/archiver.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 995798b5db..95b6cad79d 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -1343,26 +1343,18 @@ def do_umount(self, args): @with_repository(compatibility=(Manifest.Operation.READ,)) def do_list(self, args, repository, manifest, key): """List archive or repository contents""" - if not hasattr(sys.stdout, 'buffer'): - # This is a shim for supporting unit tests replacing sys.stdout with e.g. StringIO, - # which doesn't have an underlying buffer (= lower file object). - def write(bytestring): - sys.stdout.write(bytestring.decode('utf-8', errors='replace')) - else: - write = sys.stdout.buffer.write - if args.location.archive: if args.json: self.print_error('The --json option is only valid for listing archives, not archive contents.') return self.exit_code - return self._list_archive(args, repository, manifest, key, write) + return self._list_archive(args, repository, manifest, key) else: if args.json_lines: self.print_error('The --json-lines option is only valid for listing archive contents, not archives.') return self.exit_code - return self._list_repository(args, repository, manifest, key, write) + return self._list_repository(args, repository, manifest, key) - def _list_archive(self, args, repository, manifest, key, write): + def _list_archive(self, args, repository, manifest, key): matcher = self.build_matcher(args.patterns, args.paths) if args.format is not None: format = args.format @@ -1377,7 +1369,7 @@ def _list_inner(cache): formatter = ItemFormatter(archive, format, json_lines=args.json_lines) for item in archive.iter_items(lambda item: matcher.match(item.path)): - write(safe_encode(formatter.format_item(item))) + sys.stdout.write(formatter.format_item(item)) # Only load the cache if it will be used if ItemFormatter.format_needs_cache(format): @@ -1388,7 +1380,7 @@ def _list_inner(cache): return self.exit_code - def _list_repository(self, args, repository, manifest, key, write): + def _list_repository(self, args, repository, manifest, key): if args.format is not None: format = args.format elif args.short: @@ -1403,7 +1395,7 @@ def _list_repository(self, args, repository, manifest, key, write): if args.json: output_data.append(formatter.get_item_data(archive_info)) else: - write(safe_encode(formatter.format_item(archive_info))) + sys.stdout.write(formatter.format_item(archive_info)) if args.json: json_print(basic_json_data(manifest, extra={ From 49c4cbb93c1d48a2d2014281c60e9b607bcf12ee Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 5 Nov 2017 00:48:17 +0100 Subject: [PATCH 119/798] stats: do not count data volume twice when checkpointing, fixes #3224 (cherry picked from commit 66cd1cd240f078e2d5c9e32656b42dccf2d62935) --- src/borg/archive.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index 98a6e2510d..6a1aae4d57 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -942,8 +942,9 @@ def chunk_processor(data): # if we created part files, we have referenced all chunks from the part files, # but we also will reference the same chunks also from the final, complete file: + dummy_stats = Statistics() # do not count this data volume twice for chunk in item.chunks: - cache.chunk_incref(chunk.id, stats, size=chunk.size) + cache.chunk_incref(chunk.id, dummy_stats, size=chunk.size) def process_stdin(self, path, cache): uid, gid = 0, 0 From ae09b1a5bc5b87987066e102cebde25c9881c19b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 5 Nov 2017 19:13:15 +0100 Subject: [PATCH 120/798] improve docs about --stats, fixes #3260 (cherry picked from commit dc68e8d1f06f4a749559fbf8fbb6847a8078c5e3) --- src/borg/archiver.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index e3b3a06c47..89e842b470 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -2859,6 +2859,11 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): (O, C and D, respectively), then the Number of files (N) processed so far, followed by the currently processed path. + When using ``--stats``, you will get some statistics about how much data was + added - the "This Archive" deduplicated size there is most interesting as that is + how much your repository will grow. + Please note that the "All archives" stats refer to the state after creation. + See the output of the "borg help patterns" command for more help on exclude patterns. See the output of the "borg help placeholders" command for more help on placeholders. @@ -3153,6 +3158,11 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): This command deletes an archive from the repository or the complete repository. Disk space is reclaimed accordingly. If you delete the complete repository, the local cache for it (if any) is also deleted. + + When using ``--stats``, you will get some statistics about how much data was + deleted - the "Deleted data" deduplicated size there is most interesting as + that is how much your repository will shrink. + Please note that the "All archives" stats refer to the state after deletion. """) subparser = subparsers.add_parser('delete', parents=[common_parser], add_help=False, description=self.do_delete.__doc__, @@ -3377,6 +3387,11 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): The ``--keep-last N`` option is doing the same as ``--keep-secondly N`` (and it will keep the last N archives under the assumption that you do not create more than one backup archive in the same second). + + When using ``--stats``, you will get some statistics about how much data was + deleted - the "Deleted data" deduplicated size there is most interesting as + that is how much your repository will shrink. + Please note that the "All archives" stats refer to the state after pruning. """) subparser = subparsers.add_parser('prune', parents=[common_parser], add_help=False, description=self.do_prune.__doc__, From e0d967aeb71935a59c6a88ef1bdaf42b257ecb2b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 5 Nov 2017 19:24:49 +0100 Subject: [PATCH 121/798] update / rephrase cygwin / WSL status, fixes #3174 (cherry picked from commit 61d6c58241c08af2d1e51752eff5c184630af015) --- README.rst | 4 ++-- docs/installation.rst | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 6762017fb9..46e9266b13 100644 --- a/README.rst +++ b/README.rst @@ -78,8 +78,8 @@ Main features * Mac OS X * FreeBSD * OpenBSD and NetBSD (no xattrs/ACLs support or binaries yet) - * Cygwin (not supported, no binaries yet) - * Linux Subsystem of Windows 10 (not supported) + * Cygwin (experimental, no binaries yet) + * Linux Subsystem of Windows 10 (experimental) **Free and Open Source Software** * security and functionality can be audited independently diff --git a/docs/installation.rst b/docs/installation.rst index de7a582aa8..9384462150 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -250,8 +250,7 @@ Cygwin ++++++ .. note:: - Running under Cygwin is experimental and has only been tested with Cygwin - (x86-64) v2.5.2. Remote repositories are known broken, local repositories should work. + Running under Cygwin is experimental and has not been tested much yet. Use the Cygwin installer to install the dependencies:: From e7ae63f1f629407380162294a32e78a2f6be0855 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 5 Nov 2017 20:06:23 +0100 Subject: [PATCH 122/798] clarify same-filesystem requirement for borg upgrade, fixes #2083 (cherry picked from commit fe746fa5948f948bb963830298d5f86b5e53ea12) --- src/borg/archiver.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 89e842b470..ac5d0bffd0 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -3499,13 +3499,13 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): borg delete borg - Unless ``--inplace`` is specified, the upgrade process first - creates a backup copy of the repository, in - REPOSITORY.before-upgrade-DATETIME, using hardlinks. This takes - longer than in place upgrades, but is much safer and gives - progress information (as opposed to ``cp -al``). Once you are - satisfied with the conversion, you can safely destroy the - backup copy. + Unless ``--inplace`` is specified, the upgrade process first creates a backup + copy of the repository, in REPOSITORY.before-upgrade-DATETIME, using hardlinks. + This requires that the repository and its parent directory reside on same + filesystem so the hardlink copy can work. + This takes longer than in place upgrades, but is much safer and gives + progress information (as opposed to ``cp -al``). Once you are satisfied + with the conversion, you can safely destroy the backup copy. WARNING: Running the upgrade in place will make the current copy unusable with older version, with no way of going back From 304c9a1dc6608217bf6279f1ee823e4e505e9548 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 5 Nov 2017 20:48:49 +0100 Subject: [PATCH 123/798] docs: borg does not respect nodump flag by default any more (cherry picked from commit 7ddf7c451fc9d13a31f244283778b9d30d2dc12e) --- src/borg/archiver.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index ac5d0bffd0..cc37c13072 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -2883,9 +2883,6 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): and not include any other contents of the containing folder, this can be enabled through using the ``--keep-exclude-tags`` option. - Borg respects the nodump flag. Files flagged nodump will be marked as excluded (x) - in ``--list`` output. - Item flags ++++++++++ From 0c7eaec99dc27681958cea0b5bb69ecd39ea1599 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 5 Nov 2017 21:35:58 +0100 Subject: [PATCH 124/798] docs: more than 23 is not supported for CHUNK_MAX_EXP, fixes #3115 (cherry picked from commit e7fd1ff2ee141731ceb9cee8c29c24f8f4c836ab) --- docs/misc/create_chunker-params.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/misc/create_chunker-params.txt b/docs/misc/create_chunker-params.txt index 3e322b6608..af602c5c59 100644 --- a/docs/misc/create_chunker-params.txt +++ b/docs/misc/create_chunker-params.txt @@ -7,6 +7,7 @@ CHUNK_MIN_EXP and CHUNK_MAX_EXP give the exponent N of the 2^N minimum and maximum chunk size. Required: CHUNK_MIN_EXP < CHUNK_MAX_EXP. Defaults: 19 (2^19 == 512KiB) minimum, 23 (2^23 == 8MiB) maximum. +Currently it is not supported to give more than 23 as maximum. HASH_MASK_BITS is the number of least-significant bits of the rolling hash that need to be zero to trigger a chunk cut. From ff3910637d9c716624827485b9c1338a1e1e68eb Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 5 Nov 2017 06:42:59 +0100 Subject: [PATCH 125/798] update CHANGES (1.1-maint) --- docs/changes.rst | 47 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/docs/changes.rst b/docs/changes.rst index dcde0d2a31..3d3c2b6865 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -131,7 +131,7 @@ The best check that everything is ok is to run a dry-run extraction:: Changelog ========= -Version 1.1.1 (2017-10-22) +Version 1.1.2 (2017-11-05) -------------------------- Compatibility notes: @@ -149,6 +149,51 @@ Compatibility notes: You can avoid the one-time slowdown by using the pre-1.1.0rc4-compatible mode (but that is less safe for detecting changed files than the default). See the --files-cache docs for details. + +Fixes: + +- fix KeyError crash when talking to borg server < 1.0.7, #3244 +- extract: set bsdflags last (include immutable flag), #3263 +- create: don't do stat() call on excluded-norecurse directory, fix exception + handling for stat() call, #3209 +- create --stats: do not count data volume twice when checkpointing, #3224 +- recreate: move chunks_healthy when excluding hardlink master, #3228 +- recreate: get rid of chunks_healthy when rechunking (does not match), #3218 +- check: get rid of already existing not matching chunks_healthy metadata, #3218 +- list: fix stdout broken pipe handling, #3245 +- list/diff: remove tag-file options (not used), #3226 + +New features: + +- bash, zsh and fish shell auto-completions, see scripts/shell_completions/ +- added BORG_CONFIG_DIR env var, #3083 + +Other changes: + +- docs: + + - clarify using a blank passphrase in keyfile mode + - mention "!" (exclude-norecurse) type in "patterns" help + - document to first heal before running borg recreate to re-chunk stuff, + because that will have to get rid of chunks_healthy metadata. + - more than 23 is not supported for CHUNK_MAX_EXP, #3115 + - borg does not respect nodump flag by default any more + - clarify same-filesystem requirement for borg upgrade, #2083 + - update / rephrase cygwin / WSL status, #3174 + - improve docs about --stats, #3260 +- vagrant: openindiana new clang package + +Already contained in 1.1.1 (last minute fix): + +- arg parsing: fix fallback function, refactor, #3205. This is a fixup + for #3155, which was broken on at least python <= 3.4.2. + + +Version 1.1.1 (2017-10-22) +-------------------------- + +Compatibility notes: + - The deprecated --no-files-cache is not a global/common option any more, but only available for borg create (it is not needed for anything else). Use --files-cache=disabled instead of --no-files-cache. From 851e85b42671d08b367cdcbf5ce44a114cc76367 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 5 Nov 2017 23:09:14 +0100 Subject: [PATCH 126/798] build_usage --- docs/usage/create.rst.inc | 8 +++-- docs/usage/delete.rst.inc | 7 +++- docs/usage/diff.rst.inc | 71 ++++++++++++++++--------------------- docs/usage/help.rst.inc | 6 ++-- docs/usage/list.rst.inc | 9 ----- docs/usage/prune.rst.inc | 7 +++- docs/usage/recreate.rst.inc | 13 ++++++- docs/usage/upgrade.rst.inc | 14 ++++---- 8 files changed, 71 insertions(+), 64 deletions(-) diff --git a/docs/usage/create.rst.inc b/docs/usage/create.rst.inc index 1ef6c23b5b..1b6d22167d 100644 --- a/docs/usage/create.rst.inc +++ b/docs/usage/create.rst.inc @@ -212,6 +212,11 @@ The ``--progress`` option shows (from left to right) Original, Compressed and De (O, C and D, respectively), then the Number of files (N) processed so far, followed by the currently processed path. +When using ``--stats``, you will get some statistics about how much data was +added - the "This Archive" deduplicated size there is most interesting as that is +how much your repository will grow. +Please note that the "All archives" stats refer to the state after creation. + See the output of the "borg help patterns" command for more help on exclude patterns. See the output of the "borg help placeholders" command for more help on placeholders. @@ -231,9 +236,6 @@ only include the objects specified by ``--exclude-if-present`` in your backup, and not include any other contents of the containing folder, this can be enabled through using the ``--keep-exclude-tags`` option. -Borg respects the nodump flag. Files flagged nodump will be marked as excluded (x) -in ``--list`` output. - Item flags ++++++++++ diff --git a/docs/usage/delete.rst.inc b/docs/usage/delete.rst.inc index 93a3ec3cb8..5951c71a27 100644 --- a/docs/usage/delete.rst.inc +++ b/docs/usage/delete.rst.inc @@ -85,4 +85,9 @@ Description This command deletes an archive from the repository or the complete repository. Disk space is reclaimed accordingly. If you delete the complete repository, the -local cache for it (if any) is also deleted. \ No newline at end of file +local cache for it (if any) is also deleted. + +When using ``--stats``, you will get some statistics about how much data was +deleted - the "Deleted data" deduplicated size there is most interesting as +that is how much your repository will shrink. +Please note that the "All archives" stats refer to the state after deletion. \ No newline at end of file diff --git a/docs/usage/diff.rst.inc b/docs/usage/diff.rst.inc index ef5b831a60..5074a09939 100644 --- a/docs/usage/diff.rst.inc +++ b/docs/usage/diff.rst.inc @@ -12,43 +12,37 @@ borg diff .. class:: borg-options-table - +-------------------------------------------------------+-----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ - | **positional arguments** | - +-------------------------------------------------------+-----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ - | | ``REPO_ARCHIVE1`` | repository location and ARCHIVE1 name | - +-------------------------------------------------------+-----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ - | | ``ARCHIVE2`` | ARCHIVE2 name (no repository location allowed) | - +-------------------------------------------------------+-----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ - | | ``PATH`` | paths of items inside the archives to compare; patterns are supported | - +-------------------------------------------------------+-----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ - | **optional arguments** | - +-------------------------------------------------------+-----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ - | | ``--numeric-owner`` | only consider numeric user and group identifiers | - +-------------------------------------------------------+-----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ - | | ``--same-chunker-params`` | Override check of chunker parameters. | - +-------------------------------------------------------+-----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ - | | ``--sort`` | Sort the output lines by file path. | - +-------------------------------------------------------+-----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ - | .. class:: borg-common-opt-ref | - | | - | :ref:`common_options` | - +-------------------------------------------------------+-----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ - | **Exclusion options** | - +-------------------------------------------------------+-----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ - | | ``-e PATTERN``, ``--exclude PATTERN`` | exclude paths matching PATTERN | - +-------------------------------------------------------+-----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ - | | ``--exclude-from EXCLUDEFILE`` | read exclude patterns from EXCLUDEFILE, one per line | - +-------------------------------------------------------+-----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ - | | ``--pattern PATTERN`` | experimental: include/exclude paths matching PATTERN | - +-------------------------------------------------------+-----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ - | | ``--patterns-from PATTERNFILE`` | experimental: read include/exclude patterns from PATTERNFILE, one per line | - +-------------------------------------------------------+-----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ - | | ``--exclude-caches`` | exclude directories that contain a CACHEDIR.TAG file (http://www.brynosaurus.com/cachedir/spec.html) | - +-------------------------------------------------------+-----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ - | | ``--exclude-if-present NAME`` | exclude directories that are tagged by containing a filesystem object with the given NAME | - +-------------------------------------------------------+-----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ - | | ``--keep-exclude-tags``, ``--keep-tag-files`` | if tag objects are specified with ``--exclude-if-present``, don't omit the tag objects themselves from the backup archive | - +-------------------------------------------------------+-----------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ + +-------------------------------------------------------+---------------------------------------+----------------------------------------------------------------------------+ + | **positional arguments** | + +-------------------------------------------------------+---------------------------------------+----------------------------------------------------------------------------+ + | | ``REPO_ARCHIVE1`` | repository location and ARCHIVE1 name | + +-------------------------------------------------------+---------------------------------------+----------------------------------------------------------------------------+ + | | ``ARCHIVE2`` | ARCHIVE2 name (no repository location allowed) | + +-------------------------------------------------------+---------------------------------------+----------------------------------------------------------------------------+ + | | ``PATH`` | paths of items inside the archives to compare; patterns are supported | + +-------------------------------------------------------+---------------------------------------+----------------------------------------------------------------------------+ + | **optional arguments** | + +-------------------------------------------------------+---------------------------------------+----------------------------------------------------------------------------+ + | | ``--numeric-owner`` | only consider numeric user and group identifiers | + +-------------------------------------------------------+---------------------------------------+----------------------------------------------------------------------------+ + | | ``--same-chunker-params`` | Override check of chunker parameters. | + +-------------------------------------------------------+---------------------------------------+----------------------------------------------------------------------------+ + | | ``--sort`` | Sort the output lines by file path. | + +-------------------------------------------------------+---------------------------------------+----------------------------------------------------------------------------+ + | .. class:: borg-common-opt-ref | + | | + | :ref:`common_options` | + +-------------------------------------------------------+---------------------------------------+----------------------------------------------------------------------------+ + | **Exclusion options** | + +-------------------------------------------------------+---------------------------------------+----------------------------------------------------------------------------+ + | | ``-e PATTERN``, ``--exclude PATTERN`` | exclude paths matching PATTERN | + +-------------------------------------------------------+---------------------------------------+----------------------------------------------------------------------------+ + | | ``--exclude-from EXCLUDEFILE`` | read exclude patterns from EXCLUDEFILE, one per line | + +-------------------------------------------------------+---------------------------------------+----------------------------------------------------------------------------+ + | | ``--pattern PATTERN`` | experimental: include/exclude paths matching PATTERN | + +-------------------------------------------------------+---------------------------------------+----------------------------------------------------------------------------+ + | | ``--patterns-from PATTERNFILE`` | experimental: read include/exclude patterns from PATTERNFILE, one per line | + +-------------------------------------------------------+---------------------------------------+----------------------------------------------------------------------------+ .. raw:: html @@ -82,9 +76,6 @@ borg diff --exclude-from EXCLUDEFILE read exclude patterns from EXCLUDEFILE, one per line --pattern PATTERN experimental: include/exclude paths matching PATTERN --patterns-from PATTERNFILE experimental: read include/exclude patterns from PATTERNFILE, one per line - --exclude-caches exclude directories that contain a CACHEDIR.TAG file (http://www.brynosaurus.com/cachedir/spec.html) - --exclude-if-present NAME exclude directories that are tagged by containing a filesystem object with the given NAME - --keep-exclude-tags, --keep-tag-files if tag objects are specified with ``--exclude-if-present``, don't omit the tag objects themselves from the backup archive Description diff --git a/docs/usage/help.rst.inc b/docs/usage/help.rst.inc index 152e01b540..c5fb6a71e5 100644 --- a/docs/usage/help.rst.inc +++ b/docs/usage/help.rst.inc @@ -124,10 +124,12 @@ Examples:: may specify the backup roots (starting points) and patterns for inclusion/exclusion. A root path starts with the prefix `R`, followed by a path (a plain path, not a file pattern). An include rule starts with the prefix +, an exclude rule starts - with the prefix -, both followed by a pattern. + with the prefix -, an exclude-norecurse rule starts with !, all followed by a pattern. Inclusion patterns are useful to include paths that are contained in an excluded path. The first matching pattern is used so if an include pattern matches before - an exclude pattern, the file is backed up. + an exclude pattern, the file is backed up. If an exclude-norecurse pattern matches + a directory, it won't recurse into it and won't discover any potential matches for + include rules below that directory. Note that the default pattern style for ``--pattern`` and ``--patterns-from`` is shell style (`sh:`), so those patterns behave similar to rsync include/exclude diff --git a/docs/usage/list.rst.inc b/docs/usage/list.rst.inc index e6b8988840..6cf097665c 100644 --- a/docs/usage/list.rst.inc +++ b/docs/usage/list.rst.inc @@ -55,12 +55,6 @@ borg list +-----------------------------------------------------------------------------+-----------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``--patterns-from PATTERNFILE`` | experimental: read include/exclude patterns from PATTERNFILE, one per line | +-----------------------------------------------------------------------------+-----------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ - | | ``--exclude-caches`` | exclude directories that contain a CACHEDIR.TAG file (http://www.brynosaurus.com/cachedir/spec.html) | - +-----------------------------------------------------------------------------+-----------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ - | | ``--exclude-if-present NAME`` | exclude directories that are tagged by containing a filesystem object with the given NAME | - +-----------------------------------------------------------------------------+-----------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ - | | ``--keep-exclude-tags``, ``--keep-tag-files`` | if tag objects are specified with ``--exclude-if-present``, don't omit the tag objects themselves from the backup archive | - +-----------------------------------------------------------------------------+-----------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ .. raw:: html @@ -101,9 +95,6 @@ borg list --exclude-from EXCLUDEFILE read exclude patterns from EXCLUDEFILE, one per line --pattern PATTERN experimental: include/exclude paths matching PATTERN --patterns-from PATTERNFILE experimental: read include/exclude patterns from PATTERNFILE, one per line - --exclude-caches exclude directories that contain a CACHEDIR.TAG file (http://www.brynosaurus.com/cachedir/spec.html) - --exclude-if-present NAME exclude directories that are tagged by containing a filesystem object with the given NAME - --keep-exclude-tags, --keep-tag-files if tag objects are specified with ``--exclude-if-present``, don't omit the tag objects themselves from the backup archive Description diff --git a/docs/usage/prune.rst.inc b/docs/usage/prune.rst.inc index f98fbab847..81c2d32f94 100644 --- a/docs/usage/prune.rst.inc +++ b/docs/usage/prune.rst.inc @@ -134,4 +134,9 @@ negative number of archives to keep means that there is no limit. The ``--keep-last N`` option is doing the same as ``--keep-secondly N`` (and it will keep the last N archives under the assumption that you do not create more than one -backup archive in the same second). \ No newline at end of file +backup archive in the same second). + +When using ``--stats``, you will get some statistics about how much data was +deleted - the "Deleted data" deduplicated size there is most interesting as +that is how much your repository will shrink. +Please note that the "All archives" stats refer to the state after pruning. \ No newline at end of file diff --git a/docs/usage/recreate.rst.inc b/docs/usage/recreate.rst.inc index ef258e6941..b3071ab3da 100644 --- a/docs/usage/recreate.rst.inc +++ b/docs/usage/recreate.rst.inc @@ -150,4 +150,15 @@ With ``--target`` the original archive is not replaced, instead a new archive is When rechunking space usage can be substantial, expect at least the entire deduplicated size of the archives using the previous chunker params. When recompressing expect approx. (throughput / checkpoint-interval) in space usage, -assuming all chunks are recompressed. \ No newline at end of file +assuming all chunks are recompressed. + +If you recently ran borg check --repair and it had to fix lost chunks with all-zero +replacement chunks, please first run another backup for the same data and re-run +borg check --repair afterwards to heal any archives that had lost chunks which are +still generated from the input data. + +Important: running borg recreate to re-chunk will remove the chunks_healthy +metadata of all items with replacement chunks, so healing will not be possible +any more after re-chunking (it is also unlikely it would ever work: due to the +change of chunking parameters, the missing chunk likely will never be seen again +even if you still have the data that produced it). \ No newline at end of file diff --git a/docs/usage/upgrade.rst.inc b/docs/usage/upgrade.rst.inc index eafa4362ae..47a240dd06 100644 --- a/docs/usage/upgrade.rst.inc +++ b/docs/usage/upgrade.rst.inc @@ -130,13 +130,13 @@ make sure the cache files are also removed: borg delete borg -Unless ``--inplace`` is specified, the upgrade process first -creates a backup copy of the repository, in -REPOSITORY.before-upgrade-DATETIME, using hardlinks. This takes -longer than in place upgrades, but is much safer and gives -progress information (as opposed to ``cp -al``). Once you are -satisfied with the conversion, you can safely destroy the -backup copy. +Unless ``--inplace`` is specified, the upgrade process first creates a backup +copy of the repository, in REPOSITORY.before-upgrade-DATETIME, using hardlinks. +This requires that the repository and its parent directory reside on same +filesystem so the hardlink copy can work. +This takes longer than in place upgrades, but is much safer and gives +progress information (as opposed to ``cp -al``). Once you are satisfied +with the conversion, you can safely destroy the backup copy. WARNING: Running the upgrade in place will make the current copy unusable with older version, with no way of going back From 3097c059ffbe8cffcad4b3b2ffdd56b27de321b0 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 5 Nov 2017 23:09:39 +0100 Subject: [PATCH 127/798] build_man --- docs/man/borg-benchmark-crud.1 | 2 +- docs/man/borg-benchmark.1 | 2 +- docs/man/borg-break-lock.1 | 2 +- docs/man/borg-change-passphrase.1 | 2 +- docs/man/borg-check.1 | 2 +- docs/man/borg-common.1 | 2 +- docs/man/borg-compression.1 | 2 +- docs/man/borg-create.1 | 10 ++++++---- docs/man/borg-delete.1 | 7 ++++++- docs/man/borg-diff.1 | 11 +---------- docs/man/borg-export-tar.1 | 2 +- docs/man/borg-extract.1 | 2 +- docs/man/borg-info.1 | 2 +- docs/man/borg-init.1 | 2 +- docs/man/borg-key-change-passphrase.1 | 2 +- docs/man/borg-key-export.1 | 2 +- docs/man/borg-key-import.1 | 2 +- docs/man/borg-key-migrate-to-repokey.1 | 2 +- docs/man/borg-key.1 | 2 +- docs/man/borg-list.1 | 11 +---------- docs/man/borg-mount.1 | 2 +- docs/man/borg-patterns.1 | 8 +++++--- docs/man/borg-placeholders.1 | 2 +- docs/man/borg-prune.1 | 9 +++++++-- docs/man/borg-recreate.1 | 13 ++++++++++++- docs/man/borg-rename.1 | 2 +- docs/man/borg-serve.1 | 2 +- docs/man/borg-umount.1 | 2 +- docs/man/borg-upgrade.1 | 16 ++++++++-------- docs/man/borg-with-lock.1 | 2 +- docs/man/borg.1 | 3 +++ 31 files changed, 71 insertions(+), 61 deletions(-) diff --git a/docs/man/borg-benchmark-crud.1 b/docs/man/borg-benchmark-crud.1 index c9f1334229..d5fc412f44 100644 --- a/docs/man/borg-benchmark-crud.1 +++ b/docs/man/borg-benchmark-crud.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-BENCHMARK-CRUD 1 "2017-10-22" "" "borg backup tool" +.TH BORG-BENCHMARK-CRUD 1 "2017-11-05" "" "borg backup tool" .SH NAME borg-benchmark-crud \- Benchmark Create, Read, Update, Delete for archives. . diff --git a/docs/man/borg-benchmark.1 b/docs/man/borg-benchmark.1 index 1feed69bed..7e0e1a192d 100644 --- a/docs/man/borg-benchmark.1 +++ b/docs/man/borg-benchmark.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-BENCHMARK 1 "2017-10-22" "" "borg backup tool" +.TH BORG-BENCHMARK 1 "2017-11-05" "" "borg backup tool" .SH NAME borg-benchmark \- benchmark command . diff --git a/docs/man/borg-break-lock.1 b/docs/man/borg-break-lock.1 index 8f85ade5d5..7bf3d1d7a4 100644 --- a/docs/man/borg-break-lock.1 +++ b/docs/man/borg-break-lock.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-BREAK-LOCK 1 "2017-10-22" "" "borg backup tool" +.TH BORG-BREAK-LOCK 1 "2017-11-05" "" "borg backup tool" .SH NAME borg-break-lock \- Break the repository lock (e.g. in case it was left by a dead borg. . diff --git a/docs/man/borg-change-passphrase.1 b/docs/man/borg-change-passphrase.1 index a377dfaba7..5089fdb61a 100644 --- a/docs/man/borg-change-passphrase.1 +++ b/docs/man/borg-change-passphrase.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-CHANGE-PASSPHRASE 1 "2017-10-22" "" "borg backup tool" +.TH BORG-CHANGE-PASSPHRASE 1 "2017-11-05" "" "borg backup tool" .SH NAME borg-change-passphrase \- Change repository key file passphrase . diff --git a/docs/man/borg-check.1 b/docs/man/borg-check.1 index e0eadf5ab6..50282c8e9f 100644 --- a/docs/man/borg-check.1 +++ b/docs/man/borg-check.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-CHECK 1 "2017-10-22" "" "borg backup tool" +.TH BORG-CHECK 1 "2017-11-05" "" "borg backup tool" .SH NAME borg-check \- Check repository consistency . diff --git a/docs/man/borg-common.1 b/docs/man/borg-common.1 index 91e0f7ab71..9ccb7d96f1 100644 --- a/docs/man/borg-common.1 +++ b/docs/man/borg-common.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-COMMON 1 "2017-10-22" "" "borg backup tool" +.TH BORG-COMMON 1 "2017-11-05" "" "borg backup tool" .SH NAME borg-common \- Common options of Borg commands . diff --git a/docs/man/borg-compression.1 b/docs/man/borg-compression.1 index c4a444a3a1..0a504ea5d7 100644 --- a/docs/man/borg-compression.1 +++ b/docs/man/borg-compression.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-COMPRESSION 1 "2017-10-22" "" "borg backup tool" +.TH BORG-COMPRESSION 1 "2017-11-05" "" "borg backup tool" .SH NAME borg-compression \- Details regarding compression . diff --git a/docs/man/borg-create.1 b/docs/man/borg-create.1 index 0c47dc6c99..2405c1cc56 100644 --- a/docs/man/borg-create.1 +++ b/docs/man/borg-create.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-CREATE 1 "2017-10-22" "" "borg backup tool" +.TH BORG-CREATE 1 "2017-11-05" "" "borg backup tool" .SH NAME borg-create \- Create new archive . @@ -108,6 +108,11 @@ The \fB\-\-progress\fP option shows (from left to right) Original, Compressed an (O, C and D, respectively), then the Number of files (N) processed so far, followed by the currently processed path. .sp +When using \fB\-\-stats\fP, you will get some statistics about how much data was +added \- the "This Archive" deduplicated size there is most interesting as that is +how much your repository will grow. +Please note that the "All archives" stats refer to the state after creation. +.sp See the output of the "borg help patterns" command for more help on exclude patterns. See the output of the "borg help placeholders" command for more help on placeholders. .SH OPTIONS @@ -306,9 +311,6 @@ all of its contents will be omitted from the backup. If, however, you wish to only include the objects specified by \fB\-\-exclude\-if\-present\fP in your backup, and not include any other contents of the containing folder, this can be enabled through using the \fB\-\-keep\-exclude\-tags\fP option. -.sp -Borg respects the nodump flag. Files flagged nodump will be marked as excluded (x) -in \fB\-\-list\fP output. .SS Item flags .sp \fB\-\-list\fP outputs a list of all files, directories and other diff --git a/docs/man/borg-delete.1 b/docs/man/borg-delete.1 index 013ab288a5..f76ab2aa17 100644 --- a/docs/man/borg-delete.1 +++ b/docs/man/borg-delete.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-DELETE 1 "2017-10-22" "" "borg backup tool" +.TH BORG-DELETE 1 "2017-11-05" "" "borg backup tool" .SH NAME borg-delete \- Delete an existing repository or archives . @@ -38,6 +38,11 @@ borg [common options] delete [options] [TARGET] [ARCHIVE...] This command deletes an archive from the repository or the complete repository. Disk space is reclaimed accordingly. If you delete the complete repository, the local cache for it (if any) is also deleted. +.sp +When using \fB\-\-stats\fP, you will get some statistics about how much data was +deleted \- the "Deleted data" deduplicated size there is most interesting as +that is how much your repository will shrink. +Please note that the "All archives" stats refer to the state after deletion. .SH OPTIONS .sp See \fIborg\-common(1)\fP for common options of Borg commands. diff --git a/docs/man/borg-diff.1 b/docs/man/borg-diff.1 index 9a96832443..cab36023a9 100644 --- a/docs/man/borg-diff.1 +++ b/docs/man/borg-diff.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-DIFF 1 "2017-10-22" "" "borg backup tool" +.TH BORG-DIFF 1 "2017-11-05" "" "borg backup tool" .SH NAME borg-diff \- Diff contents of two archives . @@ -92,15 +92,6 @@ experimental: include/exclude paths matching PATTERN .TP .BI \-\-patterns\-from \ PATTERNFILE experimental: read include/exclude patterns from PATTERNFILE, one per line -.TP -.B \-\-exclude\-caches -exclude directories that contain a CACHEDIR.TAG file (\fI\%http://www.brynosaurus.com/cachedir/spec.html\fP) -.TP -.BI \-\-exclude\-if\-present \ NAME -exclude directories that are tagged by containing a filesystem object with the given NAME -.TP -.B \-\-keep\-exclude\-tags\fP,\fB \-\-keep\-tag\-files -if tag objects are specified with \fB\-\-exclude\-if\-present\fP, don\(aqt omit the tag objects themselves from the backup archive .UNINDENT .SH EXAMPLES .INDENT 0.0 diff --git a/docs/man/borg-export-tar.1 b/docs/man/borg-export-tar.1 index 3691b3c3b9..9d3c0f7d4e 100644 --- a/docs/man/borg-export-tar.1 +++ b/docs/man/borg-export-tar.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-EXPORT-TAR 1 "2017-10-22" "" "borg backup tool" +.TH BORG-EXPORT-TAR 1 "2017-11-05" "" "borg backup tool" .SH NAME borg-export-tar \- Export archive contents as a tarball . diff --git a/docs/man/borg-extract.1 b/docs/man/borg-extract.1 index e17adddf5a..76b090dc70 100644 --- a/docs/man/borg-extract.1 +++ b/docs/man/borg-extract.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-EXTRACT 1 "2017-10-22" "" "borg backup tool" +.TH BORG-EXTRACT 1 "2017-11-05" "" "borg backup tool" .SH NAME borg-extract \- Extract archive contents . diff --git a/docs/man/borg-info.1 b/docs/man/borg-info.1 index 0c9cce9566..ea075ff48a 100644 --- a/docs/man/borg-info.1 +++ b/docs/man/borg-info.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-INFO 1 "2017-10-22" "" "borg backup tool" +.TH BORG-INFO 1 "2017-11-05" "" "borg backup tool" .SH NAME borg-info \- Show archive details such as disk space used . diff --git a/docs/man/borg-init.1 b/docs/man/borg-init.1 index ccf81dbfd7..0fb5417d91 100644 --- a/docs/man/borg-init.1 +++ b/docs/man/borg-init.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-INIT 1 "2017-10-22" "" "borg backup tool" +.TH BORG-INIT 1 "2017-11-05" "" "borg backup tool" .SH NAME borg-init \- Initialize an empty repository . diff --git a/docs/man/borg-key-change-passphrase.1 b/docs/man/borg-key-change-passphrase.1 index f76707ddb0..3894588452 100644 --- a/docs/man/borg-key-change-passphrase.1 +++ b/docs/man/borg-key-change-passphrase.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-CHANGE-PASSPHRASE 1 "2017-10-22" "" "borg backup tool" +.TH BORG-KEY-CHANGE-PASSPHRASE 1 "2017-11-05" "" "borg backup tool" .SH NAME borg-key-change-passphrase \- Change repository key file passphrase . diff --git a/docs/man/borg-key-export.1 b/docs/man/borg-key-export.1 index 6cea595f24..31f877966f 100644 --- a/docs/man/borg-key-export.1 +++ b/docs/man/borg-key-export.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-EXPORT 1 "2017-10-22" "" "borg backup tool" +.TH BORG-KEY-EXPORT 1 "2017-11-05" "" "borg backup tool" .SH NAME borg-key-export \- Export the repository key for backup . diff --git a/docs/man/borg-key-import.1 b/docs/man/borg-key-import.1 index ee08b97954..20fa7ceb13 100644 --- a/docs/man/borg-key-import.1 +++ b/docs/man/borg-key-import.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-IMPORT 1 "2017-10-22" "" "borg backup tool" +.TH BORG-KEY-IMPORT 1 "2017-11-05" "" "borg backup tool" .SH NAME borg-key-import \- Import the repository key from backup . diff --git a/docs/man/borg-key-migrate-to-repokey.1 b/docs/man/borg-key-migrate-to-repokey.1 index 154fcfe12d..7edaa023d7 100644 --- a/docs/man/borg-key-migrate-to-repokey.1 +++ b/docs/man/borg-key-migrate-to-repokey.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-MIGRATE-TO-REPOKEY 1 "2017-10-22" "" "borg backup tool" +.TH BORG-KEY-MIGRATE-TO-REPOKEY 1 "2017-11-05" "" "borg backup tool" .SH NAME borg-key-migrate-to-repokey \- Migrate passphrase -> repokey . diff --git a/docs/man/borg-key.1 b/docs/man/borg-key.1 index 32b4d701d8..a449f50742 100644 --- a/docs/man/borg-key.1 +++ b/docs/man/borg-key.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY 1 "2017-10-22" "" "borg backup tool" +.TH BORG-KEY 1 "2017-11-05" "" "borg backup tool" .SH NAME borg-key \- Manage a keyfile or repokey of a repository . diff --git a/docs/man/borg-list.1 b/docs/man/borg-list.1 index 409a74b23f..eb8761b8ef 100644 --- a/docs/man/borg-list.1 +++ b/docs/man/borg-list.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-LIST 1 "2017-10-22" "" "borg backup tool" +.TH BORG-LIST 1 "2017-11-05" "" "borg backup tool" .SH NAME borg-list \- List archive or repository contents . @@ -97,15 +97,6 @@ experimental: include/exclude paths matching PATTERN .TP .BI \-\-patterns\-from \ PATTERNFILE experimental: read include/exclude patterns from PATTERNFILE, one per line -.TP -.B \-\-exclude\-caches -exclude directories that contain a CACHEDIR.TAG file (\fI\%http://www.brynosaurus.com/cachedir/spec.html\fP) -.TP -.BI \-\-exclude\-if\-present \ NAME -exclude directories that are tagged by containing a filesystem object with the given NAME -.TP -.B \-\-keep\-exclude\-tags\fP,\fB \-\-keep\-tag\-files -if tag objects are specified with \fB\-\-exclude\-if\-present\fP, don\(aqt omit the tag objects themselves from the backup archive .UNINDENT .SH EXAMPLES .INDENT 0.0 diff --git a/docs/man/borg-mount.1 b/docs/man/borg-mount.1 index 1bc624f40d..97ab864b55 100644 --- a/docs/man/borg-mount.1 +++ b/docs/man/borg-mount.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-MOUNT 1 "2017-10-22" "" "borg backup tool" +.TH BORG-MOUNT 1 "2017-11-05" "" "borg backup tool" .SH NAME borg-mount \- Mount archive or an entire repository as a FUSE filesystem . diff --git a/docs/man/borg-patterns.1 b/docs/man/borg-patterns.1 index 531ad5556c..0377cc6ea9 100644 --- a/docs/man/borg-patterns.1 +++ b/docs/man/borg-patterns.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-PATTERNS 1 "2017-10-22" "" "borg backup tool" +.TH BORG-PATTERNS 1 "2017-11-05" "" "borg backup tool" .SH NAME borg-patterns \- Details regarding patterns . @@ -160,10 +160,12 @@ with the experimental \fB\-\-pattern\fP and \fB\-\-patterns\-from\fP options. Us may specify the backup roots (starting points) and patterns for inclusion/exclusion. A root path starts with the prefix \fIR\fP, followed by a path (a plain path, not a file pattern). An include rule starts with the prefix +, an exclude rule starts -with the prefix \-, both followed by a pattern. +with the prefix \-, an exclude\-norecurse rule starts with !, all followed by a pattern. Inclusion patterns are useful to include paths that are contained in an excluded path. The first matching pattern is used so if an include pattern matches before -an exclude pattern, the file is backed up. +an exclude pattern, the file is backed up. If an exclude\-norecurse pattern matches +a directory, it won\(aqt recurse into it and won\(aqt discover any potential matches for +include rules below that directory. .sp Note that the default pattern style for \fB\-\-pattern\fP and \fB\-\-patterns\-from\fP is shell style (\fIsh:\fP), so those patterns behave similar to rsync include/exclude diff --git a/docs/man/borg-placeholders.1 b/docs/man/borg-placeholders.1 index 31b5fb355d..e3491fd722 100644 --- a/docs/man/borg-placeholders.1 +++ b/docs/man/borg-placeholders.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-PLACEHOLDERS 1 "2017-10-22" "" "borg backup tool" +.TH BORG-PLACEHOLDERS 1 "2017-11-05" "" "borg backup tool" .SH NAME borg-placeholders \- Details regarding placeholders . diff --git a/docs/man/borg-prune.1 b/docs/man/borg-prune.1 index e2ffdb2540..58e35ff169 100644 --- a/docs/man/borg-prune.1 +++ b/docs/man/borg-prune.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-PRUNE 1 "2017-10-22" "" "borg backup tool" +.TH BORG-PRUNE 1 "2017-11-05" "" "borg backup tool" .SH NAME borg-prune \- Prune repository archives according to specified rules . @@ -73,6 +73,11 @@ negative number of archives to keep means that there is no limit. The \fB\-\-keep\-last N\fP option is doing the same as \fB\-\-keep\-secondly N\fP (and it will keep the last N archives under the assumption that you do not create more than one backup archive in the same second). +.sp +When using \fB\-\-stats\fP, you will get some statistics about how much data was +deleted \- the "Deleted data" deduplicated size there is most interesting as +that is how much your repository will shrink. +Please note that the "All archives" stats refer to the state after pruning. .SH OPTIONS .sp See \fIborg\-common(1)\fP for common options of Borg commands. @@ -171,7 +176,7 @@ $ borg prune \-v \-\-list \-\-keep\-within=10d \-\-keep\-weekly=4 \-\-keep\-mont .UNINDENT .sp There is also a visualized prune example in \fBdocs/misc/prune\-example.txt\fP: -.IP "System Message: ERROR/3 (docs/virtmanpage.rst:, line 140)" +.IP "System Message: ERROR/3 (docs/virtmanpage.rst:, line 145)" Unknown directive type "highlight". .INDENT 0.0 .INDENT 3.5 diff --git a/docs/man/borg-recreate.1 b/docs/man/borg-recreate.1 index 0768cd7485..4169e6f842 100644 --- a/docs/man/borg-recreate.1 +++ b/docs/man/borg-recreate.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-RECREATE 1 "2017-10-22" "" "borg backup tool" +.TH BORG-RECREATE 1 "2017-11-05" "" "borg backup tool" .SH NAME borg-recreate \- Re-create archives . @@ -71,6 +71,17 @@ When rechunking space usage can be substantial, expect at least the entire deduplicated size of the archives using the previous chunker params. When recompressing expect approx. (throughput / checkpoint\-interval) in space usage, assuming all chunks are recompressed. +.sp +If you recently ran borg check \-\-repair and it had to fix lost chunks with all\-zero +replacement chunks, please first run another backup for the same data and re\-run +borg check \-\-repair afterwards to heal any archives that had lost chunks which are +still generated from the input data. +.sp +Important: running borg recreate to re\-chunk will remove the chunks_healthy +metadata of all items with replacement chunks, so healing will not be possible +any more after re\-chunking (it is also unlikely it would ever work: due to the +change of chunking parameters, the missing chunk likely will never be seen again +even if you still have the data that produced it). .SH OPTIONS .sp See \fIborg\-common(1)\fP for common options of Borg commands. diff --git a/docs/man/borg-rename.1 b/docs/man/borg-rename.1 index c628299f18..55fcd86c88 100644 --- a/docs/man/borg-rename.1 +++ b/docs/man/borg-rename.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-RENAME 1 "2017-10-22" "" "borg backup tool" +.TH BORG-RENAME 1 "2017-11-05" "" "borg backup tool" .SH NAME borg-rename \- Rename an existing archive . diff --git a/docs/man/borg-serve.1 b/docs/man/borg-serve.1 index 3a567ee840..0aa10d97ea 100644 --- a/docs/man/borg-serve.1 +++ b/docs/man/borg-serve.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-SERVE 1 "2017-10-22" "" "borg backup tool" +.TH BORG-SERVE 1 "2017-11-05" "" "borg backup tool" .SH NAME borg-serve \- Start in server mode. This command is usually not used manually. . diff --git a/docs/man/borg-umount.1 b/docs/man/borg-umount.1 index 932a407500..8bae2dff5e 100644 --- a/docs/man/borg-umount.1 +++ b/docs/man/borg-umount.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-UMOUNT 1 "2017-10-22" "" "borg backup tool" +.TH BORG-UMOUNT 1 "2017-11-05" "" "borg backup tool" .SH NAME borg-umount \- un-mount the FUSE filesystem . diff --git a/docs/man/borg-upgrade.1 b/docs/man/borg-upgrade.1 index 0d5ef5905b..abe4dbf188 100644 --- a/docs/man/borg-upgrade.1 +++ b/docs/man/borg-upgrade.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-UPGRADE 1 "2017-10-22" "" "borg backup tool" +.TH BORG-UPGRADE 1 "2017-11-05" "" "borg backup tool" .SH NAME borg-upgrade \- upgrade a repository from a previous version . @@ -104,13 +104,13 @@ borg delete borg .UNINDENT .UNINDENT .sp -Unless \fB\-\-inplace\fP is specified, the upgrade process first -creates a backup copy of the repository, in -REPOSITORY.before\-upgrade\-DATETIME, using hardlinks. This takes -longer than in place upgrades, but is much safer and gives -progress information (as opposed to \fBcp \-al\fP). Once you are -satisfied with the conversion, you can safely destroy the -backup copy. +Unless \fB\-\-inplace\fP is specified, the upgrade process first creates a backup +copy of the repository, in REPOSITORY.before\-upgrade\-DATETIME, using hardlinks. +This requires that the repository and its parent directory reside on same +filesystem so the hardlink copy can work. +This takes longer than in place upgrades, but is much safer and gives +progress information (as opposed to \fBcp \-al\fP). Once you are satisfied +with the conversion, you can safely destroy the backup copy. .sp WARNING: Running the upgrade in place will make the current copy unusable with older version, with no way of going back diff --git a/docs/man/borg-with-lock.1 b/docs/man/borg-with-lock.1 index 27321d56a8..9f7951f4bd 100644 --- a/docs/man/borg-with-lock.1 +++ b/docs/man/borg-with-lock.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-WITH-LOCK 1 "2017-10-22" "" "borg backup tool" +.TH BORG-WITH-LOCK 1 "2017-11-05" "" "borg backup tool" .SH NAME borg-with-lock \- run a user specified command with the repository lock held . diff --git a/docs/man/borg.1 b/docs/man/borg.1 index 9f3da1e24a..703bdc4b5d 100644 --- a/docs/man/borg.1 +++ b/docs/man/borg.1 @@ -432,6 +432,9 @@ security relevant data. .B BORG_CACHE_DIR Default to \(aq~/.cache/borg\(aq. This directory contains the local cache and might need a lot of space for dealing with big repositories). +.TP +.B BORG_CONFIG_DIR +Default to \(aq~/.config/borg\(aq. This directory contains the whole config directories. .UNINDENT .TP .B Building: From ce7936b608bcc8199d4a6cee7ef1e3c950ff4a03 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 5 Nov 2017 23:52:15 +0100 Subject: [PATCH 128/798] vagrant: use https pypi url for wheezy http url stopped working --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 2eb9a462d8..10f100f628 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -31,7 +31,7 @@ def packages_debianoid(user) # this way it works on older dists (like ubuntu 12.04) also: # for python 3.2 on ubuntu 12.04 we need pip<8 and virtualenv<14 as # newer versions are not compatible with py 3.2 any more. - easy_install3 'pip<8.0' + easy_install3 -i https://pypi.python.org/simple/ 'pip<8.0' pip3 install 'virtualenv<14.0' EOF end From 07f36542a60c8b5c5aa8d37b745a689374230d70 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 6 Nov 2017 10:38:04 +0100 Subject: [PATCH 129/798] update release checklist about security fixes (cherry picked from commit 818b61935bba264cb6f3be8beb54c49ccbc67e92) --- docs/development.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/development.rst b/docs/development.rst index 831c8ef2fd..69ff946972 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -309,6 +309,7 @@ Checklist: - make sure all issues for this milestone are closed or moved to the next milestone +- check if there are any pending fixes for security issues - find and fix any low hanging fruit left on the issue tracker - check that Travis CI is happy - update ``CHANGES.rst``, based on ``git log $PREVIOUS_RELEASE..`` From 31031e9578af4ced926fb48d3ddd81b54ca19a5a Mon Sep 17 00:00:00 2001 From: Milkey Mouse Date: Thu, 9 Nov 2017 16:59:03 -0800 Subject: [PATCH 130/798] Show an error when --dry-run & --stats are both used (fixes #3298) --- src/borg/archiver.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index cc37c13072..c357ab779c 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -2861,8 +2861,9 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): When using ``--stats``, you will get some statistics about how much data was added - the "This Archive" deduplicated size there is most interesting as that is - how much your repository will grow. - Please note that the "All archives" stats refer to the state after creation. + how much your repository will grow. Please note that the "All archives" stats refer to + the state after creation. Also, the ``--stats`` and ``--dry-run`` options are mutually + exclusive because the data is not actually compressed and deduplicated during a dry run. See the output of the "borg help patterns" command for more help on exclude patterns. See the output of the "borg help placeholders" command for more help on placeholders. @@ -2930,10 +2931,12 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): help='create backup') subparser.set_defaults(func=self.do_create) - subparser.add_argument('-n', '--dry-run', dest='dry_run', action='store_true', + dryrun_group = subparser.add_mutually_exclusive_group() + dryrun_group.add_argument('-n', '--dry-run', dest='dry_run', action='store_true', help='do not create a backup archive') - subparser.add_argument('-s', '--stats', dest='stats', action='store_true', + dryrun_group.add_argument('-s', '--stats', dest='stats', action='store_true', help='print statistics for the created archive') + subparser.add_argument('--list', dest='output_list', action='store_true', help='output verbose list of items (files, dirs, ...)') subparser.add_argument('--filter', metavar='STATUSCHARS', dest='output_filter', From 68bb3792fb438aac4d8e8456d67250cf9789f2a3 Mon Sep 17 00:00:00 2001 From: Milkey Mouse Date: Mon, 6 Nov 2017 14:39:08 -0800 Subject: [PATCH 131/798] Clarify encrypted key format for borg key export (fixes #3296) --- src/borg/archiver.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 84c913a5e4..aae93c2a7e 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -2688,6 +2688,9 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): key_export_epilog = process_epilog(""" If repository encryption is used, the repository is inaccessible without the key. This command allows to backup this essential key. + Note that the backup produced does not include the passphrase itself + (i.e. the exported key stays encrypted). In order to regain access to a + repository, one needs both the exported key and the original passphrase. There are two backup formats. The normal backup format is suitable for digital storage as a file. The ``--paper`` backup format is optimized From e20b1ebbc46ec67d6517400ce73fd5424d68241c Mon Sep 17 00:00:00 2001 From: Milkey Mouse Date: Thu, 9 Nov 2017 16:13:46 -0800 Subject: [PATCH 132/798] Clarify create --stats duration vs. wall time (fixes #3301) --- docs/faq.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/faq.rst b/docs/faq.rst index 6726e15bc1..9e89ac7bcc 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -288,6 +288,14 @@ needs to be ascertained and fixed. issues. We recommend to first run without ``--repair`` to assess the situation. If the found issues and proposed repairs seem right, re-run "check" with ``--repair`` enabled. +Why is the time elapsed in the archive stats different from wall clock time? +---------------------------------------------------------------------------- + +Borg needs to write the time elapsed into the archive metadata before finalizing +the archive, compacting the segments, and committing the repo & cache. This means +when Borg is run with e.g. the ``time`` command, the duration shown in the archive +stats may be shorter than the full time the command runs for. + Security ######## From 18b934496c02dc4c32510af3f256214993561598 Mon Sep 17 00:00:00 2001 From: Milkey Mouse Date: Sun, 5 Nov 2017 12:21:08 -0800 Subject: [PATCH 133/798] Don't show sub-command in borgfs help (fixes #3287) --- src/borg/archiver.py | 117 ++++++++++++++++++++++--------------------- 1 file changed, 61 insertions(+), 56 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 84c913a5e4..19e3eec322 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -2457,7 +2457,67 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): mid_common_parser.set_defaults(paths=[], patterns=[]) parser.common_options.add_common_group(mid_common_parser, '_midcommand') - subparsers = parser.add_subparsers(title='required arguments', metavar='') + mount_epilog = process_epilog(""" + This command mounts an archive as a FUSE filesystem. This can be useful for + browsing an archive or restoring individual files. Unless the ``--foreground`` + option is given the command will run in the background until the filesystem + is ``umounted``. + + The command ``borgfs`` provides a wrapper for ``borg mount``. This can also be + used in fstab entries: + ``/path/to/repo /mnt/point fuse.borgfs defaults,noauto 0 0`` + + To allow a regular user to use fstab entries, add the ``user`` option: + ``/path/to/repo /mnt/point fuse.borgfs defaults,noauto,user 0 0`` + + For mount options, see the fuse(8) manual page. Additional mount options + supported by borg: + + - versions: when used with a repository mount, this gives a merged, versioned + view of the files in the archives. EXPERIMENTAL, layout may change in future. + - allow_damaged_files: by default damaged files (where missing chunks were + replaced with runs of zeros by borg check ``--repair``) are not readable and + return EIO (I/O error). Set this option to read such files. + + The BORG_MOUNT_DATA_CACHE_ENTRIES environment variable is meant for advanced users + to tweak the performance. It sets the number of cached data chunks; additional + memory usage can be up to ~8 MiB times this number. The default is the number + of CPU cores. + + When the daemonized process receives a signal or crashes, it does not unmount. + Unmounting in these cases could cause an active rsync or similar process + to unintentionally delete data. + + When running in the foreground ^C/SIGINT unmounts cleanly, but other + signals or crashes do not. + """) + + if parser.prog == 'borgfs': + parser.description = self.do_mount.__doc__ + parser.epilog = mount_epilog + parser.formatter_class = argparse.RawDescriptionHelpFormatter + parser.help = 'mount repository' + subparser = parser + else: + subparsers = parser.add_subparsers(title='required arguments', metavar='') + subparser = subparsers.add_parser('mount', parents=[common_parser], add_help=False, + description=self.do_mount.__doc__, + epilog=mount_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help='mount repository') + subparser.set_defaults(func=self.do_mount) + subparser.add_argument('location', metavar='REPOSITORY_OR_ARCHIVE', type=location_validator(), + help='repository/archive to mount') + subparser.add_argument('mountpoint', metavar='MOUNTPOINT', type=str, + help='where to mount filesystem') + subparser.add_argument('-f', '--foreground', dest='foreground', + action='store_true', + help='stay in foreground, do not daemonize') + subparser.add_argument('-o', dest='options', type=str, + help='Extra mount options') + define_archive_filters_group(subparser) + if parser.prog == 'borgfs': + return parser serve_epilog = process_epilog(""" This command starts a repository server process. This command is usually not used manually. @@ -3226,57 +3286,6 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): define_archive_filters_group(subparser) define_exclusion_group(subparser, tag_files=True) - mount_epilog = process_epilog(""" - This command mounts an archive as a FUSE filesystem. This can be useful for - browsing an archive or restoring individual files. Unless the ``--foreground`` - option is given the command will run in the background until the filesystem - is ``umounted``. - - The command ``borgfs`` provides a wrapper for ``borg mount``. This can also be - used in fstab entries: - ``/path/to/repo /mnt/point fuse.borgfs defaults,noauto 0 0`` - - To allow a regular user to use fstab entries, add the ``user`` option: - ``/path/to/repo /mnt/point fuse.borgfs defaults,noauto,user 0 0`` - - For mount options, see the fuse(8) manual page. Additional mount options - supported by borg: - - - versions: when used with a repository mount, this gives a merged, versioned - view of the files in the archives. EXPERIMENTAL, layout may change in future. - - allow_damaged_files: by default damaged files (where missing chunks were - replaced with runs of zeros by borg check ``--repair``) are not readable and - return EIO (I/O error). Set this option to read such files. - - The BORG_MOUNT_DATA_CACHE_ENTRIES environment variable is meant for advanced users - to tweak the performance. It sets the number of cached data chunks; additional - memory usage can be up to ~8 MiB times this number. The default is the number - of CPU cores. - - When the daemonized process receives a signal or crashes, it does not unmount. - Unmounting in these cases could cause an active rsync or similar process - to unintentionally delete data. - - When running in the foreground ^C/SIGINT unmounts cleanly, but other - signals or crashes do not. - """) - subparser = subparsers.add_parser('mount', parents=[common_parser], add_help=False, - description=self.do_mount.__doc__, - epilog=mount_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='mount repository') - subparser.set_defaults(func=self.do_mount) - subparser.add_argument('location', metavar='REPOSITORY_OR_ARCHIVE', type=location_validator(), - help='repository/archive to mount') - subparser.add_argument('mountpoint', metavar='MOUNTPOINT', type=str, - help='where to mount filesystem') - subparser.add_argument('-f', '--foreground', dest='foreground', - action='store_true', - help='stay in foreground, do not daemonize') - subparser.add_argument('-o', dest='options', type=str, - help='Extra mount options') - define_archive_filters_group(subparser) - umount_epilog = process_epilog(""" This command un-mounts a FUSE filesystem that was mounted with ``borg mount``. @@ -4007,10 +4016,6 @@ def sig_trace_handler(sig_no, stack): # pragma: no cover def main(): # pragma: no cover - # provide 'borg mount' behaviour when the main script/executable is named borgfs - if os.path.basename(sys.argv[0]) == "borgfs": - sys.argv.insert(1, "mount") - # Make sure stdout and stderr have errors='replace' to avoid unicode # issues when print()-ing unicode file names sys.stdout = ErrorIgnoringTextIOWrapper(sys.stdout.buffer, sys.stdout.encoding, 'replace', line_buffering=True) From eadb74eb6ff19aef8c8bd9493f40b780cee44a13 Mon Sep 17 00:00:00 2001 From: Milkey Mouse Date: Sun, 5 Nov 2017 14:46:25 -0800 Subject: [PATCH 134/798] Generate man page for borgfs (fixes #3216) --- docs/usage/borgfs.rst | 1 + setup.py | 24 ++++++++++++++++++------ 2 files changed, 19 insertions(+), 6 deletions(-) create mode 100644 docs/usage/borgfs.rst diff --git a/docs/usage/borgfs.rst b/docs/usage/borgfs.rst new file mode 100644 index 0000000000..162589ca09 --- /dev/null +++ b/docs/usage/borgfs.rst @@ -0,0 +1 @@ +.. include:: borgfs.rst.inc diff --git a/setup.py b/setup.py index 42b272613e..eaae5c417e 100644 --- a/setup.py +++ b/setup.py @@ -238,10 +238,11 @@ def run(self): # allows us to build docs without the C modules fully loaded during help generation from borg.archiver import Archiver parser = Archiver(prog='borg').build_parser() + borgfs_parser = Archiver(prog='borgfs').build_parser() - self.generate_level("", parser, Archiver) + self.generate_level("", parser, Archiver, {'borgfs': borgfs_parser}) - def generate_level(self, prefix, parser, Archiver): + def generate_level(self, prefix, parser, Archiver, extra_choices=None): is_subcommand = False choices = {} for action in parser._actions: @@ -249,6 +250,8 @@ def generate_level(self, prefix, parser, Archiver): is_subcommand = True for cmd, parser in action.choices.items(): choices[prefix + cmd] = parser + if extra_choices is not None: + choices.update(extra_choices) if prefix and not choices: return print('found commands: %s' % list(choices.keys())) @@ -501,12 +504,13 @@ def run(self): # allows us to build docs without the C modules fully loaded during help generation from borg.archiver import Archiver parser = Archiver(prog='borg').build_parser() + borgfs_parser = Archiver(prog='borgfs').build_parser() - self.generate_level('', parser, Archiver) + self.generate_level('', parser, Archiver, {'borgfs': borgfs_parser}) self.build_topic_pages(Archiver) self.build_intro_page() - def generate_level(self, prefix, parser, Archiver): + def generate_level(self, prefix, parser, Archiver, extra_choices=None): is_subcommand = False choices = {} for action in parser._actions: @@ -514,6 +518,8 @@ def generate_level(self, prefix, parser, Archiver): is_subcommand = True for cmd, parser in action.choices.items(): choices[prefix + cmd] = parser + if extra_choices is not None: + choices.update(extra_choices) if prefix and not choices: return @@ -521,7 +527,10 @@ def generate_level(self, prefix, parser, Archiver): if command.startswith('debug') or command == 'help': continue - man_title = 'borg-' + command.replace(' ', '-') + if command == "borgfs": + man_title = command + else: + man_title = 'borg-' + command.replace(' ', '-') print('building man page', man_title + '(1)', file=sys.stderr) is_intermediary = self.generate_level(command + ' ', parser, Archiver) @@ -536,7 +545,10 @@ def generate_level(self, prefix, parser, Archiver): write('| borg', '[common options]', command, subcommand, '...') self.see_also.setdefault(command, []).append('%s-%s' % (command, subcommand)) else: - write('borg', '[common options]', command, end='') + if command == "borgfs": + write(command, end='') + else: + write('borg', '[common options]', command, end='') self.write_usage(write, parser) write('\n') From 0ec703ff40161e46c197a642bfa8916e2a51761d Mon Sep 17 00:00:00 2001 From: Milkey Mouse Date: Sun, 12 Nov 2017 18:36:06 -0800 Subject: [PATCH 135/798] Check borgfs man formatting in tests I only realized after the first PR was merged that the code used for finding which man pages to generate was duplicated in the testsuite (since setup.py can't import from the installed module and vice versa.) These are essentially the same changes as made to setup.py in #3290. --- src/borg/testsuite/archiver.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index f6ada2f8a6..f404a4506c 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -3655,14 +3655,17 @@ def get_all_parsers(): Return dict mapping command to parser. """ parser = Archiver(prog='borg').build_parser() + borgfs_parser = Archiver(prog='borgfs').build_parser() parsers = {} - def discover_level(prefix, parser, Archiver): + def discover_level(prefix, parser, Archiver, extra_choices=None): choices = {} for action in parser._actions: if action.choices is not None and 'SubParsersAction' in str(action.__class__): for cmd, parser in action.choices.items(): choices[prefix + cmd] = parser + if extra_choices is not None: + choices.update(extra_choices) if prefix and not choices: return @@ -3670,7 +3673,7 @@ def discover_level(prefix, parser, Archiver): discover_level(command + " ", parser, Archiver) parsers[command] = parser - discover_level("", parser, Archiver) + discover_level("", parser, Archiver, {'borgfs': borgfs_parser}) return parsers From e51cf4314262afd9d4cf98b902f098fe3e528919 Mon Sep 17 00:00:00 2001 From: Sam H Date: Mon, 13 Nov 2017 08:55:10 -0500 Subject: [PATCH 136/798] include item birthtime in archive (where available) (#3313) include item birthtime in archive, fixes #3272 * use `safe_ns` when reading birthtime into attributes * proper order for `birthtime` in `ITEM_KEYS` list * use `bigint` wrapper for consistency * Add tests to verify that birthtime is normally preserved, but not preserved when `--nobirthtime` is passed to `borg create`. --- src/borg/archive.py | 18 ++++++++++++++++- src/borg/archiver.py | 4 +++- src/borg/constants.py | 2 +- src/borg/item.pyx | 1 + src/borg/testsuite/__init__.py | 22 +++++++++++++++++++++ src/borg/testsuite/archiver.py | 35 +++++++++++++++++++++++++++++++++- 6 files changed, 78 insertions(+), 4 deletions(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index 6a1aae4d57..239d00b76b 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -280,7 +280,7 @@ class IncompatibleFilesystemEncodingError(Error): """Failed to encode filename "{}" into file system encoding "{}". Consider configuring the LANG environment variable.""" def __init__(self, repository, key, manifest, name, cache=None, create=False, - checkpoint_interval=300, numeric_owner=False, noatime=False, noctime=False, nobsdflags=False, + checkpoint_interval=300, numeric_owner=False, noatime=False, noctime=False, nobirthtime=False, nobsdflags=False, progress=False, chunker_params=CHUNKER_PARAMS, start=None, start_monotonic=None, end=None, consider_part_files=False, log_json=False): self.cwd = os.getcwd() @@ -298,6 +298,7 @@ def __init__(self, repository, key, manifest, name, cache=None, create=False, self.numeric_owner = numeric_owner self.noatime = noatime self.noctime = noctime + self.nobirthtime = nobirthtime self.nobsdflags = nobsdflags assert (start is None) == (start_monotonic is None), 'Logic error: if start is given, start_monotonic must be given as well and vice versa.' if start is None: @@ -683,6 +684,18 @@ def restore_attrs(self, path, item, symlink=False, fd=None): else: # old archives only had mtime in item metadata atime = mtime + if 'birthtime' in item: + birthtime = item.birthtime + try: + # This should work on FreeBSD, NetBSD, and Darwin and be harmless on other platforms. + # See utimes(2) on either of the BSDs for details. + if fd: + os.utime(fd, None, ns=(atime, birthtime)) + else: + os.utime(path, None, ns=(atime, birthtime), follow_symlinks=False) + except OSError: + # some systems don't support calling utime on a symlink + pass try: if fd: os.utime(fd, None, ns=(atime, mtime)) @@ -822,6 +835,9 @@ def stat_simple_attrs(self, st): attrs['atime'] = safe_ns(st.st_atime_ns) if not self.noctime: attrs['ctime'] = safe_ns(st.st_ctime_ns) + if not self.nobirthtime and hasattr(st, 'st_birthtime'): + # sadly, there's no stat_result.st_birthtime_ns + attrs['birthtime'] = safe_ns(int(st.st_birthtime * 10**9)) if self.numeric_owner: attrs['user'] = attrs['group'] = None else: diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 9fe0e0dff9..329dd4a6dc 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -536,7 +536,7 @@ def create_inner(archive, cache): lock_wait=self.lock_wait, permit_adhoc_cache=args.no_cache_sync) as cache: archive = Archive(repository, key, manifest, args.location.archive, cache=cache, create=True, checkpoint_interval=args.checkpoint_interval, - numeric_owner=args.numeric_owner, noatime=args.noatime, noctime=args.noctime, + numeric_owner=args.numeric_owner, noatime=args.noatime, noctime=args.noctime, nobirthtime=args.nobirthtime, nobsdflags=args.nobsdflags, progress=args.progress, chunker_params=args.chunker_params, start=t0, start_monotonic=t0_monotonic, log_json=args.log_json) @@ -2964,6 +2964,8 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): help='do not store atime into archive') fs_group.add_argument('--noctime', dest='noctime', action='store_true', help='do not store ctime into archive') + fs_group.add_argument('--nobirthtime', dest='nobirthtime', action='store_true', + help='do not store birthtime (creation date) into archive') fs_group.add_argument('--nobsdflags', dest='nobsdflags', action='store_true', help='do not read and store bsdflags (e.g. NODUMP, IMMUTABLE) into archive') fs_group.add_argument('--ignore-inode', dest='ignore_inode', action='store_true', diff --git a/src/borg/constants.py b/src/borg/constants.py index 4f2a430da7..77129ea180 100644 --- a/src/borg/constants.py +++ b/src/borg/constants.py @@ -1,6 +1,6 @@ # this set must be kept complete, otherwise the RobustUnpacker might malfunction: ITEM_KEYS = frozenset(['path', 'source', 'rdev', 'chunks', 'chunks_healthy', 'hardlink_master', - 'mode', 'user', 'group', 'uid', 'gid', 'mtime', 'atime', 'ctime', 'size', + 'mode', 'user', 'group', 'uid', 'gid', 'mtime', 'atime', 'ctime', 'birthtime', 'size', 'xattrs', 'bsdflags', 'acl_nfs4', 'acl_access', 'acl_default', 'acl_extended', 'part']) diff --git a/src/borg/item.pyx b/src/borg/item.pyx index 91fe57ee13..7c698baf76 100644 --- a/src/borg/item.pyx +++ b/src/borg/item.pyx @@ -162,6 +162,7 @@ class Item(PropDict): atime = PropDict._make_property('atime', int, 'bigint', encode=int_to_bigint, decode=bigint_to_int) ctime = PropDict._make_property('ctime', int, 'bigint', encode=int_to_bigint, decode=bigint_to_int) mtime = PropDict._make_property('mtime', int, 'bigint', encode=int_to_bigint, decode=bigint_to_int) + birthtime = PropDict._make_property('birthtime', int, 'bigint', encode=int_to_bigint, decode=bigint_to_int) # size is only present for items with a chunk list and then it is sum(chunk_sizes) # compatibility note: this is a new feature, in old archives size will be missing. diff --git a/src/borg/testsuite/__init__.py b/src/borg/testsuite/__init__.py index 640227b13b..b9c63e0540 100644 --- a/src/borg/testsuite/__init__.py +++ b/src/borg/testsuite/__init__.py @@ -116,6 +116,28 @@ def is_utime_fully_supported(): return False +@functools.lru_cache() +def is_birthtime_fully_supported(): + if not hasattr(os.stat_result, 'st_birthtime'): + return False + with unopened_tempfile() as filepath: + # Some filesystems (such as SSHFS) don't support utime on symlinks + if are_symlinks_supported(): + os.symlink('something', filepath) + else: + open(filepath, 'w').close() + try: + birthtime, mtime, atime = 946598400, 946684800, 946771200 + os.utime(filepath, (atime, birthtime), follow_symlinks=False) + os.utime(filepath, (atime, mtime), follow_symlinks=False) + new_stats = os.stat(filepath, follow_symlinks=False) + if new_stats.st_birthtime == birthtime and new_stats.st_mtime == mtime and new_stats.st_atime == atime: + return True + except OSError as err: + pass + return False + + def no_selinux(x): # selinux fails our FUSE tests, thus ignore selinux xattrs SELINUX_KEY = 'security.selinux' diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index f6ada2f8a6..a38667e6a1 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -53,7 +53,7 @@ from ..repository import Repository from . import has_lchflags, has_llfuse from . import BaseTestCase, changedir, environment_variable, no_selinux -from . import are_symlinks_supported, are_hardlinks_supported, are_fifos_supported, is_utime_fully_supported +from . import are_symlinks_supported, are_hardlinks_supported, are_fifos_supported, is_utime_fully_supported, is_birthtime_fully_supported from .platform import fakeroot_detected from .upgrader import attic_repo from . import key @@ -493,6 +493,39 @@ def has_noatime(some_file): # it touched the input file's atime while backing it up assert sto.st_atime_ns == atime * 1e9 + @pytest.mark.skipif(not is_utime_fully_supported(), reason='cannot properly setup and execute test without utime') + @pytest.mark.skipif(not is_birthtime_fully_supported(), reason='cannot properly setup and execute test without birthtime') + def test_birthtime(self): + self.create_test_files() + birthtime, mtime, atime = 946598400, 946684800, 946771200 + os.utime('input/file1', (atime, birthtime)) + os.utime('input/file1', (atime, mtime)) + self.cmd('init', '--encryption=repokey', self.repository_location) + self.cmd('create', self.repository_location + '::test', 'input') + with changedir('output'): + self.cmd('extract', self.repository_location + '::test') + sti = os.stat('input/file1') + sto = os.stat('output/input/file1') + assert int(sti.st_birthtime * 1e9) == int(sto.st_birthtime * 1e9) == birthtime * 1e9 + assert sti.st_mtime_ns == sto.st_mtime_ns == mtime * 1e9 + + @pytest.mark.skipif(not is_utime_fully_supported(), reason='cannot properly setup and execute test without utime') + @pytest.mark.skipif(not is_birthtime_fully_supported(), reason='cannot properly setup and execute test without birthtime') + def test_nobirthtime(self): + self.create_test_files() + birthtime, mtime, atime = 946598400, 946684800, 946771200 + os.utime('input/file1', (atime, birthtime)) + os.utime('input/file1', (atime, mtime)) + self.cmd('init', '--encryption=repokey', self.repository_location) + self.cmd('create', '--nobirthtime', self.repository_location + '::test', 'input') + with changedir('output'): + self.cmd('extract', self.repository_location + '::test') + sti = os.stat('input/file1') + sto = os.stat('output/input/file1') + assert int(sti.st_birthtime * 1e9) == birthtime * 1e9 + assert int(sto.st_birthtime * 1e9) == mtime * 1e9 + assert sti.st_mtime_ns == sto.st_mtime_ns == mtime * 1e9 + def _extract_repository_id(self, path): with Repository(self.repository_path) as repository: return repository.id From 3d3f3500f9af67f5dfa8fec59f759c111a4aa009 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 13 Nov 2017 16:47:03 +0100 Subject: [PATCH 137/798] crc32: deal with unaligned buffer, tests, fixes #3317 fixing only the (generic) slice-by-8 crc32 implementation, it is assumed that CPUs supporting CLMUL can also efficiently and correctly deal with unaligned accesses. slice-by-8 is used e.g. on ARM cpus and they might not (efficiently) support unaligned memory access, leading to bus errors or low performance. (cherry picked from commit f9cd6f7512a7fed267c349c9de2d857591a74f91) --- src/borg/algorithms/crc32_slice_by_8.c | 20 ++++++++++++++++++-- src/borg/testsuite/checksums.py | 17 +++++++++++------ 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/src/borg/algorithms/crc32_slice_by_8.c b/src/borg/algorithms/crc32_slice_by_8.c index dcfd8b8f97..b289fbb87c 100644 --- a/src/borg/algorithms/crc32_slice_by_8.c +++ b/src/borg/algorithms/crc32_slice_by_8.c @@ -330,12 +330,28 @@ const uint32_t Crc32Lookup[8][256] = uint32_t crc32_slice_by_8(const void* data, size_t length, uint32_t previousCrc32) { uint32_t crc = ~previousCrc32; // same as previousCrc32 ^ 0xFFFFFFFF - const uint32_t* current = (const uint32_t*) data; + + const uint32_t* current; + const uint8_t* currentChar; // enabling optimization (at least -O2) automatically unrolls the inner for-loop const size_t Unroll = 4; const size_t BytesAtOnce = 8 * Unroll; - const uint8_t* currentChar; + + currentChar = (const uint8_t*) data; + + // wanted: 32 bit / 4 Byte alignment, compute leading, unaligned bytes length + uintptr_t unaligned_length = (4 - (((uintptr_t) currentChar) & 3)) & 3; + // process unaligned bytes, if any (standard algorithm) + while ((length != 0) && (unaligned_length != 0)) + { + crc = (crc >> 8) ^ Crc32Lookup[0][(crc & 0xFF) ^ *currentChar++]; + length--; + unaligned_length--; + } + + // pointer points to 32bit aligned address now + current = (const uint32_t*) currentChar; // process 4x eight bytes at once (Slicing-by-8) while (length >= BytesAtOnce) diff --git a/src/borg/testsuite/checksums.py b/src/borg/testsuite/checksums.py index 5b0d9fb9d1..ca02d709c9 100644 --- a/src/borg/testsuite/checksums.py +++ b/src/borg/testsuite/checksums.py @@ -14,13 +14,18 @@ @pytest.mark.parametrize('implementation', crc32_implementations) def test_crc32(implementation): - # This includes many critical values, like zero length, 3/4/5, 6/7/8 and so on which are near and on - # alignment boundaries. This is of course just a sanity check ie. "did it compile all right?". - data = os.urandom(256) + # This includes many critical values, like misc. length and misc. aligned start addresses. + data = os.urandom(300) + mv = memoryview(data) initial_crc = 0x12345678 - for i in range(0, 256): - d = data[:i] - assert zlib.crc32(d, initial_crc) == implementation(d, initial_crc) + for start in range(0, 4): # 4B / int32 alignment, head processing + for length in [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, + 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, + 127, 128, 129, 130, 131, 132, 133, 134, 135, + 255, 256, 257, ]: + d = mv[start:start+length] + assert zlib.crc32(d, initial_crc) == implementation(d, initial_crc) def test_xxh64(): From fda9b7d5b93ce5b8ae51dcf915849c719c0bf6c1 Mon Sep 17 00:00:00 2001 From: Milkey Mouse Date: Mon, 20 Nov 2017 10:48:49 -0800 Subject: [PATCH 138/798] Mention break-lock in checkpointing FAQ entry (fixes #3328) --- docs/faq.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 53b1503a1c..47977e9e05 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -132,10 +132,11 @@ completed is useful because it references all the transmitted chunks up to the checkpoint. This means that in case of an interruption, you only need to retransfer the data since the last checkpoint. -If a backup was interrupted, you do not need to do any special considerations, -just invoke ``borg create`` as you always do. You may use the same archive name -as in previous attempt or a different one (e.g. if you always include the current -datetime), it does not matter. +If a backup was interrupted, you normally do not need to do anything special, +just invoke ``borg create`` as you always do. If the repository is still locked, +you may need to run ``borg break-lock`` before the next backup. You may use the +same archive name as in previous attempt or a different one (e.g. if you always +include the current datetime), it does not matter. |project_name| always does full single-pass backups, so it will start again from the beginning - but it will be much faster, because some of the data was From ebecd8bd7335a26cdac0e16774f9ff5ecdd047ff Mon Sep 17 00:00:00 2001 From: Milkey Mouse Date: Fri, 17 Nov 2017 19:09:58 -0800 Subject: [PATCH 139/798] Document sshfs rename workaround (fixes #3315) --- docs/faq.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/faq.rst b/docs/faq.rst index 53b1503a1c..599bedb0f4 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -719,6 +719,18 @@ rather not implement progress / ETA display than doing futile attempts. See also: https://xkcd.com/612/ +Why am I getting 'Operation not permitted' errors when backing up on sshfs? +--------------------------------------------------------------------------- + +By default, ``sshfs`` is not entirely POSIX-compliant when renaming files due to +a technicality in the SFTP protocol. Fortunately, it also provides a workaround_ +to make it behave correctly:: + + sshfs -o workaround=rename user@host:dir /mnt/dir + +.. _workaround: https://unix.stackexchange.com/a/123236 + + Miscellaneous ############# From 7fa5561cbc6da9ab35dc35f0a2b3b0d60fcd9daa Mon Sep 17 00:00:00 2001 From: Aidan Woods Date: Mon, 20 Nov 2017 00:58:53 +0000 Subject: [PATCH 140/798] Highlight that information is obtained from security dir (deleting the cache will not bypass this error in the event the user knows this is a legitimate repo). --- src/borg/cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/borg/cache.py b/src/borg/cache.py index 621e2d85ca..e996fa0393 100644 --- a/src/borg/cache.py +++ b/src/borg/cache.py @@ -326,7 +326,7 @@ class RepositoryIDNotUnique(Error): """Cache is newer than repository - do you have multiple, independently updated repos with same ID?""" class RepositoryReplay(Error): - """Cache is newer than repository - this is either an attack or unsafe (multiple repos with same ID)""" + """Cache, or information obtained from the security directory is newer than repository - this is either an attack or unsafe (multiple repos with same ID)""" class CacheInitAbortedError(Error): """Cache initialization aborted""" From 8735e175759af26dcb09d7a8e8b3278489d77756 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 18 Nov 2017 09:08:37 -0500 Subject: [PATCH 141/798] add FAQ about removing old files and policies This has been asked twice already this year: https://mail.python.org/pipermail/borgbackup/2017q3/000796.html https://mail.python.org/pipermail/borgbackup/2017q4/000891.html ... and I was asked again privately today, so this qualifies as FAQ to me. --- docs/faq.rst | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/faq.rst b/docs/faq.rst index 53b1503a1c..72a34eee74 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -296,6 +296,34 @@ the archive, compacting the segments, and committing the repo & cache. This mean when Borg is run with e.g. the ``time`` command, the duration shown in the archive stats may be shorter than the full time the command runs for. +How do I configure different prune policies for different directories? +---------------------------------------------------------------------- + +Say you want to prune ``/var/log`` faster than the rest of +``/``. How do we implement that? The answer is to backup to different +archive *names* and then implement different prune policies for +different prefixes. For example, you could have a script that does:: + + borg create $REPOSITORY:main-$(date +%Y-%m-%d) --exclude /var/log / + borg create $REPOSITORY:logs-$(date +%Y-%m-%d) /var/log + +Then you would have two different prune calls with different policies:: + + borg prune --verbose --list -d 30 --prefix main- "$REPOSITORY" + borg prune --verbose --list -d 7 --prefix logs- "$REPOSITORY" + +This will keep 7 days of logs and 30 days of everything else. Borg 1.1 +also supports the ``--glob-archives`` parameter. + +How do I remove files from an existing backup? +---------------------------------------------- + +Say you now want to remove old logfiles because you changed your +backup policy as described above. The only way to do this is to use +the :ref:`borg_recreate` command to rewrite all archives with a +different ``--exclude`` pattern. See the examples in the +:ref:`borg_recreate` manpage for more information. + Security ######## From 4dab6d29d8bd268e967ce4cb8068417ca6fbf5a0 Mon Sep 17 00:00:00 2001 From: Milkey Mouse Date: Tue, 21 Nov 2017 11:58:56 -0800 Subject: [PATCH 142/798] Clarify key aliases for borg list --format (fixes #3111) --- src/borg/helpers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 82e77a0eec..5a14d87b94 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -1647,20 +1647,20 @@ def keys_help(): class ArchiveFormatter(BaseFormatter): KEY_DESCRIPTIONS = { - 'name': 'archive name interpreted as text (might be missing non-text characters, see barchive)', 'archive': 'archive name interpreted as text (might be missing non-text characters, see barchive)', + 'name': 'alias of "archive"', 'barchive': 'verbatim archive name, can contain any character except NUL', 'comment': 'archive comment interpreted as text (might be missing non-text characters, see bcomment)', 'bcomment': 'verbatim archive comment, can contain any character except NUL', - 'time': 'time (start) of creation of the archive', # *start* is the key used by borg-info for this timestamp, this makes the formats more compatible 'start': 'time (start) of creation of the archive', + 'time': 'alias of "start"', 'end': 'time (end) of creation of the archive', 'id': 'internal ID of the archive', } KEY_GROUPS = ( - ('name', 'archive', 'barchive', 'comment', 'bcomment', 'id'), - ('time', 'start', 'end'), + ('archive', 'name', 'barchive', 'comment', 'bcomment', 'id'), + ('start', 'time', 'end'), ) @classmethod From 8fb8692d424cdd0a1ba081f4b36445d49d4a83fc Mon Sep 17 00:00:00 2001 From: Tom Denley Date: Sat, 11 Nov 2017 11:21:45 +0000 Subject: [PATCH 143/798] Correct usage of "fewer" in place of "less" --- docs/changes.rst | 4 ++-- docs/faq.rst | 4 ++-- docs/internals/data-structures.rst | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index 3d3c2b6865..4c85d43f74 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -1284,7 +1284,7 @@ Other changes: - pass meta-data around, #765 - move some constants to new constants module - - better readability and less errors with namedtuples, #823 + - better readability and fewer errors with namedtuples, #823 - moved source tree into src/ subdirectory, #1016 - made borg.platform a package, #1113 - removed dead crypto code, #1032 @@ -2650,7 +2650,7 @@ Version 0.23.0 (2015-06-11) Incompatible changes (compared to attic, fork related): - changed sw name and cli command to "borg", updated docs -- package name (and name in urls) uses "borgbackup" to have less collisions +- package name (and name in urls) uses "borgbackup" to have fewer collisions - changed repo / cache internal magic strings from ATTIC* to BORG*, changed cache location to .cache/borg/ - this means that it currently won't accept attic repos (see issue #21 about improving that) diff --git a/docs/faq.rst b/docs/faq.rst index beebdb54bb..62e5f8c673 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -108,7 +108,7 @@ Are there other known limitations? usually corresponding to tens or hundreds of millions of files/dirs. When trying to go beyond that limit, you will get a fatal IntegrityError exception telling that the (archive) object is too big. - An easy workaround is to create multiple archives with less items each. + An easy workaround is to create multiple archives with fewer items each. See also the :ref:`archive_limitation` and :issue:`1452`. :ref:`borg_info` shows how large (relative to the maximum size) existing @@ -216,7 +216,7 @@ I get an IntegrityError or similar - what now? A single error does not necessarily indicate bad hardware or a Borg bug. All hardware exhibits a bit error rate (BER). Hard drives are typically -specified as exhibiting less than one error every 12 to 120 TB +specified as exhibiting fewer than one error every 12 to 120 TB (one bit error in 10e14 to 10e15 bits). The specification is often called *unrecoverable read error rate* (URE rate). diff --git a/docs/internals/data-structures.rst b/docs/internals/data-structures.rst index faa1736fb5..60a0ade450 100644 --- a/docs/internals/data-structures.rst +++ b/docs/internals/data-structures.rst @@ -535,7 +535,7 @@ ACLs/xattrs), the limit will be ~32 million files/directories per archive. If one tries to create an archive object bigger than MAX_OBJECT_SIZE, a fatal IntegrityError will be raised. -A workaround is to create multiple archives with less items each, see +A workaround is to create multiple archives with fewer items each, see also :issue:`1452`. .. _item: @@ -706,7 +706,7 @@ be estimated like that:: All units are Bytes. It is assuming every chunk is referenced exactly once (if you have a lot of -duplicate chunks, you will have less chunks than estimated above). +duplicate chunks, you will have fewer chunks than estimated above). It is also assuming that typical chunk size is 2^HASH_MASK_BITS (if you have a lot of files smaller than this statistical medium chunk size, you will have From 454317dc0271ef23baca370d316e0b9a9cc715c9 Mon Sep 17 00:00:00 2001 From: Milkey Mouse Date: Sun, 5 Nov 2017 20:42:44 -0800 Subject: [PATCH 144/798] Add instructions for ntfsclone (fixes #81) --- docs/faq.rst | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/docs/faq.rst b/docs/faq.rst index 62e5f8c673..527422a4d9 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -48,6 +48,64 @@ and platform/software dependency. Combining Borg with the mechanisms provided by the platform (snapshots, hypervisor features) will be the best approach to start tackling them. +How can I decrease the size of disk image backups? +-------------------------------------------------- + +Full disk images are as large as the full disk when uncompressed and might not get much +smaller post-deduplication after heavy use. This is because virtually all file systems +don't actually delete the data on disk (that is the place of so-called "secure delete") +but instead delete the filesystem entries referring to the data. This leaves the random +data on disk until the FS eventually claims it for another file. Therefore, if a hard +drive nears capacity and files are deleted again, the change will barely decrease the +space it takes up when compressed and deduplicated. Depending on the filesystem of the +VM (or physical computer, if for some reason a normal filesystem backup can't be taken), +there are several ways to decrease the size of a full image: + +Using ntfsclone (NTFS, i.e. Windows VMs) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +ntfsclone can only operate on filesystems with the journal cleared (i.e. turned-off +machines) which somewhat limits its utility in the case of VM snapshots. However, +when it can be used, its special image format is even more efficient than just zeroing +and deduplicating. For backup, save the disk header and the contents of each partition:: + + HEADER_SIZE=$(sfdisk -lo Start $DISK | grep -A1 -P 'Start$' | tail -n1 | xargs echo) + PARTITIONS=$(sfdisk -lo Device,Type $DISK | sed -e '1,/Device\s*Type/d') + dd if=$DISK count=$HEADER_SIZE | borg create repo::hostname-partinfo - + echo "$PARTITIONS" | grep NTFS | cut -d' ' -f1 | while read x; do + PARTNUM=$(echo $x | grep -Eo "[0-9]+$") + ntfsclone -so - $x | borg create repo::hostname-part$PARTNUM - + done + # to backup non-NTFS partitions as well: + echo "$PARTITIONS" | grep -v NTFS | cut -d' ' -f1 | while read x; do + PARTNUM=$(echo $x | grep -Eo "[0-9]+$") + borg create --read-special repo::hostname-part$PARTNUM $x + done + +Restoration is similar to the above process, but done in reverse:: + + borg extract --stdout repo::hostname-partinfo | dd of=$DISK && partprobe + PARTITIONS=$(sfdisk -lo Device,Type $DISK | sed -e '1,/Device\s*Type/d') + borg list --format {archive}{NL} repo | grep 'part[0-9]*$' | while read x; do + PARTNUM=$(echo $x | grep -Eo "[0-9]+$") + PARTITION=$(echo "$PARTITIONS" | grep -E "$DISKp?$PARTNUM" | head -n1) + if echo "$PARTITION" | cut -d' ' -f2- | grep -q NTFS; then + borg extract --stdout repo::$x | ntfsclone -rO $(echo "$PARTITION" | cut -d' ' -f1) - + else + borg extract --stdout repo::$x | dd of=$(echo "$PARTITION" | cut -d' ' -f1) + fi + done + +.. note:: + + When backing up a disk image (as opposed to a real block device), mount it as + a loopback image to use the above snippets:: + + DISK=$(losetup -Pf --show /path/to/disk/image) + # do backup as shown above + sync $DISK + losetup -d $DISK + Can I backup from multiple servers into a single repository? ------------------------------------------------------------ From f9ed3b3ed7998bbb5166ca5dfb2783edbeb84efa Mon Sep 17 00:00:00 2001 From: Milkey Mouse Date: Thu, 9 Nov 2017 20:36:39 -0800 Subject: [PATCH 145/798] Add instructions for zerofree --- docs/faq.rst | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/docs/faq.rst b/docs/faq.rst index 527422a4d9..bf539ff0ef 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -103,9 +103,30 @@ Restoration is similar to the above process, but done in reverse:: DISK=$(losetup -Pf --show /path/to/disk/image) # do backup as shown above - sync $DISK losetup -d $DISK +Using zerofree (ext2, ext3, ext4) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +zerofree works similarly to ntfsclone in that it zeros out unused chunks of the FS, except +that it works in place, zeroing the original partition. This makes the backup process a bit +simpler:: + + sfdisk -lo Device,Type $DISK | sed -e '1,/Device\s*Type/d' | grep Linux | cut -d' ' -f1 | xargs -n1 zerofree + borg create --read-special repo::hostname-disk $DISK + +Because the partitions were zeroed in place, restoration is only one command:: + + borg extract --stdout repo::hostname-disk | dd of=$DISK + +.. note:: The "traditional" way to zero out space on a partition, especially one already + mounted, is to simply ``dd`` from ``/dev/zero`` to a temporary file and delete + it. This is ill-advised for the reasons mentioned in the ``zerofree`` man page: + + - it is slow + - it makes the disk image (temporarily) grow to its maximal extent + - it (temporarily) uses all free space on the disk, so other concurrent write actions may fail. + Can I backup from multiple servers into a single repository? ------------------------------------------------------------ From 46698bde6ed1af40ffd914511abda132fa9c73bc Mon Sep 17 00:00:00 2001 From: Milkey Mouse Date: Thu, 23 Nov 2017 11:36:05 -0800 Subject: [PATCH 146/798] Move image backup-related FAQ entries to a new page --- docs/deployment.rst | 1 + docs/deployment/image-backup.rst | 119 +++++++++++++++++++++++++++++++ docs/faq.rst | 119 ------------------------------- 3 files changed, 120 insertions(+), 119 deletions(-) create mode 100644 docs/deployment/image-backup.rst diff --git a/docs/deployment.rst b/docs/deployment.rst index 7b1caf9255..c7c3a6ee14 100644 --- a/docs/deployment.rst +++ b/docs/deployment.rst @@ -12,3 +12,4 @@ This chapter details deployment strategies for the following scenarios. deployment/central-backup-server deployment/hosting-repositories deployment/automated-local + deployment/image-backup diff --git a/docs/deployment/image-backup.rst b/docs/deployment/image-backup.rst new file mode 100644 index 0000000000..fd684c3f2d --- /dev/null +++ b/docs/deployment/image-backup.rst @@ -0,0 +1,119 @@ +.. include:: ../global.rst.inc +.. highlight:: none + +Backing up entire disk images +============================= + +Backing up disk images can still be efficient with Borg because its `deduplication`_ +technique makes sure only the modified parts of the file are stored. Borg also has +optional simple sparse file support for extract. + +Decreasing the size of image backups +------------------------------------ + +Disk images are as large as the full disk when uncompressed and might not get much +smaller post-deduplication after heavy use because virtually all file systems don't +actually delete file data on disk but instead delete the filesystem entries referencing +the data. Therefore, if a disk nears capacity and files are deleted again, the change +will barely decrease the space it takes up when compressed and deduplicated. Depending +on the filesystem, there are several ways to decrease the size of a disk image: + +Using ntfsclone (NTFS, i.e. Windows VMs) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``ntfsclone`` can only operate on filesystems with the journal cleared (i.e. turned-off +machines), which somewhat limits its utility in the case of VM snapshots. However, when +it can be used, its special image format is even more efficient than just zeroing and +deduplicating. For backup, save the disk header and the contents of each partition:: + + HEADER_SIZE=$(sfdisk -lo Start $DISK | grep -A1 -P 'Start$' | tail -n1 | xargs echo) + PARTITIONS=$(sfdisk -lo Device,Type $DISK | sed -e '1,/Device\s*Type/d') + dd if=$DISK count=$HEADER_SIZE | borg create repo::hostname-partinfo - + echo "$PARTITIONS" | grep NTFS | cut -d' ' -f1 | while read x; do + PARTNUM=$(echo $x | grep -Eo "[0-9]+$") + ntfsclone -so - $x | borg create repo::hostname-part$PARTNUM - + done + # to backup non-NTFS partitions as well: + echo "$PARTITIONS" | grep -v NTFS | cut -d' ' -f1 | while read x; do + PARTNUM=$(echo $x | grep -Eo "[0-9]+$") + borg create --read-special repo::hostname-part$PARTNUM $x + done + +Restoration is a similar process:: + + borg extract --stdout repo::hostname-partinfo | dd of=$DISK && partprobe + PARTITIONS=$(sfdisk -lo Device,Type $DISK | sed -e '1,/Device\s*Type/d') + borg list --format {archive}{NL} repo | grep 'part[0-9]*$' | while read x; do + PARTNUM=$(echo $x | grep -Eo "[0-9]+$") + PARTITION=$(echo "$PARTITIONS" | grep -E "$DISKp?$PARTNUM" | head -n1) + if echo "$PARTITION" | cut -d' ' -f2- | grep -q NTFS; then + borg extract --stdout repo::$x | ntfsclone -rO $(echo "$PARTITION" | cut -d' ' -f1) - + else + borg extract --stdout repo::$x | dd of=$(echo "$PARTITION" | cut -d' ' -f1) + fi + done + +.. note:: + + When backing up a disk image (as opposed to a real block device), mount it as + a loopback image to use the above snippets:: + + DISK=$(losetup -Pf --show /path/to/disk/image) + # do backup as shown above + losetup -d $DISK + +Using zerofree (ext2, ext3, ext4) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``zerofree`` works similarly to ntfsclone in that it zeros out unused chunks of the FS, +except it works in place, zeroing the original partition. This makes the backup process +a bit simpler:: + + sfdisk -lo Device,Type $DISK | sed -e '1,/Device\s*Type/d' | grep Linux | cut -d' ' -f1 | xargs -n1 zerofree + borg create --read-special repo::hostname-disk $DISK + +Because the partitions were zeroed in place, restoration is only one command:: + + borg extract --stdout repo::hostname-disk | dd of=$DISK + +.. note:: The "traditional" way to zero out space on a partition, especially one already + mounted, is to simply ``dd`` from ``/dev/zero`` to a temporary file and delete + it. This is ill-advised for the reasons mentioned in the ``zerofree`` man page: + + - it is slow + - it makes the disk image (temporarily) grow to its maximal extent + - it (temporarily) uses all free space on the disk, so other concurrent write actions may fail. + +Virtual machines +---------------- + +If you use non-snapshotting backup tools like Borg to back up virtual machines, then +the VMs should be turned off for the duration of the backup. Backing up live VMs can +(and will) result in corrupted or inconsistent backup contents: a VM image is just a +regular file to Borg with the same issues as regular files when it comes to concurrent +reading and writing from the same file. + +For backing up live VMs use filesystem snapshots on the VM host, which establishes +crash-consistency for the VM images. This means that with most file systems (that +are journaling) the FS will always be fine in the backup (but may need a journal +replay to become accessible). + +Usually this does not mean that file *contents* on the VM are consistent, since file +contents are normally not journaled. Notable exceptions are ext4 in data=journal mode, +ZFS and btrfs (unless nodatacow is used). + +Applications designed with crash-consistency in mind (most relational databases like +PostgreSQL, SQLite etc. but also for example Borg repositories) should always be able +to recover to a consistent state from a backup created with crash-consistent snapshots +(even on ext4 with data=writeback or XFS). Other applications may require a lot of work +to reach application-consistency; it's a broad and complex issue that cannot be explained +in entirety here. + +Hypervisor snapshots capturing most of the VM's state can also be used for backups and +can be a better alternative to pure file system based snapshots of the VM's disk, since +no state is lost. Depending on the application this can be the easiest and most reliable +way to create application-consistent backups. + +Borg doesn't intend to address these issues due to their huge complexity and +platform/software dependency. Combining Borg with the mechanisms provided by the platform +(snapshots, hypervisor features) will be the best approach to start tackling them. \ No newline at end of file diff --git a/docs/faq.rst b/docs/faq.rst index bf539ff0ef..495c92bf48 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -8,125 +8,6 @@ Frequently asked questions Usage & Limitations ################### -Can I backup VM disk images? ----------------------------- - -Yes, the `deduplication`_ technique used by -|project_name| makes sure only the modified parts of the file are stored. -Also, we have optional simple sparse file support for extract. - -If you use non-snapshotting backup tools like Borg to back up virtual machines, -then the VMs should be turned off for the duration of the backup. Backing up live VMs can (and will) -result in corrupted or inconsistent backup contents: a VM image is just a regular file to -Borg with the same issues as regular files when it comes to concurrent reading and writing from -the same file. - -For backing up live VMs use file system snapshots on the VM host, which establishes -crash-consistency for the VM images. This means that with most file systems -(that are journaling) the FS will always be fine in the backup (but may need a -journal replay to become accessible). - -Usually this does not mean that file *contents* on the VM are consistent, since file -contents are normally not journaled. Notable exceptions are ext4 in data=journal mode, -ZFS and btrfs (unless nodatacow is used). - -Applications designed with crash-consistency in mind (most relational databases -like PostgreSQL, SQLite etc. but also for example Borg repositories) should always -be able to recover to a consistent state from a backup created with -crash-consistent snapshots (even on ext4 with data=writeback or XFS). - -Hypervisor snapshots capturing most of the VM's state can also be used for backups -and can be a better alternative to pure file system based snapshots of the VM's disk, -since no state is lost. Depending on the application this can be the easiest and most -reliable way to create application-consistent backups. - -Other applications may require a lot of work to reach application-consistency: -It's a broad and complex issue that cannot be explained in entirety here. - -Borg doesn't intend to address these issues due to their huge complexity -and platform/software dependency. Combining Borg with the mechanisms provided -by the platform (snapshots, hypervisor features) will be the best approach -to start tackling them. - -How can I decrease the size of disk image backups? --------------------------------------------------- - -Full disk images are as large as the full disk when uncompressed and might not get much -smaller post-deduplication after heavy use. This is because virtually all file systems -don't actually delete the data on disk (that is the place of so-called "secure delete") -but instead delete the filesystem entries referring to the data. This leaves the random -data on disk until the FS eventually claims it for another file. Therefore, if a hard -drive nears capacity and files are deleted again, the change will barely decrease the -space it takes up when compressed and deduplicated. Depending on the filesystem of the -VM (or physical computer, if for some reason a normal filesystem backup can't be taken), -there are several ways to decrease the size of a full image: - -Using ntfsclone (NTFS, i.e. Windows VMs) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -ntfsclone can only operate on filesystems with the journal cleared (i.e. turned-off -machines) which somewhat limits its utility in the case of VM snapshots. However, -when it can be used, its special image format is even more efficient than just zeroing -and deduplicating. For backup, save the disk header and the contents of each partition:: - - HEADER_SIZE=$(sfdisk -lo Start $DISK | grep -A1 -P 'Start$' | tail -n1 | xargs echo) - PARTITIONS=$(sfdisk -lo Device,Type $DISK | sed -e '1,/Device\s*Type/d') - dd if=$DISK count=$HEADER_SIZE | borg create repo::hostname-partinfo - - echo "$PARTITIONS" | grep NTFS | cut -d' ' -f1 | while read x; do - PARTNUM=$(echo $x | grep -Eo "[0-9]+$") - ntfsclone -so - $x | borg create repo::hostname-part$PARTNUM - - done - # to backup non-NTFS partitions as well: - echo "$PARTITIONS" | grep -v NTFS | cut -d' ' -f1 | while read x; do - PARTNUM=$(echo $x | grep -Eo "[0-9]+$") - borg create --read-special repo::hostname-part$PARTNUM $x - done - -Restoration is similar to the above process, but done in reverse:: - - borg extract --stdout repo::hostname-partinfo | dd of=$DISK && partprobe - PARTITIONS=$(sfdisk -lo Device,Type $DISK | sed -e '1,/Device\s*Type/d') - borg list --format {archive}{NL} repo | grep 'part[0-9]*$' | while read x; do - PARTNUM=$(echo $x | grep -Eo "[0-9]+$") - PARTITION=$(echo "$PARTITIONS" | grep -E "$DISKp?$PARTNUM" | head -n1) - if echo "$PARTITION" | cut -d' ' -f2- | grep -q NTFS; then - borg extract --stdout repo::$x | ntfsclone -rO $(echo "$PARTITION" | cut -d' ' -f1) - - else - borg extract --stdout repo::$x | dd of=$(echo "$PARTITION" | cut -d' ' -f1) - fi - done - -.. note:: - - When backing up a disk image (as opposed to a real block device), mount it as - a loopback image to use the above snippets:: - - DISK=$(losetup -Pf --show /path/to/disk/image) - # do backup as shown above - losetup -d $DISK - -Using zerofree (ext2, ext3, ext4) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -zerofree works similarly to ntfsclone in that it zeros out unused chunks of the FS, except -that it works in place, zeroing the original partition. This makes the backup process a bit -simpler:: - - sfdisk -lo Device,Type $DISK | sed -e '1,/Device\s*Type/d' | grep Linux | cut -d' ' -f1 | xargs -n1 zerofree - borg create --read-special repo::hostname-disk $DISK - -Because the partitions were zeroed in place, restoration is only one command:: - - borg extract --stdout repo::hostname-disk | dd of=$DISK - -.. note:: The "traditional" way to zero out space on a partition, especially one already - mounted, is to simply ``dd`` from ``/dev/zero`` to a temporary file and delete - it. This is ill-advised for the reasons mentioned in the ``zerofree`` man page: - - - it is slow - - it makes the disk image (temporarily) grow to its maximal extent - - it (temporarily) uses all free space on the disk, so other concurrent write actions may fail. - Can I backup from multiple servers into a single repository? ------------------------------------------------------------ From 52410b6976202680ac31abea683424a17b779f99 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 7 Nov 2017 17:45:22 +0100 Subject: [PATCH 147/798] borg mount: support exclusion group options and paths, fixes #2138 borg mount [options] repo_or_archive mountpoint path [paths...] paths: you can just give some "root paths" (like for borg extract) to only partially populate the FUSE filesystem. Similar for these exclusion group options: --exclude --exclude-from --pattern --patterns-from --strip-components (cherry picked from commit 77df1cfe8c01eae932631fa7873b05b99b6b6b55) --- src/borg/archiver.py | 3 +++ src/borg/fuse.py | 18 +++++++++++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index a8cab84bef..fdeaec0f90 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -2517,6 +2517,9 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): subparser.add_argument('-o', dest='options', type=str, help='Extra mount options') define_archive_filters_group(subparser) + subparser.add_argument('paths', metavar='PATH', nargs='*', type=str, + help='paths to extract; patterns are supported') + define_exclusion_group(subparser, strip_components=True) if parser.prog == 'borgfs': return parser diff --git a/src/borg/fuse.py b/src/borg/fuse.py index 129e3b20b7..3bc82eef02 100644 --- a/src/borg/fuse.py +++ b/src/borg/fuse.py @@ -17,6 +17,7 @@ logger = create_logger() from .crypto.low_level import blake2b_128 +from .archiver import Archiver from .archive import Archive from .hashindex import FuseVersionsIndex from .helpers import daemonize, hardlinkable, signal_handler, format_file_size @@ -118,7 +119,7 @@ def get(self, inode): else: raise ValueError('Invalid entry type in self.meta') - def iter_archive_items(self, archive_item_ids): + def iter_archive_items(self, archive_item_ids, filter=None): unpacker = msgpack.Unpacker() # Current offset in the metadata stream, which consists of all metadata chunks glued together @@ -161,6 +162,11 @@ def write_bytes(append_msgpacked_bytes): # Need more data, feed the next chunk break + item = Item(internal_dict=item) + if filter and not filter(item): + msgpacked_bytes = b'' + continue + current_item = msgpacked_bytes current_item_length = len(current_item) current_spans_chunks = stream_offset - current_item_length < chunk_begin @@ -197,7 +203,7 @@ def write_bytes(append_msgpacked_bytes): inode = write_offset + self.offset write_offset += 9 - yield inode, Item(internal_dict=item) + yield inode, item self.write_offset = write_offset @@ -325,7 +331,13 @@ def process_archive(self, archive_name, prefix=[]): t0 = time.perf_counter() archive = Archive(self.repository_uncached, self.key, self.manifest, archive_name, consider_part_files=self.args.consider_part_files) - for item_inode, item in self.cache.iter_archive_items(archive.metadata.items): + strip_components = self.args.strip_components + matcher = Archiver.build_matcher(self.args.patterns, self.args.paths) + dummy = lambda x, y: None # TODO: add hardlink_master support code, see Archiver + filter = Archiver.build_filter(matcher, dummy, strip_components) + for item_inode, item in self.cache.iter_archive_items(archive.metadata.items, filter=filter): + if strip_components: + item.path = os.sep.join(item.path.split(os.sep)[strip_components:]) path = os.fsencode(item.path) is_dir = stat.S_ISDIR(item.mode) if is_dir: From 0e0764757688bd0085491c26b2f2c58ac5fa8028 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 21 Nov 2017 01:20:57 +0100 Subject: [PATCH 148/798] borg mount: support hardlinks correctly, add tests previous commit did not yet support hardlinks correctly, if the hardlink master was excluded somehow. added some tests for this, also refactored related tests slightly. (cherry picked from commit e97deafb16ac44450b542e4e23d57556631f647b) --- src/borg/fuse.py | 75 ++++++++++++++++++++++------------ src/borg/testsuite/archiver.py | 56 ++++++++++++++++++++----- 2 files changed, 94 insertions(+), 37 deletions(-) diff --git a/src/borg/fuse.py b/src/borg/fuse.py index 3bc82eef02..e06651fa6b 100644 --- a/src/borg/fuse.py +++ b/src/borg/fuse.py @@ -333,8 +333,15 @@ def process_archive(self, archive_name, prefix=[]): consider_part_files=self.args.consider_part_files) strip_components = self.args.strip_components matcher = Archiver.build_matcher(self.args.patterns, self.args.paths) - dummy = lambda x, y: None # TODO: add hardlink_master support code, see Archiver - filter = Archiver.build_filter(matcher, dummy, strip_components) + partial_extract = not matcher.empty() or strip_components + hardlink_masters = {} if partial_extract else None + + def peek_and_store_hardlink_masters(item, matched): + if (partial_extract and not matched and hardlinkable(item.mode) and + item.get('hardlink_master', True) and 'source' not in item): + hardlink_masters[item.get('path')] = (item.get('chunks'), None) + + filter = Archiver.build_filter(matcher, peek_and_store_hardlink_masters, strip_components) for item_inode, item in self.cache.iter_archive_items(archive.metadata.items, filter=filter): if strip_components: item.path = os.sep.join(item.path.split(os.sep)[strip_components:]) @@ -355,11 +362,16 @@ def process_archive(self, archive_name, prefix=[]): parent = 1 for segment in segments[:-1]: parent = self.process_inner(segment, parent) - self.process_leaf(segments[-1], item, parent, prefix, is_dir, item_inode) + self.process_leaf(segments[-1], item, parent, prefix, is_dir, item_inode, + hardlink_masters, strip_components) duration = time.perf_counter() - t0 logger.debug('fuse: process_archive completed in %.1f s for archive %s', duration, archive.name) - def process_leaf(self, name, item, parent, prefix, is_dir, item_inode): + def process_leaf(self, name, item, parent, prefix, is_dir, item_inode, hardlink_masters, stripped_components): + path = item.path + del item.path # save some space + hardlink_masters = hardlink_masters or {} + def file_version(item, path): if 'chunks' in item: file_id = blake2b_128(path) @@ -384,35 +396,44 @@ def make_versioned_name(name, version, add_dir=False): version_enc = os.fsencode('.%05d' % version) return name + version_enc + ext + if 'source' in item and hardlinkable(item.mode): + source = os.path.join(*item.source.split(os.sep)[stripped_components:]) + chunks, link_target = hardlink_masters.get(item.source, (None, source)) + if link_target: + # Hard link was extracted previously, just link + link_target = os.fsencode(link_target) + if self.versions: + # adjust link target name with version + version = self.file_versions[link_target] + link_target = make_versioned_name(link_target, version, add_dir=True) + try: + inode = self._find_inode(link_target, prefix) + except KeyError: + logger.warning('Skipping broken hard link: %s -> %s', path, source) + return + item = self.get_item(inode) + item.nlink = item.get('nlink', 1) + 1 + self.items[inode] = item + elif chunks is not None: + # assign chunks to this item, since the item which had the chunks was not extracted + item.chunks = chunks + inode = item_inode + self.items[inode] = item + if hardlink_masters: + # Update master entry with extracted item path, so that following hardlinks don't extract twice. + hardlink_masters[item.source] = (None, path) + else: + inode = item_inode + if self.versions and not is_dir: parent = self.process_inner(name, parent) - path = os.fsencode(item.path) - version = file_version(item, path) + enc_path = os.fsencode(path) + version = file_version(item, enc_path) if version is not None: # regular file, with contents - maybe a hardlink master name = make_versioned_name(name, version) - self.file_versions[path] = version + self.file_versions[enc_path] = version - path = item.path - del item.path # save some space - if 'source' in item and hardlinkable(item.mode): - # a hardlink, no contents, is the hardlink master - source = os.fsencode(item.source) - if self.versions: - # adjust source name with version - version = self.file_versions[source] - source = make_versioned_name(source, version, add_dir=True) - name = make_versioned_name(name, version) - try: - inode = self._find_inode(source, prefix) - except KeyError: - logger.warning('Skipping broken hard link: %s -> %s', path, item.source) - return - item = self.cache.get(inode) - item.nlink = item.get('nlink', 1) + 1 - self.items[inode] = item - else: - inode = item_inode self.parent[inode] = parent if name: self.contents[parent][name] = inode diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 1c2e837081..d4f716f6c4 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -759,7 +759,7 @@ def _extract_hardlinks_setup(self): os.mkdir(os.path.join(self.input_path, 'dir1')) os.mkdir(os.path.join(self.input_path, 'dir1/subdir')) - self.create_regular_file('source') + self.create_regular_file('source', contents=b'123456') os.link(os.path.join(self.input_path, 'source'), os.path.join(self.input_path, 'abba')) os.link(os.path.join(self.input_path, 'source'), @@ -777,30 +777,56 @@ def _extract_hardlinks_setup(self): requires_hardlinks = pytest.mark.skipif(not are_hardlinks_supported(), reason='hardlinks not supported') @requires_hardlinks - def test_strip_components_links(self): + @unittest.skipUnless(has_llfuse, 'llfuse not installed') + def test_mount_hardlinks(self): self._extract_hardlinks_setup() - with changedir('output'): - self.cmd('extract', self.repository_location + '::test', '--strip-components', '2') + mountpoint = os.path.join(self.tmpdir, 'mountpoint') + with self.fuse_mount(self.repository_location + '::test', mountpoint, '--strip-components=2'), \ + changedir(mountpoint): assert os.stat('hardlink').st_nlink == 2 assert os.stat('subdir/hardlink').st_nlink == 2 + assert open('subdir/hardlink', 'rb').read() == b'123456' assert os.stat('aaaa').st_nlink == 2 assert os.stat('source2').st_nlink == 2 - with changedir('output'): - self.cmd('extract', self.repository_location + '::test') + with self.fuse_mount(self.repository_location + '::test', mountpoint, 'input/dir1'), \ + changedir(mountpoint): + assert os.stat('input/dir1/hardlink').st_nlink == 2 + assert os.stat('input/dir1/subdir/hardlink').st_nlink == 2 + assert open('input/dir1/subdir/hardlink', 'rb').read() == b'123456' + assert os.stat('input/dir1/aaaa').st_nlink == 2 + assert os.stat('input/dir1/source2').st_nlink == 2 + with self.fuse_mount(self.repository_location + '::test', mountpoint), \ + changedir(mountpoint): + assert os.stat('input/source').st_nlink == 4 + assert os.stat('input/abba').st_nlink == 4 assert os.stat('input/dir1/hardlink').st_nlink == 4 + assert os.stat('input/dir1/subdir/hardlink').st_nlink == 4 + assert open('input/dir1/subdir/hardlink', 'rb').read() == b'123456' @requires_hardlinks def test_extract_hardlinks(self): self._extract_hardlinks_setup() + with changedir('output'): + self.cmd('extract', self.repository_location + '::test', '--strip-components', '2') + assert os.stat('hardlink').st_nlink == 2 + assert os.stat('subdir/hardlink').st_nlink == 2 + assert open('subdir/hardlink', 'rb').read() == b'123456' + assert os.stat('aaaa').st_nlink == 2 + assert os.stat('source2').st_nlink == 2 with changedir('output'): self.cmd('extract', self.repository_location + '::test', 'input/dir1') assert os.stat('input/dir1/hardlink').st_nlink == 2 assert os.stat('input/dir1/subdir/hardlink').st_nlink == 2 + assert open('input/dir1/subdir/hardlink', 'rb').read() == b'123456' assert os.stat('input/dir1/aaaa').st_nlink == 2 assert os.stat('input/dir1/source2').st_nlink == 2 with changedir('output'): self.cmd('extract', self.repository_location + '::test') + assert os.stat('input/source').st_nlink == 4 + assert os.stat('input/abba').st_nlink == 4 assert os.stat('input/dir1/hardlink').st_nlink == 4 + assert os.stat('input/dir1/subdir/hardlink').st_nlink == 4 + assert open('input/dir1/subdir/hardlink', 'rb').read() == b'123456' def test_extract_include_exclude(self): self.cmd('init', '--encryption=repokey', self.repository_location) @@ -2182,8 +2208,9 @@ def test_fuse_versions_view(self): self.cmd('init', '--encryption=repokey', self.repository_location) self.create_regular_file('test', contents=b'first') if are_hardlinks_supported(): - self.create_regular_file('hardlink1', contents=b'') + self.create_regular_file('hardlink1', contents=b'123456') os.link('input/hardlink1', 'input/hardlink2') + os.link('input/hardlink1', 'input/hardlink3') self.cmd('create', self.repository_location + '::archive1', 'input') self.create_regular_file('test', contents=b'second') self.cmd('create', self.repository_location + '::archive2', 'input') @@ -2195,9 +2222,18 @@ def test_fuse_versions_view(self): assert all(f.startswith('test.') for f in files) # ... with files test.xxxxx in there assert {b'first', b'second'} == {open(os.path.join(path, f), 'rb').read() for f in files} if are_hardlinks_supported(): - st1 = os.stat(os.path.join(mountpoint, 'input', 'hardlink1', 'hardlink1.00001')) - st2 = os.stat(os.path.join(mountpoint, 'input', 'hardlink2', 'hardlink2.00001')) - assert st1.st_ino == st2.st_ino + hl1 = os.path.join(mountpoint, 'input', 'hardlink1', 'hardlink1.00001') + hl2 = os.path.join(mountpoint, 'input', 'hardlink2', 'hardlink2.00001') + hl3 = os.path.join(mountpoint, 'input', 'hardlink3', 'hardlink3.00001') + assert os.stat(hl1).st_ino == os.stat(hl2).st_ino == os.stat(hl3).st_ino + assert open(hl3, 'rb').read() == b'123456' + # similar again, but exclude the hardlink master: + with self.fuse_mount(self.repository_location, mountpoint, '-o', 'versions', '-e', 'input/hardlink1'): + if are_hardlinks_supported(): + hl2 = os.path.join(mountpoint, 'input', 'hardlink2', 'hardlink2.00001') + hl3 = os.path.join(mountpoint, 'input', 'hardlink3', 'hardlink3.00001') + assert os.stat(hl2).st_ino == os.stat(hl3).st_ino + assert open(hl3, 'rb').read() == b'123456' @unittest.skipUnless(has_llfuse, 'llfuse not installed') def test_fuse_allow_damaged_files(self): From 47d16e0f6251802d643b57e8b839329e999b5b99 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 22 Nov 2017 15:08:08 +0100 Subject: [PATCH 149/798] borg mount: support --consider-part-files correctly, fixes #3347 (cherry picked from commit caece370b8931751195f439f5a5c08baa97d65f9) --- src/borg/fuse.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/borg/fuse.py b/src/borg/fuse.py index e06651fa6b..399f7d0843 100644 --- a/src/borg/fuse.py +++ b/src/borg/fuse.py @@ -119,7 +119,7 @@ def get(self, inode): else: raise ValueError('Invalid entry type in self.meta') - def iter_archive_items(self, archive_item_ids, filter=None): + def iter_archive_items(self, archive_item_ids, filter=None, consider_part_files=False): unpacker = msgpack.Unpacker() # Current offset in the metadata stream, which consists of all metadata chunks glued together @@ -163,7 +163,7 @@ def write_bytes(append_msgpacked_bytes): break item = Item(internal_dict=item) - if filter and not filter(item): + if filter and not filter(item) or not consider_part_files and 'part' in item: msgpacked_bytes = b'' continue @@ -342,7 +342,8 @@ def peek_and_store_hardlink_masters(item, matched): hardlink_masters[item.get('path')] = (item.get('chunks'), None) filter = Archiver.build_filter(matcher, peek_and_store_hardlink_masters, strip_components) - for item_inode, item in self.cache.iter_archive_items(archive.metadata.items, filter=filter): + for item_inode, item in self.cache.iter_archive_items(archive.metadata.items, filter=filter, + consider_part_files=self.args.consider_part_files): if strip_components: item.path = os.sep.join(item.path.split(os.sep)[strip_components:]) path = os.fsencode(item.path) From 2ac0bf49807e2acf1d80a1dbd8c7a1589cfcee9f Mon Sep 17 00:00:00 2001 From: Milkey Mouse Date: Fri, 10 Nov 2017 20:30:02 -0800 Subject: [PATCH 150/798] Add borg config command (fixes #3304) This command works similarly to "git config" - it parses repo and cache configs to get, set, and delete values. It only works on local repos so a malicious client can't e.g. override their storage quota or reset the append_only flag. Add tests for borg config Add documentation for borg config Change manual config edits -> borg config There were a couple places in the documentation where it was advised to edit the repository or cache config file, a process that is stream- lined by borg config. --- docs/faq.rst | 5 +-- docs/quickstart.rst | 9 +++-- docs/usage.rst | 1 + docs/usage/config.rst | 22 +++++++++++ docs/usage/notes.rst | 5 ++- src/borg/archiver.py | 67 ++++++++++++++++++++++++++++++++++ src/borg/helpers.py | 9 ++++- src/borg/testsuite/archiver.py | 17 +++++++++ 8 files changed, 124 insertions(+), 11 deletions(-) create mode 100644 docs/usage/config.rst diff --git a/docs/faq.rst b/docs/faq.rst index 495c92bf48..b01db24150 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -499,8 +499,7 @@ space for chunks.archive.d (see :issue:`235` for details): :: # this assumes you are working with the same user as the backup. - # you can get the REPOID from the "config" file inside the repository. - cd ~/.cache/borg/ + cd ~/.cache/borg/$(borg config /path/to/repo id) rm -rf chunks.archive.d ; touch chunks.archive.d This deletes all the cached archive chunk indexes and replaces the directory @@ -817,7 +816,7 @@ There are some caveats: - If the repository is in "keyfile" encryption mode, the keyfile must exist locally or it must be manually moved after performing the upgrade: - 1. Locate the repository ID, contained in the ``config`` file in the repository. + 1. Get the repository ID with ``borg config /path/to/repo id``. 2. Locate the attic key file at ``~/.attic/keys/``. The correct key for the repository starts with the line ``ATTIC_KEY ``. 3. Copy the attic key file to ``~/.config/borg/keys/`` diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 597929eac2..7d3bef090f 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -22,10 +22,11 @@ a good amount of free space on the filesystem that has your backup repository repositories. See also :ref:`cache-memory-usage`. Borg doesn't use space reserved for root on repository disks (even when run as root), -on file systems which do not support this mechanism (e.g. XFS) we recommend to -reserve some space in Borg itself just to be safe by adjusting the -``additional_free_space`` setting in the ``[repository]`` section of a repositories -``config`` file. A good starting point is ``2G``. +on file systems which do not support this mechanism (e.g. XFS) we recommend to reserve +some space in Borg itself just to be safe by adjusting the ``additional_free_space`` +setting (a good starting point is ``2G``):: + + borg config /path/to/repo additional_free_space 2G If |project_name| runs out of disk space, it tries to free as much space as it can while aborting the current operation safely, which allows to free more space diff --git a/docs/usage.rst b/docs/usage.rst index de335c1c0f..f8e2b48bfe 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -51,6 +51,7 @@ Usage usage/serve usage/lock usage/benchmark + usage/config usage/help usage/debug diff --git a/docs/usage/config.rst b/docs/usage/config.rst new file mode 100644 index 0000000000..dc21eb39da --- /dev/null +++ b/docs/usage/config.rst @@ -0,0 +1,22 @@ +.. include:: config.rst.inc + +.. note:: + + The repository & cache config files are some of the only directly manipulable + parts of a repository that aren't versioned or backed up, so be careful when + making changes\! + +Examples +~~~~~~~~ +:: + + # find cache directory + $ cd ~/.cache/borg/$(borg config /path/to/repo id) + + # reserve some space + $ borg config /path/to/repo additional_free_space 2G + + # make a repo append-only + $ borg config /path/to/repo append_only 1 + + diff --git a/docs/usage/notes.rst b/docs/usage/notes.rst index c45ef3f8ca..4342a8c12c 100644 --- a/docs/usage/notes.rst +++ b/docs/usage/notes.rst @@ -149,8 +149,9 @@ reject to delete the repository completely). This is useful for scenarios where backup client machine backups remotely to a backup server using ``borg serve``, since a hacked client machine cannot delete backups on the server permanently. -To activate append-only mode, edit the repository ``config`` file and add a line -``append_only=1`` to the ``[repository]`` section (or edit the line if it exists). +To activate append-only mode, set ``append_only`` to 1 in the repository config:: + + borg config /path/to/repo append_only 1 In append-only mode Borg will create a transaction log in the ``transactions`` file, where each line is a transaction and a UTC timestamp. diff --git a/src/borg/archiver.py b/src/borg/archiver.py index fdeaec0f90..7feb410835 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -1,5 +1,6 @@ import argparse import collections +import configparser import faulthandler import functools import hashlib @@ -1709,6 +1710,41 @@ def do_with_lock(self, args, repository): # see issue #1867. repository.commit() + @with_repository(exclusive=True, cache=True, compatibility=(Manifest.Operation.WRITE,)) + def do_config(self, args, repository, manifest, key, cache): + """get, set, and delete values in a repository or cache config file""" + try: + section, name = args.name.split('.') + except ValueError: + section = args.cache and "cache" or "repository" + name = args.name + + if args.cache: + cache.cache_config.load() + config = cache.cache_config._config + save = cache.cache_config.save + else: + config = repository.config + save = lambda: repository.save_config(repository.path, repository.config) + + if args.delete: + config.remove_option(section, name) + if len(config.options(section)) == 0: + config.remove_section(section) + save() + elif args.value: + if section not in config.sections(): + config.add_section(section) + config.set(section, name, args.value) + save() + else: + try: + print(config.get(section, name)) + except (configparser.NoOptionError, configparser.NoSectionError) as e: + print(e, file=sys.stderr) + return EXIT_WARNING + return EXIT_SUCCESS + def do_debug_info(self, args): """display system information for debugging / bug reports""" print(sysinfo()) @@ -3681,6 +3717,37 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): subparser.add_argument('args', metavar='ARGS', nargs=argparse.REMAINDER, help='command arguments') + config_epilog = process_epilog(""" + This command gets and sets options in a local repository or cache config file. + For security reasons, this command only works on local repositories. + + To delete a config value entirely, use ``--delete``. To get an existing key, pass + only the key name. To set a key, pass both the key name and the new value. Keys + can be specified in the format "section.name" or simply "name"; the section will + default to "repository" and "cache" for the repo and cache configs, respectively. + + By default, borg config manipulates the repository config file. Using ``--cache`` + edits the repository cache's config file instead. + """) + subparser = subparsers.add_parser('config', parents=[common_parser], add_help=False, + description=self.do_config.__doc__, + epilog=config_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help='get and set repository config options') + subparser.set_defaults(func=self.do_config) + subparser.add_argument('-c', '--cache', dest='cache', action='store_true', + help='get and set values from the repo cache') + subparser.add_argument('-d', '--delete', dest='delete', action='store_true', + help='delete the key from the config file') + + subparser.add_argument('location', metavar='REPOSITORY', + type=location_validator(archive=False, proto='file'), + help='repository to configure') + subparser.add_argument('name', metavar='NAME', + help='name of config key') + subparser.add_argument('value', metavar='VALUE', nargs='?', + help='new value for key') + subparser = subparsers.add_parser('help', parents=[common_parser], add_help=False, description='Extra help') subparser.add_argument('--epilog-only', dest='epilog_only', action='store_true') diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 5a14d87b94..e5f9de5ad9 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -1139,7 +1139,7 @@ def canonical_path(self): path) -def location_validator(archive=None): +def location_validator(archive=None, proto=None): def validator(text): try: loc = Location(text) @@ -1148,7 +1148,12 @@ def validator(text): if archive is True and not loc.archive: raise argparse.ArgumentTypeError('"%s": No archive specified' % text) elif archive is False and loc.archive: - raise argparse.ArgumentTypeError('"%s" No archive can be specified' % text) + raise argparse.ArgumentTypeError('"%s": No archive can be specified' % text) + if proto is not None and loc.proto != proto: + if proto == 'file': + raise argparse.ArgumentTypeError('"%s": Repository must be local' % text) + else: + raise argparse.ArgumentTypeError('"%s": Repository must be remote' % text) return loc return validator diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index d4f716f6c4..56e1e35536 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -2778,6 +2778,19 @@ def test_benchmark_crud(self): with environment_variable(_BORG_BENCHMARK_CRUD_TEST='YES'): self.cmd('benchmark', 'crud', self.repository_location, self.input_path) + def test_config(self): + self.create_test_files() + os.unlink('input/flagfile') + self.cmd('init', '--encryption=repokey', self.repository_location) + for flags in [[], ['--cache']]: + for key in {'testkey', 'testsection.testkey'}: + self.cmd('config', self.repository_location, *flags, key, exit_code=1) + self.cmd('config', self.repository_location, *flags, key, 'testcontents') + output = self.cmd('config', self.repository_location, *flags, key) + assert output == 'testcontents\n' + self.cmd('config', self.repository_location, *flags, '--delete', key) + self.cmd('config', self.repository_location, *flags, key, exit_code=1) + requires_gnutar = pytest.mark.skipif(not have_gnutar(), reason='GNU tar must be installed for this test.') requires_gzip = pytest.mark.skipif(not shutil.which('gzip'), reason='gzip must be installed for this test.') @@ -3260,6 +3273,10 @@ def test_remote_repo_restrict_to_repository(self): def test_debug_put_get_delete_obj(self): pass + @unittest.skip('only works locally') + def test_config(self): + pass + def test_strip_components_doesnt_leak(self): self.cmd('init', '--encryption=repokey', self.repository_location) self.create_regular_file('dir/file', contents=b"test file contents 1") From 15d9c94981c1b90dd690f8cc336e6f2bd3b0f011 Mon Sep 17 00:00:00 2001 From: Milkey Mouse Date: Tue, 21 Nov 2017 11:41:36 -0800 Subject: [PATCH 151/798] Fix borg config flake8 failures Suppressed E731 so lambdas can be assigned to vars --- setup.cfg | 2 +- src/borg/testsuite/archiver.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/setup.cfg b/setup.cfg index 8e9414ac6d..40ca52dba3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,7 +5,7 @@ python_files = testsuite/*.py # please note that the values are adjusted so that they do not cause failures # with existing code. if you want to change them, you should first fix all # flake8 failures that appear with your change. -ignore = E122,E123,E125,E126,E127,E128,E226,E402,E722,E741,F401,F405,F811 +ignore = E122,E123,E125,E126,E127,E128,E226,E402,E722,E731,E741,F401,F405,F811 # line length long term target: 120 max-line-length = 255 exclude = build,dist,.git,.idea,.cache,.tox,docs/conf.py diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 56e1e35536..a773bd9240 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -2783,13 +2783,13 @@ def test_config(self): os.unlink('input/flagfile') self.cmd('init', '--encryption=repokey', self.repository_location) for flags in [[], ['--cache']]: - for key in {'testkey', 'testsection.testkey'}: - self.cmd('config', self.repository_location, *flags, key, exit_code=1) - self.cmd('config', self.repository_location, *flags, key, 'testcontents') - output = self.cmd('config', self.repository_location, *flags, key) + for cfg_key in {'testkey', 'testsection.testkey'}: + self.cmd('config', self.repository_location, *flags, cfg_key, exit_code=1) + self.cmd('config', self.repository_location, *flags, cfg_key, 'testcontents') + output = self.cmd('config', self.repository_location, *flags, cfg_key) assert output == 'testcontents\n' - self.cmd('config', self.repository_location, *flags, '--delete', key) - self.cmd('config', self.repository_location, *flags, key, exit_code=1) + self.cmd('config', self.repository_location, *flags, '--delete', cfg_key) + self.cmd('config', self.repository_location, *flags, cfg_key, exit_code=1) requires_gnutar = pytest.mark.skipif(not have_gnutar(), reason='GNU tar must be installed for this test.') requires_gzip = pytest.mark.skipif(not shutil.which('gzip'), reason='gzip must be installed for this test.') From 13d339bb8519c240237a219fa5107c2ddbc9aa0d Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 24 Nov 2017 05:00:23 +0100 Subject: [PATCH 152/798] clean up and simplify packaging 1.1.2 was released with a sdist that included quite some files that did not belong into the package (I tried nuitka at some time). This was because the old MANIFEST.in / setup.py included all the stuff in the package dir. Now, setuptools_scm is to automatically deal with the INCLUDES and MANIFEST.in only handles the EXCLUDES, so only committed files get into the sdist (minus some we do not want). Also, no .c .h .pyx files will be installed - they are not needed as they have been compiled into binary files. (cherry picked from commit feb428b4e012686d42cf0240663660757dcf231a) --- MANIFEST.in | 10 ++++------ setup.py | 8 +++++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 2ec72dc240..f3233beaf8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,8 +1,6 @@ -include README.rst AUTHORS LICENSE CHANGES.rst MANIFEST.in -exclude .coafile .coveragerc .gitattributes .gitignore .travis.yml Vagrantfile +# stuff we need to include into the sdist is handled automatically by +# setuptools_scm - it includes all git-committed files. +# but we want to exclude some committed files/dirs not needed in the sdist: +exclude .coafile .coveragerc .editorconfig .gitattributes .gitignore .mailmap .travis.yml Vagrantfile prune .travis prune .github -graft src -graft docs -prune docs/_build -global-exclude *.py[co] *.orig *.so *.dll diff --git a/setup.py b/setup.py index eaae5c417e..2670bbfc88 100644 --- a/setup.py +++ b/setup.py @@ -807,10 +807,12 @@ def run(self): 'borgfs = borg.archiver:main', ] }, + # See also the MANIFEST.in file. + # We want to install all the files in the package directories... include_package_data=True, - package_data={ - 'borg': ['paperkey.html'], - 'borg.testsuite': ['attic.tar.gz'], + # ...except the source files which have been compiled (C extensions): + exclude_package_data={ + '': ['*.c', '*.h', '*.pyx', ], }, cmdclass=cmdclass, ext_modules=ext_modules, From 7bf838003ad1b66c85617e0e90e25e4619727d2e Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 25 Nov 2017 14:45:28 +0100 Subject: [PATCH 153/798] build_usage (1.1-maint) also: git add borgfs.rst.inc --- docs/usage/borgfs.rst.inc | 138 ++++++++++++++++++++++++++++++++++ docs/usage/create.rst.inc | 8 +- docs/usage/key_export.rst.inc | 3 + docs/usage/list.rst.inc | 4 +- docs/usage/mount.rst.inc | 26 ++++++- 5 files changed, 174 insertions(+), 5 deletions(-) create mode 100644 docs/usage/borgfs.rst.inc diff --git a/docs/usage/borgfs.rst.inc b/docs/usage/borgfs.rst.inc new file mode 100644 index 0000000000..3488e936bf --- /dev/null +++ b/docs/usage/borgfs.rst.inc @@ -0,0 +1,138 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + +.. _borg_borgfs: + +borg borgfs +----------- +.. code-block:: none + + borg [common options] borgfs [options] REPOSITORY_OR_ARCHIVE MOUNTPOINT [PATH...] + +.. only:: html + + .. class:: borg-options-table + + +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ + | **positional arguments** | + +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``REPOSITORY_OR_ARCHIVE`` | repository/archive to mount | + +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``MOUNTPOINT`` | where to mount filesystem | + +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``PATH`` | paths to extract; patterns are supported | + +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ + | **optional arguments** | + +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``-V``, ``--version`` | show version number and exit | + +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``-f``, ``--foreground`` | stay in foreground, do not daemonize | + +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``-o`` | Extra mount options | + +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ + | .. class:: borg-common-opt-ref | + | | + | :ref:`common_options` | + +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ + | **Archive filters** — Archive filters can be applied to repository targets. | + +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``-P PREFIX``, ``--prefix PREFIX`` | only consider archive names starting with this prefix. | + +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``-a GLOB``, ``--glob-archives GLOB`` | only consider archive names matching the glob. sh: rules apply, see "borg help patterns". ``--prefix`` and ``--glob-archives`` are mutually exclusive. | + +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``--sort-by KEYS`` | Comma-separated list of sorting keys; valid keys are: timestamp, name, id; default is: timestamp | + +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``--first N`` | consider first N archives after other filters were applied | + +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``--last N`` | consider last N archives after other filters were applied | + +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ + | **Exclusion options** | + +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``-e PATTERN``, ``--exclude PATTERN`` | exclude paths matching PATTERN | + +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``--exclude-from EXCLUDEFILE`` | read exclude patterns from EXCLUDEFILE, one per line | + +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``--pattern PATTERN`` | experimental: include/exclude paths matching PATTERN | + +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``--patterns-from PATTERNFILE`` | experimental: read include/exclude patterns from PATTERNFILE, one per line | + +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``--strip-components NUMBER`` | Remove the specified number of leading path elements. Paths with fewer elements will be silently skipped. | + +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ + + .. raw:: html + + + +.. only:: latex + + REPOSITORY_OR_ARCHIVE + repository/archive to mount + MOUNTPOINT + where to mount filesystem + PATH + paths to extract; patterns are supported + + + optional arguments + -V, --version show version number and exit + -f, --foreground stay in foreground, do not daemonize + -o Extra mount options + + + :ref:`common_options` + | + + Archive filters + -P PREFIX, --prefix PREFIX only consider archive names starting with this prefix. + -a GLOB, --glob-archives GLOB only consider archive names matching the glob. sh: rules apply, see "borg help patterns". ``--prefix`` and ``--glob-archives`` are mutually exclusive. + --sort-by KEYS Comma-separated list of sorting keys; valid keys are: timestamp, name, id; default is: timestamp + --first N consider first N archives after other filters were applied + --last N consider last N archives after other filters were applied + + + Exclusion options + -e PATTERN, --exclude PATTERN exclude paths matching PATTERN + --exclude-from EXCLUDEFILE read exclude patterns from EXCLUDEFILE, one per line + --pattern PATTERN experimental: include/exclude paths matching PATTERN + --patterns-from PATTERNFILE experimental: read include/exclude patterns from PATTERNFILE, one per line + --strip-components NUMBER Remove the specified number of leading path elements. Paths with fewer elements will be silently skipped. + + +Description +~~~~~~~~~~~ + +This command mounts an archive as a FUSE filesystem. This can be useful for +browsing an archive or restoring individual files. Unless the ``--foreground`` +option is given the command will run in the background until the filesystem +is ``umounted``. + +The command ``borgfs`` provides a wrapper for ``borg mount``. This can also be +used in fstab entries: +``/path/to/repo /mnt/point fuse.borgfs defaults,noauto 0 0`` + +To allow a regular user to use fstab entries, add the ``user`` option: +``/path/to/repo /mnt/point fuse.borgfs defaults,noauto,user 0 0`` + +For mount options, see the fuse(8) manual page. Additional mount options +supported by borg: + +- versions: when used with a repository mount, this gives a merged, versioned + view of the files in the archives. EXPERIMENTAL, layout may change in future. +- allow_damaged_files: by default damaged files (where missing chunks were + replaced with runs of zeros by borg check ``--repair``) are not readable and + return EIO (I/O error). Set this option to read such files. + +The BORG_MOUNT_DATA_CACHE_ENTRIES environment variable is meant for advanced users +to tweak the performance. It sets the number of cached data chunks; additional +memory usage can be up to ~8 MiB times this number. The default is the number +of CPU cores. + +When the daemonized process receives a signal or crashes, it does not unmount. +Unmounting in these cases could cause an active rsync or similar process +to unintentionally delete data. + +When running in the foreground ^C/SIGINT unmounts cleanly, but other +signals or crashes do not. \ No newline at end of file diff --git a/docs/usage/create.rst.inc b/docs/usage/create.rst.inc index 1b6d22167d..57ba13a68e 100644 --- a/docs/usage/create.rst.inc +++ b/docs/usage/create.rst.inc @@ -67,6 +67,8 @@ borg create +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``--noctime`` | do not store ctime into archive | +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``--nobirthtime`` | do not store birthtime (creation date) into archive | + +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``--nobsdflags`` | do not read and store bsdflags (e.g. NODUMP, IMMUTABLE) into archive | +-------------------------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``--ignore-inode`` | ignore inode data in the file metadata cache used to detect unchanged files. | @@ -133,6 +135,7 @@ borg create --numeric-owner only store numeric user and group identifiers --noatime do not store atime into archive --noctime do not store ctime into archive + --nobirthtime do not store birthtime (creation date) into archive --nobsdflags do not read and store bsdflags (e.g. NODUMP, IMMUTABLE) into archive --ignore-inode ignore inode data in the file metadata cache used to detect unchanged files. --files-cache MODE operate files cache in MODE. default: ctime,size,inode @@ -214,8 +217,9 @@ the currently processed path. When using ``--stats``, you will get some statistics about how much data was added - the "This Archive" deduplicated size there is most interesting as that is -how much your repository will grow. -Please note that the "All archives" stats refer to the state after creation. +how much your repository will grow. Please note that the "All archives" stats refer to +the state after creation. Also, the ``--stats`` and ``--dry-run`` options are mutually +exclusive because the data is not actually compressed and deduplicated during a dry run. See the output of the "borg help patterns" command for more help on exclude patterns. See the output of the "borg help placeholders" command for more help on placeholders. diff --git a/docs/usage/key_export.rst.inc b/docs/usage/key_export.rst.inc index 466ac07d9d..a295138a18 100644 --- a/docs/usage/key_export.rst.inc +++ b/docs/usage/key_export.rst.inc @@ -59,6 +59,9 @@ Description If repository encryption is used, the repository is inaccessible without the key. This command allows to backup this essential key. +Note that the backup produced does not include the passphrase itself +(i.e. the exported key stays encrypted). In order to regain access to a +repository, one needs both the exported key and the original passphrase. There are two backup formats. The normal backup format is suitable for digital storage as a file. The ``--paper`` backup format is optimized diff --git a/docs/usage/list.rst.inc b/docs/usage/list.rst.inc index 6cf097665c..fbdfc09f73 100644 --- a/docs/usage/list.rst.inc +++ b/docs/usage/list.rst.inc @@ -118,15 +118,15 @@ The following keys are available for ``--format``: Keys for listing repository archives: -- name: archive name interpreted as text (might be missing non-text characters, see barchive) - archive: archive name interpreted as text (might be missing non-text characters, see barchive) +- name: alias of "archive" - barchive: verbatim archive name, can contain any character except NUL - comment: archive comment interpreted as text (might be missing non-text characters, see bcomment) - bcomment: verbatim archive comment, can contain any character except NUL - id: internal ID of the archive -- time: time (start) of creation of the archive - start: time (start) of creation of the archive +- time: alias of "start" - end: time (end) of creation of the archive diff --git a/docs/usage/mount.rst.inc b/docs/usage/mount.rst.inc index deff366571..ee7b80a07f 100644 --- a/docs/usage/mount.rst.inc +++ b/docs/usage/mount.rst.inc @@ -6,7 +6,7 @@ borg mount ---------- .. code-block:: none - borg [common options] mount [options] REPOSITORY_OR_ARCHIVE MOUNTPOINT + borg [common options] mount [options] REPOSITORY_OR_ARCHIVE MOUNTPOINT [PATH...] .. only:: html @@ -19,6 +19,8 @@ borg mount +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``MOUNTPOINT`` | where to mount filesystem | +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``PATH`` | paths to extract; patterns are supported | + +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ | **optional arguments** | +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``-f``, ``--foreground`` | stay in foreground, do not daemonize | @@ -41,6 +43,18 @@ borg mount +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``--last N`` | consider last N archives after other filters were applied | +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ + | **Exclusion options** | + +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``-e PATTERN``, ``--exclude PATTERN`` | exclude paths matching PATTERN | + +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``--exclude-from EXCLUDEFILE`` | read exclude patterns from EXCLUDEFILE, one per line | + +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``--pattern PATTERN`` | experimental: include/exclude paths matching PATTERN | + +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``--patterns-from PATTERNFILE`` | experimental: read include/exclude patterns from PATTERNFILE, one per line | + +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``--strip-components NUMBER`` | Remove the specified number of leading path elements. Paths with fewer elements will be silently skipped. | + +-----------------------------------------------------------------------------+---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ .. raw:: html @@ -56,6 +70,8 @@ borg mount repository/archive to mount MOUNTPOINT where to mount filesystem + PATH + paths to extract; patterns are supported optional arguments @@ -74,6 +90,14 @@ borg mount --last N consider last N archives after other filters were applied + Exclusion options + -e PATTERN, --exclude PATTERN exclude paths matching PATTERN + --exclude-from EXCLUDEFILE read exclude patterns from EXCLUDEFILE, one per line + --pattern PATTERN experimental: include/exclude paths matching PATTERN + --patterns-from PATTERNFILE experimental: read include/exclude patterns from PATTERNFILE, one per line + --strip-components NUMBER Remove the specified number of leading path elements. Paths with fewer elements will be silently skipped. + + Description ~~~~~~~~~~~ From 0340fbe42040971f720a07752a02d13474478d41 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 25 Nov 2017 14:49:00 +0100 Subject: [PATCH 154/798] build_man (1.1-maint) also: git add borgfs.1 --- docs/man/borg-benchmark-crud.1 | 2 +- docs/man/borg-benchmark.1 | 2 +- docs/man/borg-break-lock.1 | 2 +- docs/man/borg-change-passphrase.1 | 2 +- docs/man/borg-check.1 | 2 +- docs/man/borg-common.1 | 2 +- docs/man/borg-compression.1 | 2 +- docs/man/borg-create.1 | 10 +- docs/man/borg-delete.1 | 2 +- docs/man/borg-diff.1 | 2 +- docs/man/borg-export-tar.1 | 2 +- docs/man/borg-extract.1 | 2 +- docs/man/borg-info.1 | 2 +- docs/man/borg-init.1 | 2 +- docs/man/borg-key-change-passphrase.1 | 2 +- docs/man/borg-key-export.1 | 5 +- docs/man/borg-key-import.1 | 2 +- docs/man/borg-key-migrate-to-repokey.1 | 2 +- docs/man/borg-key.1 | 2 +- docs/man/borg-list.1 | 10 +- docs/man/borg-mount.1 | 25 ++++- docs/man/borg-patterns.1 | 2 +- docs/man/borg-placeholders.1 | 2 +- docs/man/borg-prune.1 | 2 +- docs/man/borg-recreate.1 | 2 +- docs/man/borg-rename.1 | 2 +- docs/man/borg-serve.1 | 2 +- docs/man/borg-umount.1 | 2 +- docs/man/borg-upgrade.1 | 2 +- docs/man/borg-with-lock.1 | 2 +- docs/man/borgfs.1 | 142 +++++++++++++++++++++++++ 31 files changed, 207 insertions(+), 37 deletions(-) create mode 100644 docs/man/borgfs.1 diff --git a/docs/man/borg-benchmark-crud.1 b/docs/man/borg-benchmark-crud.1 index d5fc412f44..06c1b21a00 100644 --- a/docs/man/borg-benchmark-crud.1 +++ b/docs/man/borg-benchmark-crud.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-BENCHMARK-CRUD 1 "2017-11-05" "" "borg backup tool" +.TH BORG-BENCHMARK-CRUD 1 "2017-11-25" "" "borg backup tool" .SH NAME borg-benchmark-crud \- Benchmark Create, Read, Update, Delete for archives. . diff --git a/docs/man/borg-benchmark.1 b/docs/man/borg-benchmark.1 index 7e0e1a192d..c6d44204f0 100644 --- a/docs/man/borg-benchmark.1 +++ b/docs/man/borg-benchmark.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-BENCHMARK 1 "2017-11-05" "" "borg backup tool" +.TH BORG-BENCHMARK 1 "2017-11-25" "" "borg backup tool" .SH NAME borg-benchmark \- benchmark command . diff --git a/docs/man/borg-break-lock.1 b/docs/man/borg-break-lock.1 index 7bf3d1d7a4..a6c93a2cab 100644 --- a/docs/man/borg-break-lock.1 +++ b/docs/man/borg-break-lock.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-BREAK-LOCK 1 "2017-11-05" "" "borg backup tool" +.TH BORG-BREAK-LOCK 1 "2017-11-25" "" "borg backup tool" .SH NAME borg-break-lock \- Break the repository lock (e.g. in case it was left by a dead borg. . diff --git a/docs/man/borg-change-passphrase.1 b/docs/man/borg-change-passphrase.1 index 5089fdb61a..c56f773a81 100644 --- a/docs/man/borg-change-passphrase.1 +++ b/docs/man/borg-change-passphrase.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-CHANGE-PASSPHRASE 1 "2017-11-05" "" "borg backup tool" +.TH BORG-CHANGE-PASSPHRASE 1 "2017-11-25" "" "borg backup tool" .SH NAME borg-change-passphrase \- Change repository key file passphrase . diff --git a/docs/man/borg-check.1 b/docs/man/borg-check.1 index 50282c8e9f..5a8660a275 100644 --- a/docs/man/borg-check.1 +++ b/docs/man/borg-check.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-CHECK 1 "2017-11-05" "" "borg backup tool" +.TH BORG-CHECK 1 "2017-11-25" "" "borg backup tool" .SH NAME borg-check \- Check repository consistency . diff --git a/docs/man/borg-common.1 b/docs/man/borg-common.1 index 9ccb7d96f1..2299016084 100644 --- a/docs/man/borg-common.1 +++ b/docs/man/borg-common.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-COMMON 1 "2017-11-05" "" "borg backup tool" +.TH BORG-COMMON 1 "2017-11-25" "" "borg backup tool" .SH NAME borg-common \- Common options of Borg commands . diff --git a/docs/man/borg-compression.1 b/docs/man/borg-compression.1 index 0a504ea5d7..f4ee1bf8e7 100644 --- a/docs/man/borg-compression.1 +++ b/docs/man/borg-compression.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-COMPRESSION 1 "2017-11-05" "" "borg backup tool" +.TH BORG-COMPRESSION 1 "2017-11-25" "" "borg backup tool" .SH NAME borg-compression \- Details regarding compression . diff --git a/docs/man/borg-create.1 b/docs/man/borg-create.1 index 2405c1cc56..71b766aed3 100644 --- a/docs/man/borg-create.1 +++ b/docs/man/borg-create.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-CREATE 1 "2017-11-05" "" "borg backup tool" +.TH BORG-CREATE 1 "2017-11-25" "" "borg backup tool" .SH NAME borg-create \- Create new archive . @@ -110,8 +110,9 @@ the currently processed path. .sp When using \fB\-\-stats\fP, you will get some statistics about how much data was added \- the "This Archive" deduplicated size there is most interesting as that is -how much your repository will grow. -Please note that the "All archives" stats refer to the state after creation. +how much your repository will grow. Please note that the "All archives" stats refer to +the state after creation. Also, the \fB\-\-stats\fP and \fB\-\-dry\-run\fP options are mutually +exclusive because the data is not actually compressed and deduplicated during a dry run. .sp See the output of the "borg help patterns" command for more help on exclude patterns. See the output of the "borg help placeholders" command for more help on placeholders. @@ -193,6 +194,9 @@ do not store atime into archive .B \-\-noctime do not store ctime into archive .TP +.B \-\-nobirthtime +do not store birthtime (creation date) into archive +.TP .B \-\-nobsdflags do not read and store bsdflags (e.g. NODUMP, IMMUTABLE) into archive .TP diff --git a/docs/man/borg-delete.1 b/docs/man/borg-delete.1 index f76ab2aa17..2b0e16fc7c 100644 --- a/docs/man/borg-delete.1 +++ b/docs/man/borg-delete.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-DELETE 1 "2017-11-05" "" "borg backup tool" +.TH BORG-DELETE 1 "2017-11-25" "" "borg backup tool" .SH NAME borg-delete \- Delete an existing repository or archives . diff --git a/docs/man/borg-diff.1 b/docs/man/borg-diff.1 index cab36023a9..77697f190b 100644 --- a/docs/man/borg-diff.1 +++ b/docs/man/borg-diff.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-DIFF 1 "2017-11-05" "" "borg backup tool" +.TH BORG-DIFF 1 "2017-11-25" "" "borg backup tool" .SH NAME borg-diff \- Diff contents of two archives . diff --git a/docs/man/borg-export-tar.1 b/docs/man/borg-export-tar.1 index 9d3c0f7d4e..770d2bafed 100644 --- a/docs/man/borg-export-tar.1 +++ b/docs/man/borg-export-tar.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-EXPORT-TAR 1 "2017-11-05" "" "borg backup tool" +.TH BORG-EXPORT-TAR 1 "2017-11-25" "" "borg backup tool" .SH NAME borg-export-tar \- Export archive contents as a tarball . diff --git a/docs/man/borg-extract.1 b/docs/man/borg-extract.1 index 76b090dc70..b11d737b31 100644 --- a/docs/man/borg-extract.1 +++ b/docs/man/borg-extract.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-EXTRACT 1 "2017-11-05" "" "borg backup tool" +.TH BORG-EXTRACT 1 "2017-11-25" "" "borg backup tool" .SH NAME borg-extract \- Extract archive contents . diff --git a/docs/man/borg-info.1 b/docs/man/borg-info.1 index ea075ff48a..726fed8266 100644 --- a/docs/man/borg-info.1 +++ b/docs/man/borg-info.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-INFO 1 "2017-11-05" "" "borg backup tool" +.TH BORG-INFO 1 "2017-11-25" "" "borg backup tool" .SH NAME borg-info \- Show archive details such as disk space used . diff --git a/docs/man/borg-init.1 b/docs/man/borg-init.1 index 0fb5417d91..9c9c2e62d9 100644 --- a/docs/man/borg-init.1 +++ b/docs/man/borg-init.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-INIT 1 "2017-11-05" "" "borg backup tool" +.TH BORG-INIT 1 "2017-11-25" "" "borg backup tool" .SH NAME borg-init \- Initialize an empty repository . diff --git a/docs/man/borg-key-change-passphrase.1 b/docs/man/borg-key-change-passphrase.1 index 3894588452..a94a067273 100644 --- a/docs/man/borg-key-change-passphrase.1 +++ b/docs/man/borg-key-change-passphrase.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-CHANGE-PASSPHRASE 1 "2017-11-05" "" "borg backup tool" +.TH BORG-KEY-CHANGE-PASSPHRASE 1 "2017-11-25" "" "borg backup tool" .SH NAME borg-key-change-passphrase \- Change repository key file passphrase . diff --git a/docs/man/borg-key-export.1 b/docs/man/borg-key-export.1 index 31f877966f..a47f56cbe2 100644 --- a/docs/man/borg-key-export.1 +++ b/docs/man/borg-key-export.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-EXPORT 1 "2017-11-05" "" "borg backup tool" +.TH BORG-KEY-EXPORT 1 "2017-11-25" "" "borg backup tool" .SH NAME borg-key-export \- Export the repository key for backup . @@ -37,6 +37,9 @@ borg [common options] key export [options] [REPOSITORY] [PATH] .sp If repository encryption is used, the repository is inaccessible without the key. This command allows to backup this essential key. +Note that the backup produced does not include the passphrase itself +(i.e. the exported key stays encrypted). In order to regain access to a +repository, one needs both the exported key and the original passphrase. .sp There are two backup formats. The normal backup format is suitable for digital storage as a file. The \fB\-\-paper\fP backup format is optimized diff --git a/docs/man/borg-key-import.1 b/docs/man/borg-key-import.1 index 20fa7ceb13..34b6b486b3 100644 --- a/docs/man/borg-key-import.1 +++ b/docs/man/borg-key-import.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-IMPORT 1 "2017-11-05" "" "borg backup tool" +.TH BORG-KEY-IMPORT 1 "2017-11-25" "" "borg backup tool" .SH NAME borg-key-import \- Import the repository key from backup . diff --git a/docs/man/borg-key-migrate-to-repokey.1 b/docs/man/borg-key-migrate-to-repokey.1 index 7edaa023d7..03f54d0bd8 100644 --- a/docs/man/borg-key-migrate-to-repokey.1 +++ b/docs/man/borg-key-migrate-to-repokey.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-MIGRATE-TO-REPOKEY 1 "2017-11-05" "" "borg backup tool" +.TH BORG-KEY-MIGRATE-TO-REPOKEY 1 "2017-11-25" "" "borg backup tool" .SH NAME borg-key-migrate-to-repokey \- Migrate passphrase -> repokey . diff --git a/docs/man/borg-key.1 b/docs/man/borg-key.1 index a449f50742..88f54f1783 100644 --- a/docs/man/borg-key.1 +++ b/docs/man/borg-key.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY 1 "2017-11-05" "" "borg backup tool" +.TH BORG-KEY 1 "2017-11-25" "" "borg backup tool" .SH NAME borg-key \- Manage a keyfile or repokey of a repository . diff --git a/docs/man/borg-list.1 b/docs/man/borg-list.1 index eb8761b8ef..c47e436f93 100644 --- a/docs/man/borg-list.1 +++ b/docs/man/borg-list.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-LIST 1 "2017-11-05" "" "borg backup tool" +.TH BORG-LIST 1 "2017-11-25" "" "borg backup tool" .SH NAME borg-list \- List archive or repository contents . @@ -152,10 +152,10 @@ LF Keys for listing repository archives: .INDENT 0.0 .IP \(bu 2 -name: archive name interpreted as text (might be missing non\-text characters, see barchive) -.IP \(bu 2 archive: archive name interpreted as text (might be missing non\-text characters, see barchive) .IP \(bu 2 +name: alias of "archive" +.IP \(bu 2 barchive: verbatim archive name, can contain any character except NUL .IP \(bu 2 comment: archive comment interpreted as text (might be missing non\-text characters, see bcomment) @@ -164,10 +164,10 @@ bcomment: verbatim archive comment, can contain any character except NUL .IP \(bu 2 id: internal ID of the archive .IP \(bu 2 -time: time (start) of creation of the archive -.IP \(bu 2 start: time (start) of creation of the archive .IP \(bu 2 +time: alias of "start" +.IP \(bu 2 end: time (end) of creation of the archive .UNINDENT .sp diff --git a/docs/man/borg-mount.1 b/docs/man/borg-mount.1 index 97ab864b55..6c4ac59546 100644 --- a/docs/man/borg-mount.1 +++ b/docs/man/borg-mount.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-MOUNT 1 "2017-11-05" "" "borg backup tool" +.TH BORG-MOUNT 1 "2017-11-25" "" "borg backup tool" .SH NAME borg-mount \- Mount archive or an entire repository as a FUSE filesystem . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .. .SH SYNOPSIS .sp -borg [common options] mount [options] REPOSITORY_OR_ARCHIVE MOUNTPOINT +borg [common options] mount [options] REPOSITORY_OR_ARCHIVE MOUNTPOINT [PATH...] .SH DESCRIPTION .sp This command mounts an archive as a FUSE filesystem. This can be useful for @@ -81,6 +81,9 @@ repository/archive to mount .TP .B MOUNTPOINT where to mount filesystem +.TP +.B PATH +paths to extract; patterns are supported .UNINDENT .SS optional arguments .INDENT 0.0 @@ -109,6 +112,24 @@ consider first N archives after other filters were applied .BI \-\-last \ N consider last N archives after other filters were applied .UNINDENT +.SS Exclusion options +.INDENT 0.0 +.TP +.BI \-e \ PATTERN\fP,\fB \ \-\-exclude \ PATTERN +exclude paths matching PATTERN +.TP +.BI \-\-exclude\-from \ EXCLUDEFILE +read exclude patterns from EXCLUDEFILE, one per line +.TP +.BI \-\-pattern \ PATTERN +experimental: include/exclude paths matching PATTERN +.TP +.BI \-\-patterns\-from \ PATTERNFILE +experimental: read include/exclude patterns from PATTERNFILE, one per line +.TP +.BI \-\-strip\-components \ NUMBER +Remove the specified number of leading path elements. Paths with fewer elements will be silently skipped. +.UNINDENT .SH SEE ALSO .sp \fIborg\-common(1)\fP, \fIborg\-umount(1)\fP, \fIborg\-extract(1)\fP diff --git a/docs/man/borg-patterns.1 b/docs/man/borg-patterns.1 index 0377cc6ea9..82b347d055 100644 --- a/docs/man/borg-patterns.1 +++ b/docs/man/borg-patterns.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-PATTERNS 1 "2017-11-05" "" "borg backup tool" +.TH BORG-PATTERNS 1 "2017-11-25" "" "borg backup tool" .SH NAME borg-patterns \- Details regarding patterns . diff --git a/docs/man/borg-placeholders.1 b/docs/man/borg-placeholders.1 index e3491fd722..9f50120155 100644 --- a/docs/man/borg-placeholders.1 +++ b/docs/man/borg-placeholders.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-PLACEHOLDERS 1 "2017-11-05" "" "borg backup tool" +.TH BORG-PLACEHOLDERS 1 "2017-11-25" "" "borg backup tool" .SH NAME borg-placeholders \- Details regarding placeholders . diff --git a/docs/man/borg-prune.1 b/docs/man/borg-prune.1 index 58e35ff169..51fab8918f 100644 --- a/docs/man/borg-prune.1 +++ b/docs/man/borg-prune.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-PRUNE 1 "2017-11-05" "" "borg backup tool" +.TH BORG-PRUNE 1 "2017-11-25" "" "borg backup tool" .SH NAME borg-prune \- Prune repository archives according to specified rules . diff --git a/docs/man/borg-recreate.1 b/docs/man/borg-recreate.1 index 4169e6f842..8c478ceb1b 100644 --- a/docs/man/borg-recreate.1 +++ b/docs/man/borg-recreate.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-RECREATE 1 "2017-11-05" "" "borg backup tool" +.TH BORG-RECREATE 1 "2017-11-25" "" "borg backup tool" .SH NAME borg-recreate \- Re-create archives . diff --git a/docs/man/borg-rename.1 b/docs/man/borg-rename.1 index 55fcd86c88..d82942e7f8 100644 --- a/docs/man/borg-rename.1 +++ b/docs/man/borg-rename.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-RENAME 1 "2017-11-05" "" "borg backup tool" +.TH BORG-RENAME 1 "2017-11-25" "" "borg backup tool" .SH NAME borg-rename \- Rename an existing archive . diff --git a/docs/man/borg-serve.1 b/docs/man/borg-serve.1 index 0aa10d97ea..7af07f484d 100644 --- a/docs/man/borg-serve.1 +++ b/docs/man/borg-serve.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-SERVE 1 "2017-11-05" "" "borg backup tool" +.TH BORG-SERVE 1 "2017-11-25" "" "borg backup tool" .SH NAME borg-serve \- Start in server mode. This command is usually not used manually. . diff --git a/docs/man/borg-umount.1 b/docs/man/borg-umount.1 index 8bae2dff5e..14534a513c 100644 --- a/docs/man/borg-umount.1 +++ b/docs/man/borg-umount.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-UMOUNT 1 "2017-11-05" "" "borg backup tool" +.TH BORG-UMOUNT 1 "2017-11-25" "" "borg backup tool" .SH NAME borg-umount \- un-mount the FUSE filesystem . diff --git a/docs/man/borg-upgrade.1 b/docs/man/borg-upgrade.1 index abe4dbf188..74ddb96016 100644 --- a/docs/man/borg-upgrade.1 +++ b/docs/man/borg-upgrade.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-UPGRADE 1 "2017-11-05" "" "borg backup tool" +.TH BORG-UPGRADE 1 "2017-11-25" "" "borg backup tool" .SH NAME borg-upgrade \- upgrade a repository from a previous version . diff --git a/docs/man/borg-with-lock.1 b/docs/man/borg-with-lock.1 index 9f7951f4bd..0f2169ec68 100644 --- a/docs/man/borg-with-lock.1 +++ b/docs/man/borg-with-lock.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-WITH-LOCK 1 "2017-11-05" "" "borg backup tool" +.TH BORG-WITH-LOCK 1 "2017-11-25" "" "borg backup tool" .SH NAME borg-with-lock \- run a user specified command with the repository lock held . diff --git a/docs/man/borgfs.1 b/docs/man/borgfs.1 new file mode 100644 index 0000000000..bac62ff5ce --- /dev/null +++ b/docs/man/borgfs.1 @@ -0,0 +1,142 @@ +.\" Man page generated from reStructuredText. +. +.TH BORGFS 1 "2017-11-25" "" "borg backup tool" +.SH NAME +borgfs \- Mount archive or an entire repository as a FUSE filesystem +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.sp +borgfs [options] REPOSITORY_OR_ARCHIVE MOUNTPOINT [PATH...] +.SH DESCRIPTION +.sp +This command mounts an archive as a FUSE filesystem. This can be useful for +browsing an archive or restoring individual files. Unless the \fB\-\-foreground\fP +option is given the command will run in the background until the filesystem +is \fBumounted\fP\&. +.sp +The command \fBborgfs\fP provides a wrapper for \fBborg mount\fP\&. This can also be +used in fstab entries: +\fB/path/to/repo /mnt/point fuse.borgfs defaults,noauto 0 0\fP +.sp +To allow a regular user to use fstab entries, add the \fBuser\fP option: +\fB/path/to/repo /mnt/point fuse.borgfs defaults,noauto,user 0 0\fP +.sp +For mount options, see the fuse(8) manual page. Additional mount options +supported by borg: +.INDENT 0.0 +.IP \(bu 2 +versions: when used with a repository mount, this gives a merged, versioned +view of the files in the archives. EXPERIMENTAL, layout may change in future. +.IP \(bu 2 +allow_damaged_files: by default damaged files (where missing chunks were +replaced with runs of zeros by borg check \fB\-\-repair\fP) are not readable and +return EIO (I/O error). Set this option to read such files. +.UNINDENT +.sp +The BORG_MOUNT_DATA_CACHE_ENTRIES environment variable is meant for advanced users +to tweak the performance. It sets the number of cached data chunks; additional +memory usage can be up to ~8 MiB times this number. The default is the number +of CPU cores. +.sp +When the daemonized process receives a signal or crashes, it does not unmount. +Unmounting in these cases could cause an active rsync or similar process +to unintentionally delete data. +.sp +When running in the foreground ^C/SIGINT unmounts cleanly, but other +signals or crashes do not. +.SH OPTIONS +.sp +See \fIborg\-common(1)\fP for common options of Borg commands. +.SS arguments +.INDENT 0.0 +.TP +.B REPOSITORY_OR_ARCHIVE +repository/archive to mount +.TP +.B MOUNTPOINT +where to mount filesystem +.TP +.B PATH +paths to extract; patterns are supported +.UNINDENT +.SS optional arguments +.INDENT 0.0 +.TP +.B \-V\fP,\fB \-\-version +show version number and exit +.TP +.B \-f\fP,\fB \-\-foreground +stay in foreground, do not daemonize +.TP +.B \-o +Extra mount options +.UNINDENT +.SS Archive filters +.INDENT 0.0 +.TP +.BI \-P \ PREFIX\fP,\fB \ \-\-prefix \ PREFIX +only consider archive names starting with this prefix. +.TP +.BI \-a \ GLOB\fP,\fB \ \-\-glob\-archives \ GLOB +only consider archive names matching the glob. sh: rules apply, see "borg help patterns". \fB\-\-prefix\fP and \fB\-\-glob\-archives\fP are mutually exclusive. +.TP +.BI \-\-sort\-by \ KEYS +Comma\-separated list of sorting keys; valid keys are: timestamp, name, id; default is: timestamp +.TP +.BI \-\-first \ N +consider first N archives after other filters were applied +.TP +.BI \-\-last \ N +consider last N archives after other filters were applied +.UNINDENT +.SS Exclusion options +.INDENT 0.0 +.TP +.BI \-e \ PATTERN\fP,\fB \ \-\-exclude \ PATTERN +exclude paths matching PATTERN +.TP +.BI \-\-exclude\-from \ EXCLUDEFILE +read exclude patterns from EXCLUDEFILE, one per line +.TP +.BI \-\-pattern \ PATTERN +experimental: include/exclude paths matching PATTERN +.TP +.BI \-\-patterns\-from \ PATTERNFILE +experimental: read include/exclude patterns from PATTERNFILE, one per line +.TP +.BI \-\-strip\-components \ NUMBER +Remove the specified number of leading path elements. Paths with fewer elements will be silently skipped. +.UNINDENT +.SH SEE ALSO +.sp +\fIborg\-common(1)\fP +.SH AUTHOR +The Borg Collective +.\" Generated by docutils manpage writer. +. From 3bdf1193ad50ae2c44ea0b54655a8984bfe9f5bd Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 25 Nov 2017 15:21:43 +0100 Subject: [PATCH 155/798] docs: point out tuning options for borg create, fixes #3239 (cherry picked from commit 520a6a2ef116331964a57e857d32ff217b14fb5d) --- src/borg/archiver.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index fdeaec0f90..6f372e731f 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -2916,6 +2916,12 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): it had before a content change happened. This can be used maliciously as well as well-meant, but in both cases mtime based cache modes can be problematic. + By default, borg tries to archive all metadata that it supports archiving. + If that is not what you want or need, there are some tuning options: + + - --nobsdflags (getting bsdflags has a speed penalty under Linux) + - --noatime (if atime changes frequently, the metadata stream will dedup badly) + The mount points of filesystems or filesystem snapshots should be the same for every creation of a new archive to ensure fast operation. This is because the file cache that is used to determine changed files quickly uses absolute filenames. From 5331b378f74695cf8397ac085800fbf3bd73a857 Mon Sep 17 00:00:00 2001 From: Milkey Mouse Date: Sat, 25 Nov 2017 12:48:00 -0800 Subject: [PATCH 156/798] fixup! Add borg config command (fixes #3304) Don't use list unpacking for function calls in order to support py3.4 Reword basic borg config help It doesn't edit the repo config, but any borg-related config. --- src/borg/archiver.py | 2 +- src/borg/testsuite/archiver.py | 22 ++++++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 7feb410835..13fc3db2a2 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -3733,7 +3733,7 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): description=self.do_config.__doc__, epilog=config_epilog, formatter_class=argparse.RawDescriptionHelpFormatter, - help='get and set repository config options') + help='get and set configuration values') subparser.set_defaults(func=self.do_config) subparser.add_argument('-c', '--cache', dest='cache', action='store_true', help='get and set values from the repo cache') diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index a773bd9240..0d36a27f13 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -2782,14 +2782,20 @@ def test_config(self): self.create_test_files() os.unlink('input/flagfile') self.cmd('init', '--encryption=repokey', self.repository_location) - for flags in [[], ['--cache']]: - for cfg_key in {'testkey', 'testsection.testkey'}: - self.cmd('config', self.repository_location, *flags, cfg_key, exit_code=1) - self.cmd('config', self.repository_location, *flags, cfg_key, 'testcontents') - output = self.cmd('config', self.repository_location, *flags, cfg_key) - assert output == 'testcontents\n' - self.cmd('config', self.repository_location, *flags, '--delete', cfg_key) - self.cmd('config', self.repository_location, *flags, cfg_key, exit_code=1) + for cfg_key in {'testkey', 'testsection.testkey'}: + self.cmd('config', self.repository_location, cfg_key, exit_code=1) + self.cmd('config', self.repository_location, cfg_key, 'testcontents') + output = self.cmd('config', self.repository_location, cfg_key) + assert output == 'testcontents\n' + self.cmd('config', self.repository_location, '--delete', cfg_key) + self.cmd('config', self.repository_location, cfg_key, exit_code=1) + + self.cmd('config', self.repository_location, '--cache', cfg_key, exit_code=1) + self.cmd('config', self.repository_location, '--cache', cfg_key, 'testcontents') + output = self.cmd('config', self.repository_location, '--cache', cfg_key) + assert output == 'testcontents\n' + self.cmd('config', self.repository_location, '--cache', '--delete', cfg_key) + self.cmd('config', self.repository_location, '--cache', cfg_key, exit_code=1) requires_gnutar = pytest.mark.skipif(not have_gnutar(), reason='GNU tar must be installed for this test.') requires_gzip = pytest.mark.skipif(not shutil.which('gzip'), reason='gzip must be installed for this test.') From 938d1221913c62fc0114e5032c163ef95fae5a0d Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 25 Nov 2017 19:29:34 +0100 Subject: [PATCH 157/798] docs: fix examples with problematic option placements, fixes #3356 have options to the left OR to the right of all positional arguments, but not on BOTH sides and not in between them. (cherry picked from commit f2a1539f25813647307bd8b6f0a4788ec78884b7) --- docs/faq.rst | 4 ++-- docs/usage/create.rst | 2 +- docs/usage/tar.rst | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index b01db24150..1832f72266 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -94,7 +94,7 @@ retransfer the data since the last checkpoint. If a backup was interrupted, you normally do not need to do anything special, just invoke ``borg create`` as you always do. If the repository is still locked, -you may need to run ``borg break-lock`` before the next backup. You may use the +you may need to run ``borg break-lock`` before the next backup. You may use the same archive name as in previous attempt or a different one (e.g. if you always include the current datetime), it does not matter. @@ -265,7 +265,7 @@ Say you want to prune ``/var/log`` faster than the rest of archive *names* and then implement different prune policies for different prefixes. For example, you could have a script that does:: - borg create $REPOSITORY:main-$(date +%Y-%m-%d) --exclude /var/log / + borg create --exclude /var/log $REPOSITORY:main-$(date +%Y-%m-%d) / borg create $REPOSITORY:logs-$(date +%Y-%m-%d) /var/log Then you would have two different prune calls with different policies:: diff --git a/docs/usage/create.rst b/docs/usage/create.rst index ec49f5fa83..d221c4e65a 100644 --- a/docs/usage/create.rst +++ b/docs/usage/create.rst @@ -23,7 +23,7 @@ Examples # Backup the root filesystem into an archive named "root-YYYY-MM-DD" # use zlib compression (good, but slow) - default is lz4 (fast, low compression ratio) - $ borg create -C zlib,6 /path/to/repo::root-{now:%Y-%m-%d} / --one-file-system + $ borg create -C zlib,6 --one-file-system /path/to/repo::root-{now:%Y-%m-%d} / # Backup a remote host locally ("pull" style) using sshfs $ mkdir sshfs-mount diff --git a/docs/usage/tar.rst b/docs/usage/tar.rst index 3b63afb4d8..dc79b708ef 100644 --- a/docs/usage/tar.rst +++ b/docs/usage/tar.rst @@ -11,11 +11,11 @@ Examples $ borg export-tar /path/to/repo::Monday Monday.tar.gz --exclude '*.so' # use higher compression level with gzip - $ borg export-tar testrepo::linux --tar-filter="gzip -9" Monday.tar.gz + $ borg export-tar --tar-filter="gzip -9" testrepo::linux Monday.tar.gz - # export a gzipped tar, but instead of storing it on disk, + # export a tar, but instead of storing it on disk, # upload it to a remote site using curl. - $ borg export-tar ... --tar-filter="gzip" - | curl --data-binary @- https://somewhere/to/POST + $ borg export-tar /path/to/repo::Monday - | curl --data-binary @- https://somewhere/to/POST # remote extraction via "tarpipe" $ borg export-tar /path/to/repo::Monday - | ssh somewhere "cd extracted; tar x" From 21afd17f2f5be051e50c5fb309ab18607834d42e Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 25 Nov 2017 19:46:48 +0100 Subject: [PATCH 158/798] docs: document good and problematic option placements, see #3356 (cherry picked from commit e3cb7c3c32620e36b792e99972167925c8bfbedf) --- docs/usage_general.rst.inc | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/usage_general.rst.inc b/docs/usage_general.rst.inc index c87a8b8b47..4c704d8f0c 100644 --- a/docs/usage_general.rst.inc +++ b/docs/usage_general.rst.inc @@ -1,3 +1,20 @@ +Positional Arguments and Options: Order matters +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Borg only supports taking options (``-s`` and ``--progress`` in the example) +to the left or right of all positional arguments (``repo::archive`` and ``path`` +in the example), but not in between them: + +:: + + borg create -s --progress repo::archive path # good and preferred + borg create repo::archive path -s --progress # also works + borg create -s repo::archive path --progress # works, but ugly + borg create repo::archive -s --progress path # BAD + +This is due to a problem in the argparse module: http://bugs.python.org/issue15112 + + Repository URLs ~~~~~~~~~~~~~~~ From b175f1fdf1b6e6ccc8e000974fd55f8960d85a86 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 25 Nov 2017 19:51:33 +0100 Subject: [PATCH 159/798] docs: document birthtime support on platforms supporting it stat.st_birthtime, MacOS and (some?) BSDs. (cherry picked from commit a107233f7564e4abc80ce26ff72e73ff867b776d) --- docs/usage_general.rst.inc | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/usage_general.rst.inc b/docs/usage_general.rst.inc index 4c704d8f0c..2f1ec3f11e 100644 --- a/docs/usage_general.rst.inc +++ b/docs/usage_general.rst.inc @@ -390,6 +390,7 @@ Besides regular file and directory structures, Borg can preserve By default the metadata to create them with mknod(2), mkfifo(2) etc. is stored. * hardlinked regular files, devices, FIFOs (considering all items in the same archive) * timestamps in nanosecond precision: mtime, atime, ctime +* other timestamps: birthtime (on platforms supporting it) * permissions: * IDs of owning user and owning group From ef39dc6554e1e7a8dc747cffcd556af026d79975 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 25 Nov 2017 20:08:33 +0100 Subject: [PATCH 160/798] docs: move bsdflags tuning comments to notes docs section the atime comments are already there. (cherry picked from commit 91ff2f81a6a64523f433eab7334eb79b81857cc2) --- docs/usage/notes.rst | 10 ++++++++++ src/borg/archiver.py | 6 ------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/usage/notes.rst b/docs/usage/notes.rst index 4342a8c12c..d5415d0e9d 100644 --- a/docs/usage/notes.rst +++ b/docs/usage/notes.rst @@ -61,6 +61,16 @@ affect metadata stream deduplication: if only this timestamp changes between backups and is stored into the metadata stream, the metadata stream chunks won't deduplicate just because of that. +``--nobsdflags`` +~~~~~~~~~~~~~~~~ + +You can use this to not query and store (or not extract and set) bsdflags - +in case you don't need them or if they are broken somehow for your fs. + +On Linux, dealing with the bsflags needs some additional syscalls. +Especially when dealing with lots of small files, this causes a noticable +overhead, so you can use this option also for speeding up operations. + ``--umask`` ~~~~~~~~~~~ diff --git a/src/borg/archiver.py b/src/borg/archiver.py index d99822d4fc..13fc3db2a2 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -2952,12 +2952,6 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): it had before a content change happened. This can be used maliciously as well as well-meant, but in both cases mtime based cache modes can be problematic. - By default, borg tries to archive all metadata that it supports archiving. - If that is not what you want or need, there are some tuning options: - - - --nobsdflags (getting bsdflags has a speed penalty under Linux) - - --noatime (if atime changes frequently, the metadata stream will dedup badly) - The mount points of filesystems or filesystem snapshots should be the same for every creation of a new archive to ensure fast operation. This is because the file cache that is used to determine changed files quickly uses absolute filenames. From 25b40a261b5d9554112bd42905004644712c195a Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 25 Nov 2017 20:18:59 +0100 Subject: [PATCH 161/798] docs: give examples for borg mount exclusion options (cherry picked from commit 0eaec603846968af10bb5cc9ef466af58c325907) --- docs/usage/mount.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/usage/mount.rst b/docs/usage/mount.rst index 46c7d554c5..f170fb317f 100644 --- a/docs/usage/mount.rst +++ b/docs/usage/mount.rst @@ -36,6 +36,12 @@ Examples # which does not support lazy processing of archives. $ borg mount -o versions --glob-archives '*-my-home' --last 10 /path/to/repo /tmp/mymountpoint + # Exclusion options are supported. + # These can speed up mounting and lower memory needs significantly. + $ borg mount /path/to/repo /tmp/mymountpoint only/that/path + $ borg mount --exclude '...' /path/to/repo /tmp/mymountpoint + + borgfs ++++++ From e488e69f41860f78b70a1257167ea4fa32592b71 Mon Sep 17 00:00:00 2001 From: Milkey Mouse Date: Sat, 25 Nov 2017 17:06:02 -0800 Subject: [PATCH 162/798] Use name of command for "virtual man page" name This makes it slightly easier to debug manpage issues without removing the virtmanpage hack. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2670bbfc88..7140ac161b 100644 --- a/setup.py +++ b/setup.py @@ -674,7 +674,7 @@ def issue(name, rawtext, text, lineno, inliner, options={}, content=[]): roles.register_local_role('issue', issue) # We give the source_path so that docutils can find relative includes # as-if the document where located in the docs/ directory. - man_page = publish_string(source=rst, source_path='docs/virtmanpage.rst', writer=manpage.Writer()) + man_page = publish_string(source=rst, source_path='docs/%s.rst' % name, writer=manpage.Writer()) with open('docs/man/%s.1' % name, 'wb') as fd: fd.write(man_page) From 248c408b5b68384ae60704d9b9b25f214c39c941 Mon Sep 17 00:00:00 2001 From: Milkey Mouse Date: Sat, 25 Nov 2017 17:16:28 -0800 Subject: [PATCH 163/798] Fix build_man issues (fixes #3364) --- src/borg/archiver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 13fc3db2a2..a1f24b1d34 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -3953,7 +3953,7 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): R- == borg extract (extract archive, dry-run, do everything, but do not write files to disk) R-Z- == all zero files. Measuring heavily duplicated files. R-R- == random files. No duplication here, measuring throughput through all processing - stages, except writing to disk. + stages, except writing to disk. U- == borg create (2nd archive creation of unchanged input files, measure files cache speed) The throughput value is kind of virtual here, it does not actually read the file. From a40dd4c3b65a13a3955d40fbda12e78dfe47b2e8 Mon Sep 17 00:00:00 2001 From: Michael Rupert Date: Sat, 25 Nov 2017 23:30:09 -0500 Subject: [PATCH 164/798] fixed support links (#3379) (#3381) fixed support links, #3379 (cherry picked from commit 6dd54f93cefe1a1e273fce796a5fb3a39a882b27) --- README.rst | 3 ++- docs/development.rst | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 46e9266b13..00874d9125 100644 --- a/README.rst +++ b/README.rst @@ -127,7 +127,8 @@ Spread the word, give feedback, help with documentation, testing or development. You can also give monetary support to the project, see there for details: -https://borgbackup.readthedocs.io/en/stable/support.html#bounties-and-fundraisers +https://www.borgbackup.org/support/free.html#bounties-and-fundraisers + Links ----- diff --git a/docs/development.rst b/docs/development.rst index 69ff946972..0968b5acbd 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -50,7 +50,8 @@ Branching model Borg development happens on the ``master`` branch and uses GitHub pull requests (if you don't have GitHub or don't want to use it you can -send smaller patches via the borgbackup :ref:`mailing_list` to the maintainers). +send smaller patches via the borgbackup `Mailing List`_. +to the maintainers). Stable releases are maintained on maintenance branches named x.y-maint, eg. the maintenance branch of the 1.0.x series is 1.0-maint. From d8ff84c2f49bd524249c0471d39c4930a014cc1a Mon Sep 17 00:00:00 2001 From: TW Date: Sun, 26 Nov 2017 18:47:20 +0100 Subject: [PATCH 165/798] WIP: update CHANGES (1.1-maint) (#3368) update CHANGES (1.1-maint) --- docs/changes.rst | 77 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 75 insertions(+), 2 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index 4c85d43f74..e80a968bb2 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -131,14 +131,15 @@ The best check that everything is ok is to run a dry-run extraction:: Changelog ========= -Version 1.1.2 (2017-11-05) --------------------------- +Version 1.1.3 (not released yet) +-------------------------------- Compatibility notes: - When upgrading from borg 1.0.x to 1.1.x, please note: - read all the compatibility notes for 1.1.0*, starting from 1.1.0b1. + - borg upgrade: you do not need to and you also should not run it. - borg might ask some security-related questions once after upgrading. You can answer them either manually or via environment variable. One known case is if you use unencrypted repositories, then it will ask @@ -152,6 +153,65 @@ Compatibility notes: Fixes: +- XXX SECFIX XXX +- crc32: deal with unaligned buffer, add tests - this broke borg on older ARM + CPUs that can not deal with unaligned 32bit memory accesses and raise a bus + error in such cases. the fix might also improve performance on some CPUs as + all 32bit memory accesses by the crc32 code are properly aligned now. #3317 +- mount: fixed support of --consider-part-files and do not show .borg_part_N + files by default in the mounted FUSE filesystem. #3347 +- fixed cache/repo timestamp inconsistency message, highlight that information + is obtained from security dir (deleting the cache will not bypass this error + in case the user knows this is a legitimate repo). +- borgfs: don't show sub-command in borgfs help, #3287 +- create: show an error when --dry-run and --stats are used together, #3298 + +New features: + +- mount: added exclusion group options and paths, #2138 + + Reused some code to support similar options/paths as borg extract offers - + making good use of these to only mount a smaller subset of dirs/files can + speed up mounting a lot and also will consume way less memory. + + borg mount [options] repo_or_archive mountpoint path [paths...] + + paths: you can just give some "root paths" (like for borg extract) to + only partially populate the FUSE filesystem. + + new options: --exclude[-from], --pattern[s-from], --strip-components +- create/extract: support st_birthtime on platforms supporting it, #3272 +- add "borg config" command for querying/setting/deleting config values, #3304 + +Other changes: + +- clean up and simplify packaging (only package committed files, do not install + .c/.h/.pyx files) +- docs: + + - point out tuning options for borg create, #3239 + - add instructions for using ntfsclone, zerofree, #81 + - move image backup-related FAQ entries to a new page + - clarify key aliases for borg list --format, #3111 + - mention break-lock in checkpointing FAQ entry, #3328 + - document sshfs rename workaround, #3315 + - add FAQ about removing files from existing archives + - add FAQ about different prune policies + - usage and man page for borgfs, #3216 + - clarify create --stats duration vs. wall time, #3301 + - clarify encrypted key format for borg key export, #3296 + - update release checklist about security fixes + - document good and problematic option placements, fix examples, #3356 + - add note about using --nobsdflags to avoid speed penalty related to + bsdflags, #3239 + - move most of support section to www.borgbackup.org + + +Version 1.1.2 (2017-11-05) +-------------------------- + +Fixes: + - fix KeyError crash when talking to borg server < 1.0.7, #3244 - extract: set bsdflags last (include immutable flag), #3263 - create: don't do stat() call on excluded-norecurse directory, fix exception @@ -246,6 +306,19 @@ Other changes: Version 1.1.0 (2017-10-07) -------------------------- +Compatibility notes: + +- borg command line: do not put options in between positional arguments + + This sometimes works (e.g. it worked in borg 1.0.x), but can easily stop + working if we make positional arguments optional (like it happened for + borg create's "paths" argument in 1.1). There are also places in borg 1.0 + where we do that, so it doesn't work there in general either. #3356 + + Good: borg create -v --stats repo::archive path + Good: borg create repo::archive path -v --stats + Bad: borg create repo::archive -v --stats path + Fixes: - fix LD_LIBRARY_PATH restoration for subprocesses, #3077 From 2e365f46de4d41d4dc7eaefd587c1e8c66b53a96 Mon Sep 17 00:00:00 2001 From: Michael Rupert Date: Fri, 24 Nov 2017 21:15:18 -0500 Subject: [PATCH 166/798] do remaining part of #3360 backport to 1.1-maint --- docs/development.rst | 2 +- docs/support.rst | 69 ++------------------------------------------ 2 files changed, 3 insertions(+), 68 deletions(-) diff --git a/docs/development.rst b/docs/development.rst index 0968b5acbd..43552e6cf7 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -50,7 +50,7 @@ Branching model Borg development happens on the ``master`` branch and uses GitHub pull requests (if you don't have GitHub or don't want to use it you can -send smaller patches via the borgbackup `Mailing List`_. +send smaller patches via the borgbackup `Mailing List`_. to the maintainers). Stable releases are maintained on maintenance branches named x.y-maint, eg. diff --git a/docs/support.rst b/docs/support.rst index 8bd3fcbedf..2c1f40adc3 100644 --- a/docs/support.rst +++ b/docs/support.rst @@ -1,71 +1,6 @@ -.. include:: global.rst.inc -.. _support: -Support -======= - -Please first read the docs, the existing issue tracker issues and mailing -list posts -- a lot of stuff is already documented / explained / discussed / -filed there. - -Issue Tracker -------------- - -If you've found a bug or have a concrete feature request, please create a new -ticket on the project's `issue tracker`_. - -For more general questions or discussions, IRC or mailing list are preferred. - -Chat (IRC) ----------- -Join us on channel #borgbackup on chat.freenode.net. - -As usual on IRC, just ask or tell directly and then patiently wait for replies. -Stay connected. - -You could use the following link (after connecting, you can change the random -nickname you get by typing "/nick mydesirednickname"): - -http://webchat.freenode.net/?randomnick=1&channels=%23borgbackup&uio=MTY9dHJ1ZSY5PXRydWUa8 - -.. _mailing_list: - -Mailing list ------------- - -To find out about the mailing list, its topic, how to subscribe, how to -unsubscribe and where you can find the archives of the list, see the -`mailing list homepage -`_. - -Twitter -------- - -Follow @borgbackup for announcements. You can also add @borgbackup if you -would like to get retweeted for a borg related tweet. - -Please understand that Twitter is not suitable for longer / more complex -discussions - use one of the other channels for that. - -Bounties and Fundraisers ------------------------- - -We use `BountySource `_ to allow -monetary contributions to the project and the developers, who push it forward. - -There, you can give general funds to the borgbackup members (the developers will -then spend the funds as they deem fit). If you do not have some specific bounty -(see below), you can use this as a general way to say "Thank You!" and support -the software / project you like. - -If you want to encourage developers to fix some specific issue or implement some -specific feature suggestion, you can post a new bounty or back an existing one -(they always refer to an issue in our `issue tracker`_). - -As a developer, you can become a Bounty Hunter and win bounties (earn money) by -contributing to |project_name|, a free and open source software project. - -We might also use BountySource to fund raise for some bigger goals. +Security +======== .. _security-contact: From 7438cb2ef48e3cad9c220fc25de0a462a425bed8 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 26 Nov 2017 23:34:10 +0100 Subject: [PATCH 167/798] link to the homepage for support/service (1.1-maint) otherwise people ONLY reading the docs (and not coming from the homepage) will miss these options. --- docs/support.rst | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/support.rst b/docs/support.rst index 2c1f40adc3..730ffd0b8f 100644 --- a/docs/support.rst +++ b/docs/support.rst @@ -1,6 +1,11 @@ +Support +======= + +Support and Services +-------------------- + +Please see https://www.borgbackup.org/ for free and paid support and service options. -Security -======== .. _security-contact: From c7994b598b696d7fb585c21e5a8c489af30a37e4 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 26 Nov 2017 23:56:32 +0100 Subject: [PATCH 168/798] docs: fix broken links (1.1-maint) --- docs/development.rst | 3 +-- docs/support.rst | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/development.rst b/docs/development.rst index 43552e6cf7..f3b60cd110 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -50,8 +50,7 @@ Branching model Borg development happens on the ``master`` branch and uses GitHub pull requests (if you don't have GitHub or don't want to use it you can -send smaller patches via the borgbackup `Mailing List`_. -to the maintainers). +send smaller patches via the borgbackup mailing list to the maintainers). Stable releases are maintained on maintenance branches named x.y-maint, eg. the maintenance branch of the 1.0.x series is 1.0-maint. diff --git a/docs/support.rst b/docs/support.rst index 730ffd0b8f..0e583dbea9 100644 --- a/docs/support.rst +++ b/docs/support.rst @@ -1,3 +1,5 @@ +.. _support: + Support ======= From ea0203bb0de557cd29de5ab0a0efe5f6015ca59d Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 26 Nov 2017 19:51:10 +0100 Subject: [PATCH 169/798] security: fix enforcement of --restrict-to-path in args processing Fixes CVE-2017-15914 (affects releases 1.1.0, 1.1.1, 1.1.2, but not 1.0.x). Thanks to Florian Apolloner for discovering/reporting this! Also: added tests for this. --- src/borg/archiver.py | 1 + src/borg/testsuite/archiver.py | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index a1f24b1d34..9bc5db614b 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -3997,6 +3997,7 @@ def get_args(self, argv, cmd): return forced_result # we only take specific options from the forced "borg serve" command: result.restrict_to_paths = forced_result.restrict_to_paths + result.restrict_to_repositories = forced_result.restrict_to_repositories result.append_only = forced_result.append_only return result diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 0d36a27f13..c7def2c7cf 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -3552,10 +3552,22 @@ def test_get_args(): assert args.restrict_to_paths == ['/p1', '/p2'] assert args.umask == 0o027 assert args.log_level == 'info' + # similar, but with --restrict-to-repository + args = archiver.get_args(['borg', 'serve', '--restrict-to-repository=/r1', '--restrict-to-repository=/r2', ], + 'borg serve --info --umask=0027') + assert args.restrict_to_repositories == ['/r1', '/r2'] # trying to cheat - break out of path restriction args = archiver.get_args(['borg', 'serve', '--restrict-to-path=/p1', '--restrict-to-path=/p2', ], 'borg serve --restrict-to-path=/') assert args.restrict_to_paths == ['/p1', '/p2'] + # trying to cheat - break out of repository restriction + args = archiver.get_args(['borg', 'serve', '--restrict-to-repository=/r1', '--restrict-to-repository=/r2', ], + 'borg serve --restrict-to-repository=/') + assert args.restrict_to_repositories == ['/r1', '/r2'] + # trying to cheat - break below repository restriction + args = archiver.get_args(['borg', 'serve', '--restrict-to-repository=/r1', '--restrict-to-repository=/r2', ], + 'borg serve --restrict-to-repository=/r1/below') + assert args.restrict_to_repositories == ['/r1', '/r2'] # trying to cheat - try to execute different subcommand args = archiver.get_args(['borg', 'serve', '--restrict-to-path=/p1', '--restrict-to-path=/p2', ], 'borg init --encryption=repokey /') From 54883434b92aed5259e8babf9e71aa2bbeb494cc Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 26 Nov 2017 19:48:39 +0100 Subject: [PATCH 170/798] update CHANGES (1.1-maint) for 1.1.3 release --- docs/changes.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index e80a968bb2..2a2cc1a40e 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -131,8 +131,8 @@ The best check that everything is ok is to run a dry-run extraction:: Changelog ========= -Version 1.1.3 (not released yet) --------------------------------- +Version 1.1.3 (2017-11-27) +-------------------------- Compatibility notes: @@ -153,7 +153,11 @@ Compatibility notes: Fixes: -- XXX SECFIX XXX +- Security Fix for CVE-2017-15914: Incorrect implementation of access controls + allows remote users to override repository restrictions in Borg servers. + A user able to access a remote Borg SSH server is able to circumvent access + controls post-authentication. + Affected releases: 1.1.0, 1.1.1, 1.1.2. Releases 1.0.x are NOT affected. - crc32: deal with unaligned buffer, add tests - this broke borg on older ARM CPUs that can not deal with unaligned 32bit memory accesses and raise a bus error in such cases. the fix might also improve performance on some CPUs as From d373772517d70052720fe8f06ccad4666ff14d0b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 26 Nov 2017 20:16:44 +0100 Subject: [PATCH 171/798] build_usage --- docs/usage/benchmark_crud.rst.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage/benchmark_crud.rst.inc b/docs/usage/benchmark_crud.rst.inc index b76c091dc6..6e1d16dd01 100644 --- a/docs/usage/benchmark_crud.rst.inc +++ b/docs/usage/benchmark_crud.rst.inc @@ -70,7 +70,7 @@ C- == borg create (1st archive creation, no compression, do not use files cache) R- == borg extract (extract archive, dry-run, do everything, but do not write files to disk) R-Z- == all zero files. Measuring heavily duplicated files. R-R- == random files. No duplication here, measuring throughput through all processing - stages, except writing to disk. + stages, except writing to disk. U- == borg create (2nd archive creation of unchanged input files, measure files cache speed) The throughput value is kind of virtual here, it does not actually read the file. From 51504bad18b9d1ef5def07811c5137be28be303a Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 26 Nov 2017 20:16:01 +0100 Subject: [PATCH 172/798] build_man --- docs/man/borg-benchmark-crud.1 | 8 +- docs/man/borg-benchmark.1 | 2 +- docs/man/borg-break-lock.1 | 2 +- docs/man/borg-change-passphrase.1 | 2 +- docs/man/borg-check.1 | 2 +- docs/man/borg-common.1 | 2 +- docs/man/borg-compression.1 | 2 +- docs/man/borg-config.1 | 105 +++++++++++++++++++++++++ docs/man/borg-create.1 | 4 +- docs/man/borg-delete.1 | 2 +- docs/man/borg-diff.1 | 2 +- docs/man/borg-export-tar.1 | 8 +- docs/man/borg-extract.1 | 2 +- docs/man/borg-info.1 | 2 +- docs/man/borg-init.1 | 2 +- docs/man/borg-key-change-passphrase.1 | 2 +- docs/man/borg-key-export.1 | 2 +- docs/man/borg-key-import.1 | 2 +- docs/man/borg-key-migrate-to-repokey.1 | 2 +- docs/man/borg-key.1 | 2 +- docs/man/borg-list.1 | 2 +- docs/man/borg-mount.1 | 2 +- docs/man/borg-patterns.1 | 2 +- docs/man/borg-placeholders.1 | 2 +- docs/man/borg-prune.1 | 4 +- docs/man/borg-recreate.1 | 2 +- docs/man/borg-rename.1 | 2 +- docs/man/borg-serve.1 | 2 +- docs/man/borg-umount.1 | 7 +- docs/man/borg-upgrade.1 | 2 +- docs/man/borg-with-lock.1 | 2 +- docs/man/borg.1 | 22 ++++++ docs/man/borgfs.1 | 2 +- 33 files changed, 168 insertions(+), 42 deletions(-) create mode 100644 docs/man/borg-config.1 diff --git a/docs/man/borg-benchmark-crud.1 b/docs/man/borg-benchmark-crud.1 index 06c1b21a00..f30a93b2e9 100644 --- a/docs/man/borg-benchmark-crud.1 +++ b/docs/man/borg-benchmark-crud.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-BENCHMARK-CRUD 1 "2017-11-25" "" "borg backup tool" +.TH BORG-BENCHMARK-CRUD 1 "2017-11-26" "" "borg backup tool" .SH NAME borg-benchmark-crud \- Benchmark Create, Read, Update, Delete for archives. . @@ -60,13 +60,7 @@ C\-R\- == random files. no dedup, measuring throughput through all processing st .B R\- == borg extract (extract archive, dry\-run, do everything, but do not write files to disk) R\-Z\- == all zero files. Measuring heavily duplicated files. R\-R\- == random files. No duplication here, measuring throughput through all processing -.IP "System Message: ERROR/3 (docs/virtmanpage.rst:, line 56)" -Unexpected indentation. -.INDENT 7.0 -.INDENT 3.5 stages, except writing to disk. -.UNINDENT -.UNINDENT .TP .B U\- == borg create (2nd archive creation of unchanged input files, measure files cache speed) The throughput value is kind of virtual here, it does not actually read the file. diff --git a/docs/man/borg-benchmark.1 b/docs/man/borg-benchmark.1 index c6d44204f0..ec71b4fac0 100644 --- a/docs/man/borg-benchmark.1 +++ b/docs/man/borg-benchmark.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-BENCHMARK 1 "2017-11-25" "" "borg backup tool" +.TH BORG-BENCHMARK 1 "2017-11-26" "" "borg backup tool" .SH NAME borg-benchmark \- benchmark command . diff --git a/docs/man/borg-break-lock.1 b/docs/man/borg-break-lock.1 index a6c93a2cab..138cce29ad 100644 --- a/docs/man/borg-break-lock.1 +++ b/docs/man/borg-break-lock.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-BREAK-LOCK 1 "2017-11-25" "" "borg backup tool" +.TH BORG-BREAK-LOCK 1 "2017-11-26" "" "borg backup tool" .SH NAME borg-break-lock \- Break the repository lock (e.g. in case it was left by a dead borg. . diff --git a/docs/man/borg-change-passphrase.1 b/docs/man/borg-change-passphrase.1 index c56f773a81..aee189f37f 100644 --- a/docs/man/borg-change-passphrase.1 +++ b/docs/man/borg-change-passphrase.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-CHANGE-PASSPHRASE 1 "2017-11-25" "" "borg backup tool" +.TH BORG-CHANGE-PASSPHRASE 1 "2017-11-26" "" "borg backup tool" .SH NAME borg-change-passphrase \- Change repository key file passphrase . diff --git a/docs/man/borg-check.1 b/docs/man/borg-check.1 index 5a8660a275..7ae6964d3c 100644 --- a/docs/man/borg-check.1 +++ b/docs/man/borg-check.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-CHECK 1 "2017-11-25" "" "borg backup tool" +.TH BORG-CHECK 1 "2017-11-26" "" "borg backup tool" .SH NAME borg-check \- Check repository consistency . diff --git a/docs/man/borg-common.1 b/docs/man/borg-common.1 index 2299016084..2f78e8bb22 100644 --- a/docs/man/borg-common.1 +++ b/docs/man/borg-common.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-COMMON 1 "2017-11-25" "" "borg backup tool" +.TH BORG-COMMON 1 "2017-11-26" "" "borg backup tool" .SH NAME borg-common \- Common options of Borg commands . diff --git a/docs/man/borg-compression.1 b/docs/man/borg-compression.1 index f4ee1bf8e7..c6aa534e47 100644 --- a/docs/man/borg-compression.1 +++ b/docs/man/borg-compression.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-COMPRESSION 1 "2017-11-25" "" "borg backup tool" +.TH BORG-COMPRESSION 1 "2017-11-26" "" "borg backup tool" .SH NAME borg-compression \- Details regarding compression . diff --git a/docs/man/borg-config.1 b/docs/man/borg-config.1 new file mode 100644 index 0000000000..b136ec6c31 --- /dev/null +++ b/docs/man/borg-config.1 @@ -0,0 +1,105 @@ +.\" Man page generated from reStructuredText. +. +.TH BORG-CONFIG 1 "2017-11-26" "" "borg backup tool" +.SH NAME +borg-config \- get, set, and delete values in a repository or cache config file +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.sp +borg [common options] config [options] REPOSITORY NAME [VALUE] +.SH DESCRIPTION +.sp +This command gets and sets options in a local repository or cache config file. +For security reasons, this command only works on local repositories. +.sp +To delete a config value entirely, use \fB\-\-delete\fP\&. To get an existing key, pass +only the key name. To set a key, pass both the key name and the new value. Keys +can be specified in the format "section.name" or simply "name"; the section will +default to "repository" and "cache" for the repo and cache configs, respectively. +.sp +By default, borg config manipulates the repository config file. Using \fB\-\-cache\fP +edits the repository cache\(aqs config file instead. +.SH OPTIONS +.sp +See \fIborg\-common(1)\fP for common options of Borg commands. +.SS arguments +.INDENT 0.0 +.TP +.B REPOSITORY +repository to configure +.TP +.B NAME +name of config key +.TP +.B VALUE +new value for key +.UNINDENT +.SS optional arguments +.INDENT 0.0 +.TP +.B \-c\fP,\fB \-\-cache +get and set values from the repo cache +.TP +.B \-d\fP,\fB \-\-delete +delete the key from the config file +.UNINDENT +.SH EXAMPLES +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +The repository & cache config files are some of the only directly manipulable +parts of a repository that aren\(aqt versioned or backed up, so be careful when +making changes! +.UNINDENT +.UNINDENT +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +# find cache directory +$ cd ~/.cache/borg/$(borg config /path/to/repo id) + +# reserve some space +$ borg config /path/to/repo additional_free_space 2G + +# make a repo append\-only +$ borg config /path/to/repo append_only 1 +.ft P +.fi +.UNINDENT +.UNINDENT +.SH SEE ALSO +.sp +\fIborg\-common(1)\fP +.SH AUTHOR +The Borg Collective +.\" Generated by docutils manpage writer. +. diff --git a/docs/man/borg-create.1 b/docs/man/borg-create.1 index 71b766aed3..c50666ea6d 100644 --- a/docs/man/borg-create.1 +++ b/docs/man/borg-create.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-CREATE 1 "2017-11-25" "" "borg backup tool" +.TH BORG-CREATE 1 "2017-11-26" "" "borg backup tool" .SH NAME borg-create \- Create new archive . @@ -252,7 +252,7 @@ $ borg create /path/to/repo::my\-files /home \e # Backup the root filesystem into an archive named "root\-YYYY\-MM\-DD" # use zlib compression (good, but slow) \- default is lz4 (fast, low compression ratio) -$ borg create \-C zlib,6 /path/to/repo::root\-{now:%Y\-%m\-%d} / \-\-one\-file\-system +$ borg create \-C zlib,6 \-\-one\-file\-system /path/to/repo::root\-{now:%Y\-%m\-%d} / # Backup a remote host locally ("pull" style) using sshfs $ mkdir sshfs\-mount diff --git a/docs/man/borg-delete.1 b/docs/man/borg-delete.1 index 2b0e16fc7c..3c73d68ffd 100644 --- a/docs/man/borg-delete.1 +++ b/docs/man/borg-delete.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-DELETE 1 "2017-11-25" "" "borg backup tool" +.TH BORG-DELETE 1 "2017-11-26" "" "borg backup tool" .SH NAME borg-delete \- Delete an existing repository or archives . diff --git a/docs/man/borg-diff.1 b/docs/man/borg-diff.1 index 77697f190b..45e3856e5d 100644 --- a/docs/man/borg-diff.1 +++ b/docs/man/borg-diff.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-DIFF 1 "2017-11-25" "" "borg backup tool" +.TH BORG-DIFF 1 "2017-11-26" "" "borg backup tool" .SH NAME borg-diff \- Diff contents of two archives . diff --git a/docs/man/borg-export-tar.1 b/docs/man/borg-export-tar.1 index 770d2bafed..4a59ce80e8 100644 --- a/docs/man/borg-export-tar.1 +++ b/docs/man/borg-export-tar.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-EXPORT-TAR 1 "2017-11-25" "" "borg backup tool" +.TH BORG-EXPORT-TAR 1 "2017-11-26" "" "borg backup tool" .SH NAME borg-export-tar \- Export archive contents as a tarball . @@ -127,11 +127,11 @@ $ borg export\-tar /path/to/repo::Monday Monday.tar $ borg export\-tar /path/to/repo::Monday Monday.tar.gz \-\-exclude \(aq*.so\(aq # use higher compression level with gzip -$ borg export\-tar testrepo::linux \-\-tar\-filter="gzip \-9" Monday.tar.gz +$ borg export\-tar \-\-tar\-filter="gzip \-9" testrepo::linux Monday.tar.gz -# export a gzipped tar, but instead of storing it on disk, +# export a tar, but instead of storing it on disk, # upload it to a remote site using curl. -$ borg export\-tar ... \-\-tar\-filter="gzip" \- | curl \-\-data\-binary @\- https://somewhere/to/POST +$ borg export\-tar /path/to/repo::Monday \- | curl \-\-data\-binary @\- https://somewhere/to/POST # remote extraction via "tarpipe" $ borg export\-tar /path/to/repo::Monday \- | ssh somewhere "cd extracted; tar x" diff --git a/docs/man/borg-extract.1 b/docs/man/borg-extract.1 index b11d737b31..823999bdd1 100644 --- a/docs/man/borg-extract.1 +++ b/docs/man/borg-extract.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-EXTRACT 1 "2017-11-25" "" "borg backup tool" +.TH BORG-EXTRACT 1 "2017-11-26" "" "borg backup tool" .SH NAME borg-extract \- Extract archive contents . diff --git a/docs/man/borg-info.1 b/docs/man/borg-info.1 index 726fed8266..412060464b 100644 --- a/docs/man/borg-info.1 +++ b/docs/man/borg-info.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-INFO 1 "2017-11-25" "" "borg backup tool" +.TH BORG-INFO 1 "2017-11-26" "" "borg backup tool" .SH NAME borg-info \- Show archive details such as disk space used . diff --git a/docs/man/borg-init.1 b/docs/man/borg-init.1 index 9c9c2e62d9..66ec6e46e5 100644 --- a/docs/man/borg-init.1 +++ b/docs/man/borg-init.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-INIT 1 "2017-11-25" "" "borg backup tool" +.TH BORG-INIT 1 "2017-11-26" "" "borg backup tool" .SH NAME borg-init \- Initialize an empty repository . diff --git a/docs/man/borg-key-change-passphrase.1 b/docs/man/borg-key-change-passphrase.1 index a94a067273..80788292a1 100644 --- a/docs/man/borg-key-change-passphrase.1 +++ b/docs/man/borg-key-change-passphrase.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-CHANGE-PASSPHRASE 1 "2017-11-25" "" "borg backup tool" +.TH BORG-KEY-CHANGE-PASSPHRASE 1 "2017-11-26" "" "borg backup tool" .SH NAME borg-key-change-passphrase \- Change repository key file passphrase . diff --git a/docs/man/borg-key-export.1 b/docs/man/borg-key-export.1 index a47f56cbe2..365a4f2d87 100644 --- a/docs/man/borg-key-export.1 +++ b/docs/man/borg-key-export.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-EXPORT 1 "2017-11-25" "" "borg backup tool" +.TH BORG-KEY-EXPORT 1 "2017-11-26" "" "borg backup tool" .SH NAME borg-key-export \- Export the repository key for backup . diff --git a/docs/man/borg-key-import.1 b/docs/man/borg-key-import.1 index 34b6b486b3..18375e877c 100644 --- a/docs/man/borg-key-import.1 +++ b/docs/man/borg-key-import.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-IMPORT 1 "2017-11-25" "" "borg backup tool" +.TH BORG-KEY-IMPORT 1 "2017-11-26" "" "borg backup tool" .SH NAME borg-key-import \- Import the repository key from backup . diff --git a/docs/man/borg-key-migrate-to-repokey.1 b/docs/man/borg-key-migrate-to-repokey.1 index 03f54d0bd8..18090122e5 100644 --- a/docs/man/borg-key-migrate-to-repokey.1 +++ b/docs/man/borg-key-migrate-to-repokey.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY-MIGRATE-TO-REPOKEY 1 "2017-11-25" "" "borg backup tool" +.TH BORG-KEY-MIGRATE-TO-REPOKEY 1 "2017-11-26" "" "borg backup tool" .SH NAME borg-key-migrate-to-repokey \- Migrate passphrase -> repokey . diff --git a/docs/man/borg-key.1 b/docs/man/borg-key.1 index 88f54f1783..bc550e8bd0 100644 --- a/docs/man/borg-key.1 +++ b/docs/man/borg-key.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-KEY 1 "2017-11-25" "" "borg backup tool" +.TH BORG-KEY 1 "2017-11-26" "" "borg backup tool" .SH NAME borg-key \- Manage a keyfile or repokey of a repository . diff --git a/docs/man/borg-list.1 b/docs/man/borg-list.1 index c47e436f93..cf38d7670e 100644 --- a/docs/man/borg-list.1 +++ b/docs/man/borg-list.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-LIST 1 "2017-11-25" "" "borg backup tool" +.TH BORG-LIST 1 "2017-11-26" "" "borg backup tool" .SH NAME borg-list \- List archive or repository contents . diff --git a/docs/man/borg-mount.1 b/docs/man/borg-mount.1 index 6c4ac59546..8f97be5b94 100644 --- a/docs/man/borg-mount.1 +++ b/docs/man/borg-mount.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-MOUNT 1 "2017-11-25" "" "borg backup tool" +.TH BORG-MOUNT 1 "2017-11-26" "" "borg backup tool" .SH NAME borg-mount \- Mount archive or an entire repository as a FUSE filesystem . diff --git a/docs/man/borg-patterns.1 b/docs/man/borg-patterns.1 index 82b347d055..e0d91c407e 100644 --- a/docs/man/borg-patterns.1 +++ b/docs/man/borg-patterns.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-PATTERNS 1 "2017-11-25" "" "borg backup tool" +.TH BORG-PATTERNS 1 "2017-11-26" "" "borg backup tool" .SH NAME borg-patterns \- Details regarding patterns . diff --git a/docs/man/borg-placeholders.1 b/docs/man/borg-placeholders.1 index 9f50120155..3e61b79228 100644 --- a/docs/man/borg-placeholders.1 +++ b/docs/man/borg-placeholders.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-PLACEHOLDERS 1 "2017-11-25" "" "borg backup tool" +.TH BORG-PLACEHOLDERS 1 "2017-11-26" "" "borg backup tool" .SH NAME borg-placeholders \- Details regarding placeholders . diff --git a/docs/man/borg-prune.1 b/docs/man/borg-prune.1 index 51fab8918f..48016872c2 100644 --- a/docs/man/borg-prune.1 +++ b/docs/man/borg-prune.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-PRUNE 1 "2017-11-25" "" "borg backup tool" +.TH BORG-PRUNE 1 "2017-11-26" "" "borg backup tool" .SH NAME borg-prune \- Prune repository archives according to specified rules . @@ -176,7 +176,7 @@ $ borg prune \-v \-\-list \-\-keep\-within=10d \-\-keep\-weekly=4 \-\-keep\-mont .UNINDENT .sp There is also a visualized prune example in \fBdocs/misc/prune\-example.txt\fP: -.IP "System Message: ERROR/3 (docs/virtmanpage.rst:, line 145)" +.IP "System Message: ERROR/3 (docs/borg-prune.rst:, line 145)" Unknown directive type "highlight". .INDENT 0.0 .INDENT 3.5 diff --git a/docs/man/borg-recreate.1 b/docs/man/borg-recreate.1 index 8c478ceb1b..caa377aeb4 100644 --- a/docs/man/borg-recreate.1 +++ b/docs/man/borg-recreate.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-RECREATE 1 "2017-11-25" "" "borg backup tool" +.TH BORG-RECREATE 1 "2017-11-26" "" "borg backup tool" .SH NAME borg-recreate \- Re-create archives . diff --git a/docs/man/borg-rename.1 b/docs/man/borg-rename.1 index d82942e7f8..285fe681cd 100644 --- a/docs/man/borg-rename.1 +++ b/docs/man/borg-rename.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-RENAME 1 "2017-11-25" "" "borg backup tool" +.TH BORG-RENAME 1 "2017-11-26" "" "borg backup tool" .SH NAME borg-rename \- Rename an existing archive . diff --git a/docs/man/borg-serve.1 b/docs/man/borg-serve.1 index 7af07f484d..3b2c442fdc 100644 --- a/docs/man/borg-serve.1 +++ b/docs/man/borg-serve.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-SERVE 1 "2017-11-25" "" "borg backup tool" +.TH BORG-SERVE 1 "2017-11-26" "" "borg backup tool" .SH NAME borg-serve \- Start in server mode. This command is usually not used manually. . diff --git a/docs/man/borg-umount.1 b/docs/man/borg-umount.1 index 14534a513c..c9f1d61243 100644 --- a/docs/man/borg-umount.1 +++ b/docs/man/borg-umount.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-UMOUNT 1 "2017-11-25" "" "borg backup tool" +.TH BORG-UMOUNT 1 "2017-11-26" "" "borg backup tool" .SH NAME borg-umount \- un-mount the FUSE filesystem . @@ -82,6 +82,11 @@ $ borg umount /tmp/mymountpoint # These are especially handy for the "versions view", # which does not support lazy processing of archives. $ borg mount \-o versions \-\-glob\-archives \(aq*\-my\-home\(aq \-\-last 10 /path/to/repo /tmp/mymountpoint + +# Exclusion options are supported. +# These can speed up mounting and lower memory needs significantly. +$ borg mount /path/to/repo /tmp/mymountpoint only/that/path +$ borg mount \-\-exclude \(aq...\(aq /path/to/repo /tmp/mymountpoint .ft P .fi .UNINDENT diff --git a/docs/man/borg-upgrade.1 b/docs/man/borg-upgrade.1 index 74ddb96016..0b191045c0 100644 --- a/docs/man/borg-upgrade.1 +++ b/docs/man/borg-upgrade.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-UPGRADE 1 "2017-11-25" "" "borg backup tool" +.TH BORG-UPGRADE 1 "2017-11-26" "" "borg backup tool" .SH NAME borg-upgrade \- upgrade a repository from a previous version . diff --git a/docs/man/borg-with-lock.1 b/docs/man/borg-with-lock.1 index 0f2169ec68..7b662c9b6e 100644 --- a/docs/man/borg-with-lock.1 +++ b/docs/man/borg-with-lock.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORG-WITH-LOCK 1 "2017-11-25" "" "borg backup tool" +.TH BORG-WITH-LOCK 1 "2017-11-26" "" "borg backup tool" .SH NAME borg-with-lock \- run a user specified command with the repository lock held . diff --git a/docs/man/borg.1 b/docs/man/borg.1 index 703bdc4b5d..36ac4b3c53 100644 --- a/docs/man/borg.1 +++ b/docs/man/borg.1 @@ -186,6 +186,26 @@ get other informational messages. .UNINDENT .UNINDENT .SH NOTES +.SS Positional Arguments and Options: Order matters +.sp +Borg only supports taking options (\fB\-s\fP and \fB\-\-progress\fP in the example) +to the left or right of all positional arguments (\fBrepo::archive\fP and \fBpath\fP +in the example), but not in between them: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +borg create \-s \-\-progress repo::archive path # good and preferred +borg create repo::archive path \-s \-\-progress # also works +borg create \-s repo::archive path \-\-progress # works, but ugly +borg create repo::archive \-s \-\-progress path # BAD +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +This is due to a problem in the argparse module: \fI\%http://bugs.python.org/issue15112\fP .SS Repository URLs .sp \fBLocal filesystem\fP (or locally mounted network filesystem): @@ -614,6 +634,8 @@ hardlinked regular files, devices, FIFOs (considering all items in the same arch .IP \(bu 2 timestamps in nanosecond precision: mtime, atime, ctime .IP \(bu 2 +other timestamps: birthtime (on platforms supporting it) +.IP \(bu 2 permissions: .INDENT 2.0 .IP \(bu 2 diff --git a/docs/man/borgfs.1 b/docs/man/borgfs.1 index bac62ff5ce..2c5f9357ed 100644 --- a/docs/man/borgfs.1 +++ b/docs/man/borgfs.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH BORGFS 1 "2017-11-25" "" "borg backup tool" +.TH BORGFS 1 "2017-11-26" "" "borg backup tool" .SH NAME borgfs \- Mount archive or an entire repository as a FUSE filesystem . From 6becbb259b75f87e550166bf6636dddc3801ab4c Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 26 Nov 2017 20:22:00 +0100 Subject: [PATCH 173/798] manually fix build_man issue with highlight directive --- docs/man/borg-prune.1 | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/docs/man/borg-prune.1 b/docs/man/borg-prune.1 index 48016872c2..e051fa3a2e 100644 --- a/docs/man/borg-prune.1 +++ b/docs/man/borg-prune.1 @@ -175,20 +175,7 @@ $ borg prune \-v \-\-list \-\-keep\-within=10d \-\-keep\-weekly=4 \-\-keep\-mont .UNINDENT .UNINDENT .sp -There is also a visualized prune example in \fBdocs/misc/prune\-example.txt\fP: -.IP "System Message: ERROR/3 (docs/borg-prune.rst:, line 145)" -Unknown directive type "highlight". -.INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -\&.. highlight:: none - -.ft P -.fi -.UNINDENT -.UNINDENT +There is also a visualized prune example in \fBdocs/misc/prune\-example.txt\fP. .SH SEE ALSO .sp \fIborg\-common(1)\fP From 831a06a07d93626b33b094f75c63ae017c09974f Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 26 Nov 2017 22:29:41 +0100 Subject: [PATCH 174/798] move 3rd party docs/license file from package dir to docs/3rd_party setup.py excludes all .c .h .pyx files from installation, but such docs / license files would still be copied to the target directory if they are inside the python "borg" package dir. --- docs/3rd_party/README | 5 +++++ {src/borg/algorithms => docs/3rd_party}/blake2/COPYING | 0 {src/borg/algorithms => docs/3rd_party}/blake2/README.md | 0 3 files changed, 5 insertions(+) create mode 100644 docs/3rd_party/README rename {src/borg/algorithms => docs/3rd_party}/blake2/COPYING (100%) rename {src/borg/algorithms => docs/3rd_party}/blake2/README.md (100%) diff --git a/docs/3rd_party/README b/docs/3rd_party/README new file mode 100644 index 0000000000..1f7b9b6ad9 --- /dev/null +++ b/docs/3rd_party/README @@ -0,0 +1,5 @@ +Here we store 3rd party documentation, licenses, etc. + +Please note that all files inside the "borg" package directory (except the +stuff excluded in setup.py) will be INSTALLED, so don't keep docs or licenses +there. diff --git a/src/borg/algorithms/blake2/COPYING b/docs/3rd_party/blake2/COPYING similarity index 100% rename from src/borg/algorithms/blake2/COPYING rename to docs/3rd_party/blake2/COPYING diff --git a/src/borg/algorithms/blake2/README.md b/docs/3rd_party/blake2/README.md similarity index 100% rename from src/borg/algorithms/blake2/README.md rename to docs/3rd_party/blake2/README.md From 23a1d62b2580f9462c02524eb23af99c04ca65d0 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 27 Nov 2017 01:01:34 +0100 Subject: [PATCH 175/798] crc32-slice-by-8.c: fix for C90 compilers crc32_slice_by_8.c:344:3: error: ISO C90 forbids mixed declarations and code [-Werror=declaration-after-statement] --- src/borg/algorithms/crc32_slice_by_8.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/borg/algorithms/crc32_slice_by_8.c b/src/borg/algorithms/crc32_slice_by_8.c index b289fbb87c..3ab0c84090 100644 --- a/src/borg/algorithms/crc32_slice_by_8.c +++ b/src/borg/algorithms/crc32_slice_by_8.c @@ -332,14 +332,12 @@ uint32_t crc32_slice_by_8(const void* data, size_t length, uint32_t previousCrc3 uint32_t crc = ~previousCrc32; // same as previousCrc32 ^ 0xFFFFFFFF const uint32_t* current; - const uint8_t* currentChar; + const uint8_t* currentChar = (const uint8_t*) data; // enabling optimization (at least -O2) automatically unrolls the inner for-loop const size_t Unroll = 4; const size_t BytesAtOnce = 8 * Unroll; - currentChar = (const uint8_t*) data; - // wanted: 32 bit / 4 Byte alignment, compute leading, unaligned bytes length uintptr_t unaligned_length = (4 - (((uintptr_t) currentChar) & 3)) & 3; // process unaligned bytes, if any (standard algorithm) From dc8de361095eea509a28d416ab826850047432ce Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 27 Nov 2017 04:10:17 +0100 Subject: [PATCH 176/798] borg mount: fix hardlink processing, fixes #3388 when the result list after stripping was empty, os.path.join(*emptylist) fails as it want 1+ args. --- src/borg/fuse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/borg/fuse.py b/src/borg/fuse.py index 399f7d0843..48ec6e4978 100644 --- a/src/borg/fuse.py +++ b/src/borg/fuse.py @@ -398,7 +398,7 @@ def make_versioned_name(name, version, add_dir=False): return name + version_enc + ext if 'source' in item and hardlinkable(item.mode): - source = os.path.join(*item.source.split(os.sep)[stripped_components:]) + source = os.sep.join(item.source.split(os.sep)[stripped_components:]) chunks, link_target = hardlink_masters.get(item.source, (None, source)) if link_target: # Hard link was extracted previously, just link From 953f3a4a2f42a0b9d2629b60a65c5607ea403649 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 2 Dec 2017 17:43:38 +0100 Subject: [PATCH 177/798] exclude broken pytest 3.3.0 release https://github.com/pytest-dev/pytest/issues/2957 (cherry picked from commit 43184f2dd12a33b4ee462918e8c1a5eb07355f88) --- requirements.d/development.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.d/development.txt b/requirements.d/development.txt index 88535435ae..23f6f39e5c 100644 --- a/requirements.d/development.txt +++ b/requirements.d/development.txt @@ -3,7 +3,7 @@ setuptools_scm pip virtualenv tox -pytest +pytest!=3.3.0 pytest-xdist pytest-cov pytest-benchmark From 25bfb5b00d826e82826c23187d36bb5cae4b8d8e Mon Sep 17 00:00:00 2001 From: Milkey Mouse Date: Fri, 1 Dec 2017 14:21:20 -0800 Subject: [PATCH 178/798] Don't generate HTML docs page for borgfs (fixes #3404) --- setup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 7140ac161b..891138ebe2 100644 --- a/setup.py +++ b/setup.py @@ -238,9 +238,11 @@ def run(self): # allows us to build docs without the C modules fully loaded during help generation from borg.archiver import Archiver parser = Archiver(prog='borg').build_parser() - borgfs_parser = Archiver(prog='borgfs').build_parser() + # borgfs has a separate man page to satisfy debian's "every program from a package + # must have a man page" requirement, but it doesn't need a separate HTML docs page + #borgfs_parser = Archiver(prog='borgfs').build_parser() - self.generate_level("", parser, Archiver, {'borgfs': borgfs_parser}) + self.generate_level("", parser, Archiver) def generate_level(self, prefix, parser, Archiver, extra_choices=None): is_subcommand = False From 278ed640e909a50641a8e410d414dd4de9ba8879 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 9 Dec 2017 13:41:03 +0100 Subject: [PATCH 179/798] add auto-generated docs for borg config also: move a bit upwards in the use docs --- docs/usage.rst | 2 +- docs/usage/config.rst.inc | 73 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 docs/usage/config.rst.inc diff --git a/docs/usage.rst b/docs/usage.rst index f8e2b48bfe..9c0adfdb5f 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -49,9 +49,9 @@ Usage usage/recreate usage/tar usage/serve + usage/config usage/lock usage/benchmark - usage/config usage/help usage/debug diff --git a/docs/usage/config.rst.inc b/docs/usage/config.rst.inc new file mode 100644 index 0000000000..2a64c1276a --- /dev/null +++ b/docs/usage/config.rst.inc @@ -0,0 +1,73 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + +.. _borg_config: + +borg config +----------- +.. code-block:: none + + borg [common options] config [options] REPOSITORY NAME [VALUE] + +.. only:: html + + .. class:: borg-options-table + + +-------------------------------------------------------+----------------------+----------------------------------------+ + | **positional arguments** | + +-------------------------------------------------------+----------------------+----------------------------------------+ + | | ``REPOSITORY`` | repository to configure | + +-------------------------------------------------------+----------------------+----------------------------------------+ + | | ``NAME`` | name of config key | + +-------------------------------------------------------+----------------------+----------------------------------------+ + | | ``VALUE`` | new value for key | + +-------------------------------------------------------+----------------------+----------------------------------------+ + | **optional arguments** | + +-------------------------------------------------------+----------------------+----------------------------------------+ + | | ``-c``, ``--cache`` | get and set values from the repo cache | + +-------------------------------------------------------+----------------------+----------------------------------------+ + | | ``-d``, ``--delete`` | delete the key from the config file | + +-------------------------------------------------------+----------------------+----------------------------------------+ + | .. class:: borg-common-opt-ref | + | | + | :ref:`common_options` | + +-------------------------------------------------------+----------------------+----------------------------------------+ + + .. raw:: html + + + +.. only:: latex + + REPOSITORY + repository to configure + NAME + name of config key + VALUE + new value for key + + + optional arguments + -c, --cache get and set values from the repo cache + -d, --delete delete the key from the config file + + + :ref:`common_options` + | + +Description +~~~~~~~~~~~ + +This command gets and sets options in a local repository or cache config file. +For security reasons, this command only works on local repositories. + +To delete a config value entirely, use ``--delete``. To get an existing key, pass +only the key name. To set a key, pass both the key name and the new value. Keys +can be specified in the format "section.name" or simply "name"; the section will +default to "repository" and "cache" for the repo and cache configs, respectively. + +By default, borg config manipulates the repository config file. Using ``--cache`` +edits the repository cache's config file instead. \ No newline at end of file From 747ada77e639fcb5fa12a0378080ff9561a28a24 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 13 Dec 2017 04:33:50 +0100 Subject: [PATCH 180/798] fix building the "borg prune" man page, fixes #3398 will fix all ".. highlight:: none" issues (just killing it). also a slight punctuation fix, so there is a dot at the end of the prune man page. (cherry picked from commit 1f5b8c9219eb5e0d8c4799e8e3c984c6073d6f68) --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 891138ebe2..1cd2bd0061 100644 --- a/setup.py +++ b/setup.py @@ -651,6 +651,8 @@ def write_examples(self, write, command): examples = examples.replace(usage_include, '') examples = examples.replace('Examples\n~~~~~~~~', '') examples = examples.replace('Miscellaneous Help\n------------------', '') + examples = examples.replace('``docs/misc/prune-example.txt``:', '``docs/misc/prune-example.txt``.') + examples = examples.replace('.. highlight:: none\n', '') # we don't support highlight examples = re.sub('^(~+)$', lambda matches: '+' * len(matches.group(0)), examples, flags=re.MULTILINE) examples = examples.strip() if examples: From 981a936f47473e8c3a16595264a1be0b064fad12 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 13 Dec 2017 04:01:59 +0100 Subject: [PATCH 181/798] add parens for C preprocessor macro argument usages this is needed for correctness because the preprocessor is just doing text replacement. This is the correct way: #define MUL(x, y) ((x) * (y)) MUL(1+2, 3-4) -> ((1+2) * (3-4)) # not: (1+2 * 3-4) I didn't put parens around all arg usages for readability. Some stuff (like index) is not expected to be an expression. Also, when a arg is only used in another macro or function call, no parens are needed either. I reviewed the code: no harm was done (yet) due to this fault. Thanks to @rciorba who found this. (cherry picked from commit a3cecf599f42173b2fc485407aa439a9c6365d9c) --- src/borg/_chunker.c | 2 +- src/borg/_hashindex.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/borg/_chunker.c b/src/borg/_chunker.c index b72f6bd8b1..4add32ac3d 100644 --- a/src/borg/_chunker.c +++ b/src/borg/_chunker.c @@ -63,7 +63,7 @@ static uint32_t table_base[] = 0xc5ae37bb, 0xa76ce12a, 0x8150d8f3, 0x2ec29218, 0xa35f0984, 0x48c0647e, 0x0b5ff98c, 0x71893f7b }; -#define BARREL_SHIFT(v, shift) ( ((v) << shift) | ((v) >> ((32 - shift) & 0x1f)) ) +#define BARREL_SHIFT(v, shift) ( ((v) << (shift)) | ((v) >> ((32 - (shift)) & 0x1f)) ) size_t pagemask; diff --git a/src/borg/_hashindex.c b/src/borg/_hashindex.c index 9fbd1ebda1..c7a5b0d0f7 100644 --- a/src/borg/_hashindex.c +++ b/src/borg/_hashindex.c @@ -71,7 +71,7 @@ static int hash_sizes[] = { #define EMPTY _htole32(0xffffffff) #define DELETED _htole32(0xfffffffe) -#define BUCKET_ADDR(index, idx) (index->buckets + (idx * index->bucket_size)) +#define BUCKET_ADDR(index, idx) (index->buckets + ((idx) * index->bucket_size)) #define BUCKET_MATCHES_KEY(index, idx, key) (memcmp(key, BUCKET_ADDR(index, idx), index->key_size) == 0) From dbf8d582f92e4bdcb1904138ff28d0d223606ef5 Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Mon, 11 Dec 2017 23:00:38 +0100 Subject: [PATCH 182/798] Add placeholder for fqdn in reverse notation (cherry picked from commit 85fb38e2f3d7c4df581f2aa58605a507003db492) --- docs/man/borg-placeholders.1 | 3 +++ docs/usage/help.rst.inc | 3 +++ src/borg/archiver.py | 3 +++ src/borg/helpers.py | 1 + 4 files changed, 10 insertions(+) diff --git a/docs/man/borg-placeholders.1 b/docs/man/borg-placeholders.1 index 3e61b79228..ba7a631a71 100644 --- a/docs/man/borg-placeholders.1 +++ b/docs/man/borg-placeholders.1 @@ -42,6 +42,9 @@ The (short) hostname of the machine. .B {fqdn} The full name of the machine. .TP +.B {reverse-fqdn} +The full name of the machine in reverse domain name notation. +.TP .B {now} The current local date and time, by default in ISO\-8601 format. You can also supply your own \fI\%format string\fP, e.g. {now:%Y\-%m\-%d_%H:%M:%S} diff --git a/docs/usage/help.rst.inc b/docs/usage/help.rst.inc index c5fb6a71e5..bed681768f 100644 --- a/docs/usage/help.rst.inc +++ b/docs/usage/help.rst.inc @@ -169,6 +169,9 @@ placeholders: {fqdn} The full name of the machine. +{reverse-fqdn} + The full name of the machine in reverse domain name notation. + {now} The current local date and time, by default in ISO-8601 format. You can also supply your own `format string `_, e.g. {now:%Y-%m-%d_%H:%M:%S} diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 9bc5db614b..888e66025d 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -2093,6 +2093,9 @@ def do_break_lock(self, args, repository): {fqdn} The full name of the machine. + {reverse-fqdn} + The full name of the machine in reverse domain name notation. + {now} The current local date and time, by default in ISO-8601 format. You can also supply your own `format string `_, e.g. {now:%Y-%m-%d_%H:%M:%S} diff --git a/src/borg/helpers.py b/src/borg/helpers.py index e5f9de5ad9..5008f0188c 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -664,6 +664,7 @@ def replace_placeholders(text): data = { 'pid': os.getpid(), 'fqdn': socket.getfqdn(), + 'reverse-fqdn': '.'.join(reversed(socket.getfqdn().split('.'))), 'hostname': socket.gethostname(), 'now': DatetimeWrapper(current_time.now()), 'utcnow': DatetimeWrapper(current_time.utcnow()), From 96675e06c8c0acef9c67eddb912a5cd32524881c Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Tue, 12 Dec 2017 12:44:17 +0100 Subject: [PATCH 183/798] Refactor: call getfqdn() once per call of replace_placeholders() (cherry picked from commit 294f06b565885f2e78d64fa6c26c90fd8d8c3431) --- src/borg/helpers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 5008f0188c..cb3e10cac2 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -661,10 +661,11 @@ def format_line(format, data): def replace_placeholders(text): """Replace placeholders in text with their values.""" current_time = datetime.now() + fqdn = socket.getfqdn() data = { 'pid': os.getpid(), - 'fqdn': socket.getfqdn(), - 'reverse-fqdn': '.'.join(reversed(socket.getfqdn().split('.'))), + 'fqdn': fqdn, + 'reverse-fqdn': '.'.join(reversed(fqdn.split('.'))), 'hostname': socket.gethostname(), 'now': DatetimeWrapper(current_time.now()), 'utcnow': DatetimeWrapper(current_time.utcnow()), From 00865ae89e075cd4844b3c191407987688bea42b Mon Sep 17 00:00:00 2001 From: Nils Steinger Date: Fri, 6 Oct 2017 02:09:17 +0200 Subject: [PATCH 184/798] List help topics when invalid topic is requested (cherry picked from commit 9e81a7617216e16adaefad83fb58e22af9ddc9f8) --- src/borg/archiver.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 888e66025d..b2db6cc983 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -2205,7 +2205,12 @@ def do_help(self, parser, commands, args): else: commands[args.topic].print_help() else: - parser.error('No help available on %s' % (args.topic,)) + msg_lines = [] + msg_lines += ['No help available on %s.' % args.topic] + msg_lines += ['Try one of the following:'] + msg_lines += [' Commands: %s' % ', '.join(sorted(commands.keys()))] + msg_lines += [' Topics: %s' % ', '.join(sorted(self.helptext.keys()))] + parser.error('\n'.join(msg_lines)) return self.exit_code def do_subcommand_help(self, parser, args): From 6bad2395ddde38232c7669165105d47979632198 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 2 Dec 2017 22:43:34 +0100 Subject: [PATCH 185/798] add zstd compression based on willyvmm's work in PR #3116, but some changes: - removed any mulithreading changes - add zstandard in setup.py install_requires - tests - fix: minimum compression level is 1 (not 0) - use 3 for the default compression level - use ID 03 00 for zstd - only convert to bytes if we don't have bytes yet - move zstd code so that code blocks are ordered by ID - other cosmetic fixes (cherry picked from commit 11b2311e6ebc43546cef88a39bce744d67940c5a) --- setup.py | 2 +- src/borg/compress.pyx | 55 ++++++++++++++++++++++++++++++++-- src/borg/helpers.py | 2 +- src/borg/testsuite/compress.py | 33 +++++++++++++++++++- 4 files changed, 86 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 1cd2bd0061..4307590cf0 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ # msgpack pure python data corruption was fixed in 0.4.6. # Also, we might use some rather recent API features. -install_requires = ['msgpack-python>=0.4.6', ] +install_requires = ['msgpack-python>=0.4.6', 'zstandard', ] # note for package maintainers: if you package borgbackup for distribution, # please add llfuse as a *requirement* on all platforms that have a working diff --git a/src/borg/compress.pyx b/src/borg/compress.pyx index 8e509213e3..475f967dae 100644 --- a/src/borg/compress.pyx +++ b/src/borg/compress.pyx @@ -22,9 +22,15 @@ try: except ImportError: lzma = None +try: + import zstd +except ImportError: + zstd = None + + from .helpers import Buffer, DecompressionError -API_VERSION = '1.1_03' +API_VERSION = '1.1_04' cdef extern from "lz4.h": int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) nogil @@ -186,6 +192,38 @@ class LZMA(CompressorBase): raise DecompressionError(str(e)) from None +class ZSTD(CompressorBase): + """zstd compression / decompression (pypi: zstandard, gh: python-zstandard)""" + # This is a NOT THREAD SAFE implementation. + # Only ONE python context must to be created at a time. + # It should work flawlessly as long as borg will call ONLY ONE compression job at time. + ID = b'\x03\x00' + name = 'zstd' + + def __init__(self, level=3, **kwargs): + super().__init__(**kwargs) + self.level = level + if zstd is None: + raise ValueError('No zstd support found.') + + def compress(self, data): + if not isinstance(data, bytes): + data = bytes(data) # zstd < 0.9.0 does not work with memoryview + cctx = zstd.ZstdCompressor(level=self.level, write_content_size=True) + data = cctx.compress(data) + return super().compress(data) + + def decompress(self, data): + if not isinstance(data, bytes): + data = bytes(data) # zstd < 0.9.0 does not work with memoryview + dctx = zstd.ZstdDecompressor() + data = super().decompress(data) + try: + return dctx.decompress(data) + except zstd.ZstdError as e: + raise DecompressionError(str(e)) from None + + class ZLIB(CompressorBase): """ zlib compression / decompression (python stdlib) @@ -289,9 +327,10 @@ COMPRESSOR_TABLE = { ZLIB.name: ZLIB, LZMA.name: LZMA, Auto.name: Auto, + ZSTD.name: ZSTD, } # List of possible compression types. Does not include Auto, since it is a meta-Compressor. -COMPRESSOR_LIST = [LZ4, CNONE, ZLIB, LZMA, ] # check fast stuff first +COMPRESSOR_LIST = [LZ4, ZSTD, CNONE, ZLIB, LZMA, ] # check fast stuff first def get_compressor(name, **kwargs): cls = COMPRESSOR_TABLE[name] @@ -344,6 +383,16 @@ class CompressionSpec: else: raise ValueError self.level = level + elif self.name in ('zstd', ): + if count < 2: + level = 3 # default compression level in zstd + elif count == 2: + level = int(values[1]) + if not 1 <= level <= 22: + raise ValueError + else: + raise ValueError + self.level = level elif self.name == 'auto': if 2 <= count <= 3: compression = ','.join(values[1:]) @@ -357,7 +406,7 @@ class CompressionSpec: def compressor(self): if self.name in ('none', 'lz4', ): return get_compressor(self.name) - elif self.name in ('zlib', 'lzma', ): + elif self.name in ('zlib', 'lzma', 'zstd', ): return get_compressor(self.name, level=self.level) elif self.name == 'auto': return get_compressor(self.name, compressor=self.inner.compressor) diff --git a/src/borg/helpers.py b/src/borg/helpers.py index cb3e10cac2..3aea2833da 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -135,7 +135,7 @@ def check_extension_modules(): raise ExtensionModuleError if chunker.API_VERSION != '1.1_01': raise ExtensionModuleError - if compress.API_VERSION != '1.1_03': + if compress.API_VERSION != '1.1_04': raise ExtensionModuleError if borg.crypto.low_level.API_VERSION != '1.1_02': raise ExtensionModuleError diff --git a/src/borg/testsuite/compress.py b/src/borg/testsuite/compress.py index f881ad2c7c..b995b0ec37 100644 --- a/src/borg/testsuite/compress.py +++ b/src/borg/testsuite/compress.py @@ -5,9 +5,14 @@ except ImportError: lzma = None +try: + import zstd +except ImportError: + zstd = None + import pytest -from ..compress import get_compressor, Compressor, CompressionSpec, CNONE, ZLIB, LZ4, LZMA, Auto +from ..compress import get_compressor, Compressor, CompressionSpec, CNONE, ZLIB, LZ4, LZMA, ZSTD, Auto buffer = bytes(2**16) @@ -69,6 +74,16 @@ def test_lzma(): assert data == Compressor(**params).decompress(cdata) # autodetect +def test_zstd(): + if zstd is None: + pytest.skip("No zstd support found.") + c = get_compressor(name='zstd') + cdata = c.compress(data) + assert len(cdata) < len(data) + assert data == c.decompress(cdata) + assert data == Compressor(**params).decompress(cdata) # autodetect + + def test_autodetect_invalid(): with pytest.raises(ValueError): Compressor(**params).decompress(b'\xff\xfftotalcrap') @@ -104,6 +119,12 @@ def test_compressor(): dict(name='lzma', level=6), # we do not test lzma on level 9 because of the huge memory needs ] + if zstd: + params_list += [ + dict(name='zstd', level=1), + dict(name='zstd', level=3), + # also avoiding high zstd levels, memory needs unclear + ] for params in params_list: c = Compressor(**params) assert data == c.decompress(c.compress(data)) @@ -154,6 +175,16 @@ def test_compression_specs(): assert isinstance(lzma, LZMA) assert lzma.level == 9 + zstd = CompressionSpec('zstd').compressor + assert isinstance(zstd, ZSTD) + assert zstd.level == 3 + zstd = CompressionSpec('zstd,1').compressor + assert isinstance(zstd, ZSTD) + assert zstd.level == 1 + zstd = CompressionSpec('zstd,22').compressor + assert isinstance(zstd, ZSTD) + assert zstd.level == 22 + with pytest.raises(ValueError): CompressionSpec('lzma,9,invalid') with pytest.raises(ValueError): From 01078328e26295045055eac9822c8dcf7d3a1d45 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 3 Dec 2017 05:38:23 +0100 Subject: [PATCH 186/798] zstd: use own Cython-based binding, remove python-zstandard dep currently requires an externally available libzstd >= 1.3.0, no bundled zstd yet. (cherry picked from commit aec36f64a2e7271ab04d70ed2cad2bceb33930d4) --- setup.py | 26 +++++++++- src/borg/algorithms/zstd-libselect.h | 5 ++ src/borg/compress.pyx | 73 ++++++++++++++++++++-------- 3 files changed, 81 insertions(+), 23 deletions(-) create mode 100644 src/borg/algorithms/zstd-libselect.h diff --git a/setup.py b/setup.py index 4307590cf0..07740708e1 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ # msgpack pure python data corruption was fixed in 0.4.6. # Also, we might use some rather recent API features. -install_requires = ['msgpack-python>=0.4.6', 'zstandard', ] +install_requires = ['msgpack-python>=0.4.6', ] # note for package maintainers: if you package borgbackup for distribution, # please add llfuse as a *requirement* on all platforms that have a working @@ -155,10 +155,20 @@ def detect_libb2(prefixes): return prefix +def detect_libzstd(prefixes): + for prefix in prefixes: + filename = os.path.join(prefix, 'include', 'zstd.h') + if os.path.exists(filename): + with open(filename, 'r') as fd: + if 'ZSTD_getFrameContentSize' in fd.read(): + return prefix + + include_dirs = [] library_dirs = [] define_macros = [] crypto_libraries = ['crypto'] +compression_libraries = ['lz4'] possible_openssl_prefixes = ['/usr', '/usr/local', '/usr/local/opt/openssl', '/usr/local/ssl', '/usr/local/openssl', '/usr/local/borg', '/opt/local', '/opt/pkg', ] @@ -194,6 +204,18 @@ def detect_libb2(prefixes): crypto_libraries.append('b2') define_macros.append(('BORG_USE_LIBB2', 'YES')) +possible_libzstd_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libzstd', '/usr/local/libzstd', + '/usr/local/borg', '/opt/local', '/opt/pkg', ] +if os.environ.get('BORG_LIBZSTD_PREFIX'): + possible_libzstd_prefixes.insert(0, os.environ.get('BORG_LIBZSTD_PREFIX')) +libzstd_prefix = detect_libzstd(possible_libzstd_prefixes) +if libzstd_prefix: + print('Detected and preferring libzstd over bundled ZSTD') + include_dirs.append(os.path.join(libzstd_prefix, 'include')) + library_dirs.append(os.path.join(libzstd_prefix, 'lib')) + compression_libraries.append('zstd') + define_macros.append(('BORG_USE_LIBZSTD', 'YES')) + with open('README.rst', 'r') as fd: long_description = fd.read() @@ -754,7 +776,7 @@ def run(self): ext_modules = [] if not on_rtd: ext_modules += [ - Extension('borg.compress', [compress_source], libraries=['lz4'], include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros), + Extension('borg.compress', [compress_source], libraries=compression_libraries, include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros), Extension('borg.crypto.low_level', [crypto_ll_source], libraries=crypto_libraries, include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros), Extension('borg.hashindex', [hashindex_source]), Extension('borg.item', [item_source]), diff --git a/src/borg/algorithms/zstd-libselect.h b/src/borg/algorithms/zstd-libselect.h new file mode 100644 index 0000000000..bb71553c14 --- /dev/null +++ b/src/borg/algorithms/zstd-libselect.h @@ -0,0 +1,5 @@ +#ifdef BORG_USE_LIBZSTD +#include +#else +#error "TODO" +#endif diff --git a/src/borg/compress.pyx b/src/borg/compress.pyx index 475f967dae..be2c3c3d11 100644 --- a/src/borg/compress.pyx +++ b/src/borg/compress.pyx @@ -22,11 +22,6 @@ try: except ImportError: lzma = None -try: - import zstd -except ImportError: - zstd = None - from .helpers import Buffer, DecompressionError @@ -38,6 +33,17 @@ cdef extern from "lz4.h": int LZ4_compressBound(int inputSize) nogil +cdef extern from "algorithms/zstd-libselect.h": + size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel) nogil + size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t compressedSize) nogil + size_t ZSTD_compressBound(size_t srcSize) nogil + unsigned long long ZSTD_CONTENTSIZE_UNKNOWN + unsigned long long ZSTD_CONTENTSIZE_ERROR + unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) nogil + unsigned ZSTD_isError(size_t code) nogil + const char* ZSTD_getErrorName(size_t code) nogil + + buffer = Buffer(bytearray, size=0) @@ -203,25 +209,50 @@ class ZSTD(CompressorBase): def __init__(self, level=3, **kwargs): super().__init__(**kwargs) self.level = level - if zstd is None: - raise ValueError('No zstd support found.') - def compress(self, data): - if not isinstance(data, bytes): - data = bytes(data) # zstd < 0.9.0 does not work with memoryview - cctx = zstd.ZstdCompressor(level=self.level, write_content_size=True) - data = cctx.compress(data) - return super().compress(data) + def compress(self, idata): + if not isinstance(idata, bytes): + idata = bytes(idata) # code below does not work with memoryview + cdef int isize = len(idata) + cdef size_t osize + cdef char *source = idata + cdef char *dest + cdef int level = self.level + osize = ZSTD_compressBound(isize) + buf = buffer.get(osize) + dest = buf + with nogil: + osize = ZSTD_compress(dest, osize, source, isize, level) + if ZSTD_isError(osize): + raise Exception('zstd compress failed: %s' % ZSTD_getErrorName(osize)) + return super().compress(dest[:osize]) - def decompress(self, data): - if not isinstance(data, bytes): - data = bytes(data) # zstd < 0.9.0 does not work with memoryview - dctx = zstd.ZstdDecompressor() - data = super().decompress(data) + def decompress(self, idata): + if not isinstance(idata, bytes): + idata = bytes(idata) # code below does not work with memoryview + idata = super().decompress(idata) + cdef int isize = len(idata) + cdef unsigned long long osize + cdef unsigned long long rsize + cdef char *source = idata + cdef char *dest + osize = ZSTD_getFrameContentSize(source, isize) + if osize == ZSTD_CONTENTSIZE_ERROR: + raise DecompressionError('zstd get size failed: data was not compressed by zstd') + if osize == ZSTD_CONTENTSIZE_UNKNOWN: + raise DecompressionError('zstd get size failed: original size unknown') try: - return dctx.decompress(data) - except zstd.ZstdError as e: - raise DecompressionError(str(e)) from None + buf = buffer.get(osize) + except MemoryError: + raise DecompressionError('MemoryError') + dest = buf + with nogil: + rsize = ZSTD_decompress(dest, osize, source, isize) + if ZSTD_isError(rsize): + raise DecompressionError('zstd decompress failed: %s' % ZSTD_getErrorName(rsize)) + if rsize != osize: + raise DecompressionError('zstd decompress failed: size mismatch') + return dest[:osize] class ZLIB(CompressorBase): From 15ec4c4259ea4fe36b074a92620bd23cfc322395 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 4 Dec 2017 23:59:35 +0100 Subject: [PATCH 187/798] add setup_zstd.py from python-zstandard project, as is (cherry picked from commit 50c08318240c747ce4c9355ba90a1cf466abadb1) --- setup_zstd.py | 125 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 setup_zstd.py diff --git a/setup_zstd.py b/setup_zstd.py new file mode 100644 index 0000000000..8e1f86c514 --- /dev/null +++ b/setup_zstd.py @@ -0,0 +1,125 @@ +# Copyright (c) 2016-present, Gregory Szorc +# All rights reserved. +# +# This software may be modified and distributed under the terms +# of the BSD license. See the LICENSE file for details. + +import os +from distutils.extension import Extension + + +zstd_sources = ['zstd/%s' % p for p in ( + 'common/entropy_common.c', + 'common/error_private.c', + 'common/fse_decompress.c', + 'common/pool.c', + 'common/threading.c', + 'common/xxhash.c', + 'common/zstd_common.c', + 'compress/fse_compress.c', + 'compress/huf_compress.c', + 'compress/zstd_compress.c', + 'compress/zstdmt_compress.c', + 'decompress/huf_decompress.c', + 'decompress/zstd_decompress.c', + 'dictBuilder/cover.c', + 'dictBuilder/divsufsort.c', + 'dictBuilder/zdict.c', +)] + +zstd_sources_legacy = ['zstd/%s' % p for p in ( + 'deprecated/zbuff_common.c', + 'deprecated/zbuff_compress.c', + 'deprecated/zbuff_decompress.c', + 'legacy/zstd_v01.c', + 'legacy/zstd_v02.c', + 'legacy/zstd_v03.c', + 'legacy/zstd_v04.c', + 'legacy/zstd_v05.c', + 'legacy/zstd_v06.c', + 'legacy/zstd_v07.c' +)] + +zstd_includes = [ + 'zstd', + 'zstd/common', + 'zstd/compress', + 'zstd/decompress', + 'zstd/dictBuilder', +] + +zstd_includes_legacy = [ + 'zstd/deprecated', + 'zstd/legacy', +] + +ext_includes = [ + 'c-ext', + 'zstd/common', +] + +ext_sources = [ + 'zstd/common/pool.c', + 'zstd/common/threading.c', + 'zstd.c', + 'c-ext/bufferutil.c', + 'c-ext/compressiondict.c', + 'c-ext/compressobj.c', + 'c-ext/compressor.c', + 'c-ext/compressoriterator.c', + 'c-ext/compressionparams.c', + 'c-ext/compressionreader.c', + 'c-ext/compressionwriter.c', + 'c-ext/constants.c', + 'c-ext/decompressobj.c', + 'c-ext/decompressor.c', + 'c-ext/decompressoriterator.c', + 'c-ext/decompressionreader.c', + 'c-ext/decompressionwriter.c', + 'c-ext/frameparams.c', +] + +zstd_depends = [ + 'c-ext/python-zstandard.h', +] + + +def get_c_extension(support_legacy=False, system_zstd=False, name='zstd'): + """Obtain a distutils.extension.Extension for the C extension.""" + root = os.path.abspath(os.path.dirname(__file__)) + + sources = set([os.path.join(root, p) for p in ext_sources]) + if not system_zstd: + sources.update([os.path.join(root, p) for p in zstd_sources]) + if support_legacy: + sources.update([os.path.join(root, p) for p in zstd_sources_legacy]) + sources = list(sources) + + include_dirs = set([os.path.join(root, d) for d in ext_includes]) + if not system_zstd: + include_dirs.update([os.path.join(root, d) for d in zstd_includes]) + if support_legacy: + include_dirs.update([os.path.join(root, d) for d in zstd_includes_legacy]) + include_dirs = list(include_dirs) + + depends = [os.path.join(root, p) for p in zstd_depends] + + extra_args = ['-DZSTD_MULTITHREAD'] + + if not system_zstd: + extra_args.append('-DZSTDLIB_VISIBILITY=') + extra_args.append('-DZDICTLIB_VISIBILITY=') + extra_args.append('-DZSTDERRORLIB_VISIBILITY=') + extra_args.append('-fvisibility=hidden') + + if not system_zstd and support_legacy: + extra_args.append('-DZSTD_LEGACY_SUPPORT=1') + + libraries = ['zstd'] if system_zstd else [] + + # TODO compile with optimizations. + return Extension(name, sources, + include_dirs=include_dirs, + depends=depends, + extra_compile_args=extra_args, + libraries=libraries) From 0271ae6f9521ed8b4de5ee886f4a5f3ef051899d Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 5 Dec 2017 04:16:25 +0100 Subject: [PATCH 188/798] support code to build bundled zstd code into the compress extension setup_zstd.py modified so it is just amending the Extension() kwargs, but the Extension is initialized by the caller. this way, amending can happend multiple times (e.g. for multiple compression algorithms). also: - move include/library dirs processing for system-library case - move system zstd prefix detection to setup_zstd module - cosmetic: setup.py whitespace fixes - prefer system zstd option, document zstd min. requirement (cherry picked from commit 34b92ffdaac59f2a7fd30a7a4328062f502ff3b2) --- setup.py | 45 +++--- setup_zstd.py | 216 +++++++++++++-------------- src/borg/algorithms/zstd-libselect.h | 2 +- 3 files changed, 131 insertions(+), 132 deletions(-) diff --git a/setup.py b/setup.py index 07740708e1..036bfc1069 100644 --- a/setup.py +++ b/setup.py @@ -12,6 +12,11 @@ import textwrap +import setup_zstd + +# True: use the shared libzstd (>= 1.3.0) from the system, False: use the bundled zstd code +prefer_system_libzstd = True + min_python = (3, 4) my_python = sys.version_info @@ -155,15 +160,6 @@ def detect_libb2(prefixes): return prefix -def detect_libzstd(prefixes): - for prefix in prefixes: - filename = os.path.join(prefix, 'include', 'zstd.h') - if os.path.exists(filename): - with open(filename, 'r') as fd: - if 'ZSTD_getFrameContentSize' in fd.read(): - return prefix - - include_dirs = [] library_dirs = [] define_macros = [] @@ -208,13 +204,13 @@ def detect_libzstd(prefixes): '/usr/local/borg', '/opt/local', '/opt/pkg', ] if os.environ.get('BORG_LIBZSTD_PREFIX'): possible_libzstd_prefixes.insert(0, os.environ.get('BORG_LIBZSTD_PREFIX')) -libzstd_prefix = detect_libzstd(possible_libzstd_prefixes) -if libzstd_prefix: +libzstd_prefix = setup_zstd.zstd_system_prefix(possible_libzstd_prefixes) +if prefer_system_libzstd and libzstd_prefix: print('Detected and preferring libzstd over bundled ZSTD') - include_dirs.append(os.path.join(libzstd_prefix, 'include')) - library_dirs.append(os.path.join(libzstd_prefix, 'lib')) - compression_libraries.append('zstd') define_macros.append(('BORG_USE_LIBZSTD', 'YES')) + libzstd_system = True +else: + libzstd_system = False with open('README.rst', 'r') as fd: @@ -775,18 +771,21 @@ def run(self): ext_modules = [] if not on_rtd: + compress_ext_kwargs = dict(sources=[compress_source], include_dirs=include_dirs, library_dirs=library_dirs, + libraries=compression_libraries, define_macros=define_macros) + compress_ext_kwargs = setup_zstd.zstd_ext_kwargs(bundled_path='src/borg/algorithms/zstd', + system_prefix=libzstd_prefix, system=libzstd_system, + multithreaded=False, legacy=False, **compress_ext_kwargs) ext_modules += [ - Extension('borg.compress', [compress_source], libraries=compression_libraries, include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros), - Extension('borg.crypto.low_level', [crypto_ll_source], libraries=crypto_libraries, include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros), - Extension('borg.hashindex', [hashindex_source]), - Extension('borg.item', [item_source]), - Extension('borg.chunker', [chunker_source]), - Extension('borg.algorithms.checksums', [checksums_source]), - -] + Extension('borg.compress', **compress_ext_kwargs), + Extension('borg.crypto.low_level', [crypto_ll_source], libraries=crypto_libraries, include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros), + Extension('borg.hashindex', [hashindex_source]), + Extension('borg.item', [item_source]), + Extension('borg.chunker', [chunker_source]), + Extension('borg.algorithms.checksums', [checksums_source]), + ] if not sys.platform.startswith(('win32', )): ext_modules.append(Extension('borg.platform.posix', [platform_posix_source])) - if sys.platform == 'linux': ext_modules.append(Extension('borg.platform.linux', [platform_linux_source], libraries=['acl'])) elif sys.platform.startswith('freebsd'): diff --git a/setup_zstd.py b/setup_zstd.py index 8e1f86c514..bf23cb5846 100644 --- a/setup_zstd.py +++ b/setup_zstd.py @@ -1,125 +1,125 @@ +# Support code for building a C extension with zstd files +# # Copyright (c) 2016-present, Gregory Szorc +# 2017-present, Thomas Waldmann (mods to make it more generic) # All rights reserved. # # This software may be modified and distributed under the terms # of the BSD license. See the LICENSE file for details. import os -from distutils.extension import Extension - - -zstd_sources = ['zstd/%s' % p for p in ( - 'common/entropy_common.c', - 'common/error_private.c', - 'common/fse_decompress.c', - 'common/pool.c', - 'common/threading.c', - 'common/xxhash.c', - 'common/zstd_common.c', - 'compress/fse_compress.c', - 'compress/huf_compress.c', - 'compress/zstd_compress.c', - 'compress/zstdmt_compress.c', - 'decompress/huf_decompress.c', - 'decompress/zstd_decompress.c', - 'dictBuilder/cover.c', - 'dictBuilder/divsufsort.c', - 'dictBuilder/zdict.c', -)] - -zstd_sources_legacy = ['zstd/%s' % p for p in ( - 'deprecated/zbuff_common.c', - 'deprecated/zbuff_compress.c', - 'deprecated/zbuff_decompress.c', - 'legacy/zstd_v01.c', - 'legacy/zstd_v02.c', - 'legacy/zstd_v03.c', - 'legacy/zstd_v04.c', - 'legacy/zstd_v05.c', - 'legacy/zstd_v06.c', - 'legacy/zstd_v07.c' -)] - -zstd_includes = [ - 'zstd', - 'zstd/common', - 'zstd/compress', - 'zstd/decompress', - 'zstd/dictBuilder', -] -zstd_includes_legacy = [ - 'zstd/deprecated', - 'zstd/legacy', +# zstd files, structure as seen in zstd project repository: + +zstd_sources = [ + 'lib/common/entropy_common.c', + 'lib/common/error_private.c', + 'lib/common/fse_decompress.c', + 'lib/common/pool.c', + 'lib/common/threading.c', + 'lib/common/xxhash.c', + 'lib/common/zstd_common.c', + 'lib/compress/fse_compress.c', + 'lib/compress/huf_compress.c', + 'lib/compress/zstd_compress.c', + 'lib/compress/zstd_double_fast.c', + 'lib/compress/zstd_fast.c', + 'lib/compress/zstd_lazy.c', + 'lib/compress/zstd_ldm.c', + 'lib/compress/zstd_opt.c', + 'lib/compress/zstdmt_compress.c', + 'lib/decompress/huf_decompress.c', + 'lib/decompress/zstd_decompress.c', + 'lib/dictBuilder/cover.c', + 'lib/dictBuilder/divsufsort.c', + 'lib/dictBuilder/zdict.c', ] -ext_includes = [ - 'c-ext', - 'zstd/common', +zstd_sources_legacy = [ + 'lib/deprecated/zbuff_common.c', + 'lib/deprecated/zbuff_compress.c', + 'lib/deprecated/zbuff_decompress.c', + 'lib/legacy/zstd_v01.c', + 'lib/legacy/zstd_v02.c', + 'lib/legacy/zstd_v03.c', + 'lib/legacy/zstd_v04.c', + 'lib/legacy/zstd_v05.c', + 'lib/legacy/zstd_v06.c', + 'lib/legacy/zstd_v07.c', ] -ext_sources = [ - 'zstd/common/pool.c', - 'zstd/common/threading.c', - 'zstd.c', - 'c-ext/bufferutil.c', - 'c-ext/compressiondict.c', - 'c-ext/compressobj.c', - 'c-ext/compressor.c', - 'c-ext/compressoriterator.c', - 'c-ext/compressionparams.c', - 'c-ext/compressionreader.c', - 'c-ext/compressionwriter.c', - 'c-ext/constants.c', - 'c-ext/decompressobj.c', - 'c-ext/decompressor.c', - 'c-ext/decompressoriterator.c', - 'c-ext/decompressionreader.c', - 'c-ext/decompressionwriter.c', - 'c-ext/frameparams.c', +zstd_includes = [ + 'lib', + 'lib/common', + 'lib/compress', + 'lib/decompress', + 'lib/dictBuilder', ] -zstd_depends = [ - 'c-ext/python-zstandard.h', +zstd_includes_legacy = [ + 'lib/deprecated', + 'lib/legacy', ] -def get_c_extension(support_legacy=False, system_zstd=False, name='zstd'): - """Obtain a distutils.extension.Extension for the C extension.""" - root = os.path.abspath(os.path.dirname(__file__)) - - sources = set([os.path.join(root, p) for p in ext_sources]) - if not system_zstd: - sources.update([os.path.join(root, p) for p in zstd_sources]) - if support_legacy: - sources.update([os.path.join(root, p) for p in zstd_sources_legacy]) - sources = list(sources) - - include_dirs = set([os.path.join(root, d) for d in ext_includes]) - if not system_zstd: - include_dirs.update([os.path.join(root, d) for d in zstd_includes]) - if support_legacy: - include_dirs.update([os.path.join(root, d) for d in zstd_includes_legacy]) - include_dirs = list(include_dirs) - - depends = [os.path.join(root, p) for p in zstd_depends] - - extra_args = ['-DZSTD_MULTITHREAD'] - - if not system_zstd: - extra_args.append('-DZSTDLIB_VISIBILITY=') - extra_args.append('-DZDICTLIB_VISIBILITY=') - extra_args.append('-DZSTDERRORLIB_VISIBILITY=') - extra_args.append('-fvisibility=hidden') - - if not system_zstd and support_legacy: - extra_args.append('-DZSTD_LEGACY_SUPPORT=1') - - libraries = ['zstd'] if system_zstd else [] - - # TODO compile with optimizations. - return Extension(name, sources, - include_dirs=include_dirs, - depends=depends, - extra_compile_args=extra_args, - libraries=libraries) +def zstd_system_prefix(prefixes): + for prefix in prefixes: + filename = os.path.join(prefix, 'include', 'zstd.h') + if os.path.exists(filename): + with open(filename, 'r') as fd: + if 'ZSTD_getFrameContentSize' in fd.read(): # checks for zstd >= 1.3.0 + return prefix + + +def zstd_ext_kwargs(bundled_path, system_prefix=None, system=False, multithreaded=False, legacy=False, **kwargs): + """amend kwargs with zstd suff for a distutils.extension.Extension initialization. + + bundled_path: relative (to this file) path to the bundled library source code files + system_prefix: where the system-installed library can be found + system: True: use the system-installed shared library, False: use the bundled library code + multithreaded: True: define ZSTD_MULTITHREAD + legacy: include legacy API support + kwargs: distutils.extension.Extension kwargs that should be amended + returns: amended kwargs + """ + def multi_join(paths, *path_segments): + """apply os.path.join on a list of paths""" + return [os.path.join(*(path_segments + (path, ))) for path in paths] + + use_system = system and system_prefix is not None + + sources = kwargs.get('sources', []) + if not use_system: + sources += multi_join(zstd_sources, bundled_path) + if legacy: + sources += multi_join(zstd_sources_legacy, bundled_path) + + include_dirs = kwargs.get('include_dirs', []) + if use_system: + include_dirs += multi_join(['include'], system_prefix) + else: + include_dirs += multi_join(zstd_includes, bundled_path) + if legacy: + include_dirs += multi_join(zstd_includes_legacy, bundled_path) + + library_dirs = kwargs.get('library_dirs', []) + if use_system: + library_dirs += multi_join(['lib'], system_prefix) + + libraries = kwargs.get('libraries', []) + if use_system: + libraries += ['zstd', ] + + extra_compile_args = kwargs.get('extra_compile_args', []) + if multithreaded: + extra_compile_args += ['-DZSTD_MULTITHREAD', ] + if not use_system: + extra_compile_args += ['-DZSTDLIB_VISIBILITY=', '-DZDICTLIB_VISIBILITY=', '-DZSTDERRORLIB_VISIBILITY=', ] + # '-fvisibility=hidden' does not work, doesn't find PyInit_compress then + if legacy: + extra_compile_args += ['-DZSTD_LEGACY_SUPPORT=1', ] + + ret = dict(**kwargs) + ret.update(dict(sources=sources, extra_compile_args=extra_compile_args, + include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries)) + return ret diff --git a/src/borg/algorithms/zstd-libselect.h b/src/borg/algorithms/zstd-libselect.h index bb71553c14..9a4ded364b 100644 --- a/src/borg/algorithms/zstd-libselect.h +++ b/src/borg/algorithms/zstd-libselect.h @@ -1,5 +1,5 @@ #ifdef BORG_USE_LIBZSTD #include #else -#error "TODO" +#include "zstd/lib/zstd.h" #endif From c7383589ccea8045ebabcc90699fe2e75f44df64 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 5 Dec 2017 04:18:57 +0100 Subject: [PATCH 189/798] bundle zstd 1.3.2 source code only .c and .h files + license (cherry picked from commit dc883f62ae300f82eb4e0ad8881794fe66750b95) --- docs/3rd_party/zstd/LICENSE | 30 + .../algorithms/zstd/lib/common/bitstream.h | 471 ++ .../algorithms/zstd/lib/common/compiler.h | 86 + .../zstd/lib/common/entropy_common.c | 221 + .../zstd/lib/common/error_private.c | 47 + .../zstd/lib/common/error_private.h | 76 + src/borg/algorithms/zstd/lib/common/fse.h | 704 +++ .../zstd/lib/common/fse_decompress.c | 309 ++ src/borg/algorithms/zstd/lib/common/huf.h | 302 ++ src/borg/algorithms/zstd/lib/common/mem.h | 360 ++ src/borg/algorithms/zstd/lib/common/pool.c | 255 + src/borg/algorithms/zstd/lib/common/pool.h | 65 + .../algorithms/zstd/lib/common/threading.c | 75 + .../algorithms/zstd/lib/common/threading.h | 123 + src/borg/algorithms/zstd/lib/common/xxhash.c | 875 ++++ src/borg/algorithms/zstd/lib/common/xxhash.h | 305 ++ .../algorithms/zstd/lib/common/zstd_common.c | 80 + .../algorithms/zstd/lib/common/zstd_errors.h | 83 + .../zstd/lib/common/zstd_internal.h | 409 ++ .../zstd/lib/compress/fse_compress.c | 841 +++ .../zstd/lib/compress/huf_compress.c | 690 +++ .../zstd/lib/compress/zstd_compress.c | 3023 +++++++++++ .../zstd/lib/compress/zstd_compress.h | 307 ++ .../zstd/lib/compress/zstd_double_fast.c | 308 ++ .../zstd/lib/compress/zstd_double_fast.h | 28 + .../algorithms/zstd/lib/compress/zstd_fast.c | 242 + .../algorithms/zstd/lib/compress/zstd_fast.h | 30 + .../algorithms/zstd/lib/compress/zstd_lazy.c | 749 +++ .../algorithms/zstd/lib/compress/zstd_lazy.h | 38 + .../algorithms/zstd/lib/compress/zstd_ldm.c | 707 +++ .../algorithms/zstd/lib/compress/zstd_ldm.h | 67 + .../algorithms/zstd/lib/compress/zstd_opt.c | 957 ++++ .../algorithms/zstd/lib/compress/zstd_opt.h | 30 + .../zstd/lib/compress/zstdmt_compress.c | 1099 ++++ .../zstd/lib/compress/zstdmt_compress.h | 132 + .../zstd/lib/decompress/huf_decompress.c | 996 ++++ .../zstd/lib/decompress/zstd_decompress.c | 2655 ++++++++++ .../algorithms/zstd/lib/deprecated/zbuff.h | 213 + .../zstd/lib/deprecated/zbuff_common.c | 26 + .../zstd/lib/deprecated/zbuff_compress.c | 146 + .../zstd/lib/deprecated/zbuff_decompress.c | 75 + .../algorithms/zstd/lib/dictBuilder/cover.c | 1045 ++++ .../zstd/lib/dictBuilder/divsufsort.c | 1913 +++++++ .../zstd/lib/dictBuilder/divsufsort.h | 67 + .../algorithms/zstd/lib/dictBuilder/zdict.c | 1075 ++++ .../algorithms/zstd/lib/dictBuilder/zdict.h | 211 + .../algorithms/zstd/lib/legacy/zstd_legacy.h | 379 ++ .../algorithms/zstd/lib/legacy/zstd_v01.c | 2127 ++++++++ .../algorithms/zstd/lib/legacy/zstd_v01.h | 89 + .../algorithms/zstd/lib/legacy/zstd_v02.c | 3556 +++++++++++++ .../algorithms/zstd/lib/legacy/zstd_v02.h | 88 + .../algorithms/zstd/lib/legacy/zstd_v03.c | 3197 ++++++++++++ .../algorithms/zstd/lib/legacy/zstd_v03.h | 88 + .../algorithms/zstd/lib/legacy/zstd_v04.c | 3824 ++++++++++++++ .../algorithms/zstd/lib/legacy/zstd_v04.h | 137 + .../algorithms/zstd/lib/legacy/zstd_v05.c | 4083 +++++++++++++++ .../algorithms/zstd/lib/legacy/zstd_v05.h | 157 + .../algorithms/zstd/lib/legacy/zstd_v06.c | 4200 +++++++++++++++ .../algorithms/zstd/lib/legacy/zstd_v06.h | 167 + .../algorithms/zstd/lib/legacy/zstd_v07.c | 4578 +++++++++++++++++ .../algorithms/zstd/lib/legacy/zstd_v07.h | 182 + src/borg/algorithms/zstd/lib/zstd.h | 1386 +++++ 62 files changed, 50784 insertions(+) create mode 100644 docs/3rd_party/zstd/LICENSE create mode 100644 src/borg/algorithms/zstd/lib/common/bitstream.h create mode 100644 src/borg/algorithms/zstd/lib/common/compiler.h create mode 100644 src/borg/algorithms/zstd/lib/common/entropy_common.c create mode 100644 src/borg/algorithms/zstd/lib/common/error_private.c create mode 100644 src/borg/algorithms/zstd/lib/common/error_private.h create mode 100644 src/borg/algorithms/zstd/lib/common/fse.h create mode 100644 src/borg/algorithms/zstd/lib/common/fse_decompress.c create mode 100644 src/borg/algorithms/zstd/lib/common/huf.h create mode 100644 src/borg/algorithms/zstd/lib/common/mem.h create mode 100644 src/borg/algorithms/zstd/lib/common/pool.c create mode 100644 src/borg/algorithms/zstd/lib/common/pool.h create mode 100644 src/borg/algorithms/zstd/lib/common/threading.c create mode 100644 src/borg/algorithms/zstd/lib/common/threading.h create mode 100644 src/borg/algorithms/zstd/lib/common/xxhash.c create mode 100644 src/borg/algorithms/zstd/lib/common/xxhash.h create mode 100644 src/borg/algorithms/zstd/lib/common/zstd_common.c create mode 100644 src/borg/algorithms/zstd/lib/common/zstd_errors.h create mode 100644 src/borg/algorithms/zstd/lib/common/zstd_internal.h create mode 100644 src/borg/algorithms/zstd/lib/compress/fse_compress.c create mode 100644 src/borg/algorithms/zstd/lib/compress/huf_compress.c create mode 100644 src/borg/algorithms/zstd/lib/compress/zstd_compress.c create mode 100644 src/borg/algorithms/zstd/lib/compress/zstd_compress.h create mode 100644 src/borg/algorithms/zstd/lib/compress/zstd_double_fast.c create mode 100644 src/borg/algorithms/zstd/lib/compress/zstd_double_fast.h create mode 100644 src/borg/algorithms/zstd/lib/compress/zstd_fast.c create mode 100644 src/borg/algorithms/zstd/lib/compress/zstd_fast.h create mode 100644 src/borg/algorithms/zstd/lib/compress/zstd_lazy.c create mode 100644 src/borg/algorithms/zstd/lib/compress/zstd_lazy.h create mode 100644 src/borg/algorithms/zstd/lib/compress/zstd_ldm.c create mode 100644 src/borg/algorithms/zstd/lib/compress/zstd_ldm.h create mode 100644 src/borg/algorithms/zstd/lib/compress/zstd_opt.c create mode 100644 src/borg/algorithms/zstd/lib/compress/zstd_opt.h create mode 100644 src/borg/algorithms/zstd/lib/compress/zstdmt_compress.c create mode 100644 src/borg/algorithms/zstd/lib/compress/zstdmt_compress.h create mode 100644 src/borg/algorithms/zstd/lib/decompress/huf_decompress.c create mode 100644 src/borg/algorithms/zstd/lib/decompress/zstd_decompress.c create mode 100644 src/borg/algorithms/zstd/lib/deprecated/zbuff.h create mode 100644 src/borg/algorithms/zstd/lib/deprecated/zbuff_common.c create mode 100644 src/borg/algorithms/zstd/lib/deprecated/zbuff_compress.c create mode 100644 src/borg/algorithms/zstd/lib/deprecated/zbuff_decompress.c create mode 100644 src/borg/algorithms/zstd/lib/dictBuilder/cover.c create mode 100644 src/borg/algorithms/zstd/lib/dictBuilder/divsufsort.c create mode 100644 src/borg/algorithms/zstd/lib/dictBuilder/divsufsort.h create mode 100644 src/borg/algorithms/zstd/lib/dictBuilder/zdict.c create mode 100644 src/borg/algorithms/zstd/lib/dictBuilder/zdict.h create mode 100644 src/borg/algorithms/zstd/lib/legacy/zstd_legacy.h create mode 100644 src/borg/algorithms/zstd/lib/legacy/zstd_v01.c create mode 100644 src/borg/algorithms/zstd/lib/legacy/zstd_v01.h create mode 100644 src/borg/algorithms/zstd/lib/legacy/zstd_v02.c create mode 100644 src/borg/algorithms/zstd/lib/legacy/zstd_v02.h create mode 100644 src/borg/algorithms/zstd/lib/legacy/zstd_v03.c create mode 100644 src/borg/algorithms/zstd/lib/legacy/zstd_v03.h create mode 100644 src/borg/algorithms/zstd/lib/legacy/zstd_v04.c create mode 100644 src/borg/algorithms/zstd/lib/legacy/zstd_v04.h create mode 100644 src/borg/algorithms/zstd/lib/legacy/zstd_v05.c create mode 100644 src/borg/algorithms/zstd/lib/legacy/zstd_v05.h create mode 100644 src/borg/algorithms/zstd/lib/legacy/zstd_v06.c create mode 100644 src/borg/algorithms/zstd/lib/legacy/zstd_v06.h create mode 100644 src/borg/algorithms/zstd/lib/legacy/zstd_v07.c create mode 100644 src/borg/algorithms/zstd/lib/legacy/zstd_v07.h create mode 100644 src/borg/algorithms/zstd/lib/zstd.h diff --git a/docs/3rd_party/zstd/LICENSE b/docs/3rd_party/zstd/LICENSE new file mode 100644 index 0000000000..a793a80289 --- /dev/null +++ b/docs/3rd_party/zstd/LICENSE @@ -0,0 +1,30 @@ +BSD License + +For Zstandard software + +Copyright (c) 2016-present, Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/borg/algorithms/zstd/lib/common/bitstream.h b/src/borg/algorithms/zstd/lib/common/bitstream.h new file mode 100644 index 0000000000..2094823fe2 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/common/bitstream.h @@ -0,0 +1,471 @@ +/* ****************************************************************** + bitstream + Part of FSE library + header file (to include) + Copyright (C) 2013-2017, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Source repository : https://github.com/Cyan4973/FiniteStateEntropy +****************************************************************** */ +#ifndef BITSTREAM_H_MODULE +#define BITSTREAM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + +/* +* This API consists of small unitary functions, which must be inlined for best performance. +* Since link-time-optimization is not available for all compilers, +* these functions are defined into a .h to be included. +*/ + +/*-**************************************** +* Dependencies +******************************************/ +#include "mem.h" /* unaligned access routines */ +#include "error_private.h" /* error codes and messages */ + + +/*-************************************* +* Debug +***************************************/ +#if defined(BIT_DEBUG) && (BIT_DEBUG>=1) +# include +#else +# ifndef assert +# define assert(condition) ((void)0) +# endif +#endif + + +/*========================================= +* Target specific +=========================================*/ +#if defined(__BMI__) && defined(__GNUC__) +# include /* support for bextr (experimental) */ +#endif + +#define STREAM_ACCUMULATOR_MIN_32 25 +#define STREAM_ACCUMULATOR_MIN_64 57 +#define STREAM_ACCUMULATOR_MIN ((U32)(MEM_32bits() ? STREAM_ACCUMULATOR_MIN_32 : STREAM_ACCUMULATOR_MIN_64)) + + +/*-****************************************** +* bitStream encoding API (write forward) +********************************************/ +/* bitStream can mix input from multiple sources. + * A critical property of these streams is that they encode and decode in **reverse** direction. + * So the first bit sequence you add will be the last to be read, like a LIFO stack. + */ +typedef struct +{ + size_t bitContainer; + unsigned bitPos; + char* startPtr; + char* ptr; + char* endPtr; +} BIT_CStream_t; + +MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity); +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC); +MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); + +/* Start with initCStream, providing the size of buffer to write into. +* bitStream will never write outside of this buffer. +* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code. +* +* bits are first added to a local register. +* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems. +* Writing data into memory is an explicit operation, performed by the flushBits function. +* Hence keep track how many bits are potentially stored into local register to avoid register overflow. +* After a flushBits, a maximum of 7 bits might still be stored into local register. +* +* Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers. +* +* Last operation is to close the bitStream. +* The function returns the final size of CStream in bytes. +* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable) +*/ + + +/*-******************************************** +* bitStream decoding API (read backward) +**********************************************/ +typedef struct +{ + size_t bitContainer; + unsigned bitsConsumed; + const char* ptr; + const char* start; + const char* limitPtr; +} BIT_DStream_t; + +typedef enum { BIT_DStream_unfinished = 0, + BIT_DStream_endOfBuffer = 1, + BIT_DStream_completed = 2, + BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */ + /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */ + +MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize); +MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits); +MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD); +MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); + + +/* Start by invoking BIT_initDStream(). +* A chunk of the bitStream is then stored into a local register. +* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). +* You can then retrieve bitFields stored into the local register, **in reverse order**. +* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method. +* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished. +* Otherwise, it can be less than that, so proceed accordingly. +* Checking if DStream has reached its end can be performed with BIT_endOfDStream(). +*/ + + +/*-**************************************** +* unsafe API +******************************************/ +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */ + +MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC); +/* unsafe version; does not check buffer overflow */ + +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits); +/* faster, but works only if nbBits >= 1 */ + + + +/*-************************************************************** +* Internal functions +****************************************************************/ +MEM_STATIC unsigned BIT_highbit32 (register U32 val) +{ + assert(val != 0); + { +# if defined(_MSC_VER) /* Visual */ + unsigned long r=0; + _BitScanReverse ( &r, val ); + return (unsigned) r; +# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ + return 31 - __builtin_clz (val); +# else /* Software version */ + static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, + 11, 14, 16, 18, 22, 25, 3, 30, + 8, 12, 20, 28, 15, 17, 24, 7, + 19, 27, 23, 6, 26, 5, 4, 31 }; + U32 v = val; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27]; +# endif + } +} + +/*===== Local Constants =====*/ +static const unsigned BIT_mask[] = { + 0, 1, 3, 7, 0xF, 0x1F, + 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, + 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, + 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, + 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, 0x7FFFFFF, 0xFFFFFFF, 0x1FFFFFFF, + 0x3FFFFFFF, 0x7FFFFFFF}; /* up to 31 bits */ +#define BIT_MASK_SIZE (sizeof(BIT_mask) / sizeof(BIT_mask[0])) + +/*-************************************************************** +* bitStream encoding +****************************************************************/ +/*! BIT_initCStream() : + * `dstCapacity` must be > sizeof(size_t) + * @return : 0 if success, + * otherwise an error code (can be tested using ERR_isError()) */ +MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, + void* startPtr, size_t dstCapacity) +{ + bitC->bitContainer = 0; + bitC->bitPos = 0; + bitC->startPtr = (char*)startPtr; + bitC->ptr = bitC->startPtr; + bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->bitContainer); + if (dstCapacity <= sizeof(bitC->bitContainer)) return ERROR(dstSize_tooSmall); + return 0; +} + +/*! BIT_addBits() : + * can add up to 31 bits into `bitC`. + * Note : does not check for register overflow ! */ +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, + size_t value, unsigned nbBits) +{ + MEM_STATIC_ASSERT(BIT_MASK_SIZE == 32); + assert(nbBits < BIT_MASK_SIZE); + assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); + bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos; + bitC->bitPos += nbBits; +} + +/*! BIT_addBitsFast() : + * works only if `value` is _clean_, meaning all high bits above nbBits are 0 */ +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, + size_t value, unsigned nbBits) +{ + assert((value>>nbBits) == 0); + assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); + bitC->bitContainer |= value << bitC->bitPos; + bitC->bitPos += nbBits; +} + +/*! BIT_flushBitsFast() : + * assumption : bitContainer has not overflowed + * unsafe version; does not check buffer overflow */ +MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC) +{ + size_t const nbBytes = bitC->bitPos >> 3; + assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8); + MEM_writeLEST(bitC->ptr, bitC->bitContainer); + bitC->ptr += nbBytes; + assert(bitC->ptr <= bitC->endPtr); + bitC->bitPos &= 7; + bitC->bitContainer >>= nbBytes*8; +} + +/*! BIT_flushBits() : + * assumption : bitContainer has not overflowed + * safe version; check for buffer overflow, and prevents it. + * note : does not signal buffer overflow. + * overflow will be revealed later on using BIT_closeCStream() */ +MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC) +{ + size_t const nbBytes = bitC->bitPos >> 3; + assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8); + MEM_writeLEST(bitC->ptr, bitC->bitContainer); + bitC->ptr += nbBytes; + if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; + bitC->bitPos &= 7; + bitC->bitContainer >>= nbBytes*8; +} + +/*! BIT_closeCStream() : + * @return : size of CStream, in bytes, + * or 0 if it could not fit into dstBuffer */ +MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC) +{ + BIT_addBitsFast(bitC, 1, 1); /* endMark */ + BIT_flushBits(bitC); + if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */ + return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); +} + + +/*-******************************************************** +* bitStream decoding +**********************************************************/ +/*! BIT_initDStream() : + * Initialize a BIT_DStream_t. + * `bitD` : a pointer to an already allocated BIT_DStream_t structure. + * `srcSize` must be the *exact* size of the bitStream, in bytes. + * @return : size of stream (== srcSize), or an errorCode if a problem is detected + */ +MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize) +{ + if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); } + + bitD->start = (const char*)srcBuffer; + bitD->limitPtr = bitD->start + sizeof(bitD->bitContainer); + + if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */ + bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); + bitD->bitContainer = MEM_readLEST(bitD->ptr); + { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; + bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ + if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } + } else { + bitD->ptr = bitD->start; + bitD->bitContainer = *(const BYTE*)(bitD->start); + switch(srcSize) + { + case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); + /* fall-through */ + + case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); + /* fall-through */ + + case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); + /* fall-through */ + + case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; + /* fall-through */ + + case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; + /* fall-through */ + + case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; + /* fall-through */ + + default: break; + } + { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; + bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; + if (lastByte == 0) return ERROR(corruption_detected); /* endMark not present */ + } + bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8; + } + + return srcSize; +} + +MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start) +{ + return bitContainer >> start; +} + +MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) +{ +#if defined(__BMI__) && defined(__GNUC__) && __GNUC__*1000+__GNUC_MINOR__ >= 4008 /* experimental */ +# if defined(__x86_64__) + if (sizeof(bitContainer)==8) + return _bextr_u64(bitContainer, start, nbBits); + else +# endif + return _bextr_u32(bitContainer, start, nbBits); +#else + assert(nbBits < BIT_MASK_SIZE); + return (bitContainer >> start) & BIT_mask[nbBits]; +#endif +} + +MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) +{ + assert(nbBits < BIT_MASK_SIZE); + return bitContainer & BIT_mask[nbBits]; +} + +/*! BIT_lookBits() : + * Provides next n bits from local register. + * local register is not modified. + * On 32-bits, maxNbBits==24. + * On 64-bits, maxNbBits==56. + * @return : value extracted */ +MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) +{ +#if defined(__BMI__) && defined(__GNUC__) /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */ + return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits); +#else + U32 const regMask = sizeof(bitD->bitContainer)*8 - 1; + return ((bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> 1) >> ((regMask-nbBits) & regMask); +#endif +} + +/*! BIT_lookBitsFast() : + * unsafe version; only works if nbBits >= 1 */ +MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) +{ + U32 const regMask = sizeof(bitD->bitContainer)*8 - 1; + assert(nbBits >= 1); + return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask); +} + +MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) +{ + bitD->bitsConsumed += nbBits; +} + +/*! BIT_readBits() : + * Read (consume) next n bits from local register and update. + * Pay attention to not read more than nbBits contained into local register. + * @return : extracted value. */ +MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits) +{ + size_t const value = BIT_lookBits(bitD, nbBits); + BIT_skipBits(bitD, nbBits); + return value; +} + +/*! BIT_readBitsFast() : + * unsafe version; only works only if nbBits >= 1 */ +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits) +{ + size_t const value = BIT_lookBitsFast(bitD, nbBits); + assert(nbBits >= 1); + BIT_skipBits(bitD, nbBits); + return value; +} + +/*! BIT_reloadDStream() : + * Refill `bitD` from buffer previously set in BIT_initDStream() . + * This function is safe, it guarantees it will not read beyond src buffer. + * @return : status of `BIT_DStream_t` internal register. + * when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */ +MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) +{ + if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* overflow detected, like end of stream */ + return BIT_DStream_overflow; + + if (bitD->ptr >= bitD->limitPtr) { + bitD->ptr -= bitD->bitsConsumed >> 3; + bitD->bitsConsumed &= 7; + bitD->bitContainer = MEM_readLEST(bitD->ptr); + return BIT_DStream_unfinished; + } + if (bitD->ptr == bitD->start) { + if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer; + return BIT_DStream_completed; + } + /* start < ptr < limitPtr */ + { U32 nbBytes = bitD->bitsConsumed >> 3; + BIT_DStream_status result = BIT_DStream_unfinished; + if (bitD->ptr - nbBytes < bitD->start) { + nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */ + result = BIT_DStream_endOfBuffer; + } + bitD->ptr -= nbBytes; + bitD->bitsConsumed -= nbBytes*8; + bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD->bitContainer), otherwise bitD->ptr == bitD->start */ + return result; + } +} + +/*! BIT_endOfDStream() : + * @return : 1 if DStream has _exactly_ reached its end (all bits consumed). + */ +MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) +{ + return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8)); +} + +#if defined (__cplusplus) +} +#endif + +#endif /* BITSTREAM_H_MODULE */ diff --git a/src/borg/algorithms/zstd/lib/common/compiler.h b/src/borg/algorithms/zstd/lib/common/compiler.h new file mode 100644 index 0000000000..3a7553c380 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/common/compiler.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPILER_H +#define ZSTD_COMPILER_H + +/*-******************************************************* +* Compiler specifics +*********************************************************/ +/* force inlining */ +#if defined (__GNUC__) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# define INLINE_KEYWORD inline +#else +# define INLINE_KEYWORD +#endif + +#if defined(__GNUC__) +# define FORCE_INLINE_ATTR __attribute__((always_inline)) +#elif defined(_MSC_VER) +# define FORCE_INLINE_ATTR __forceinline +#else +# define FORCE_INLINE_ATTR +#endif + +/** + * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant + * parameters. They must be inlined for the compiler to elimininate the constant + * branches. + */ +#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR +/** + * HINT_INLINE is used to help the compiler generate better code. It is *not* + * used for "templates", so it can be tweaked based on the compilers + * performance. + * + * gcc-4.8 and gcc-4.9 have been shown to benefit from leaving off the + * always_inline attribute. + * + * clang up to 5.0.0 (trunk) benefit tremendously from the always_inline + * attribute. + */ +#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5 +# define HINT_INLINE static INLINE_KEYWORD +#else +# define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR +#endif + +/* force no inlining */ +#ifdef _MSC_VER +# define FORCE_NOINLINE static __declspec(noinline) +#else +# ifdef __GNUC__ +# define FORCE_NOINLINE static __attribute__((__noinline__)) +# else +# define FORCE_NOINLINE static +# endif +#endif + +/* prefetch */ +#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */ +# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# define PREFETCH(ptr) _mm_prefetch((const char*)ptr, _MM_HINT_T0) +#elif defined(__GNUC__) +# define PREFETCH(ptr) __builtin_prefetch(ptr, 0, 0) +#else +# define PREFETCH(ptr) /* disabled */ +#endif + +/* disable warnings */ +#ifdef _MSC_VER /* Visual Studio */ +# include /* For Visual 2005 */ +# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ +# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ +# pragma warning(disable : 4324) /* disable: C4324: padded structure */ +#endif + +#endif /* ZSTD_COMPILER_H */ diff --git a/src/borg/algorithms/zstd/lib/common/entropy_common.c b/src/borg/algorithms/zstd/lib/common/entropy_common.c new file mode 100644 index 0000000000..b37a082fee --- /dev/null +++ b/src/borg/algorithms/zstd/lib/common/entropy_common.c @@ -0,0 +1,221 @@ +/* + Common functions of New Generation Entropy library + Copyright (C) 2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +*************************************************************************** */ + +/* ************************************* +* Dependencies +***************************************/ +#include "mem.h" +#include "error_private.h" /* ERR_*, ERROR */ +#define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */ +#include "fse.h" +#define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */ +#include "huf.h" + + +/*=== Version ===*/ +unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; } + + +/*=== Error Management ===*/ +unsigned FSE_isError(size_t code) { return ERR_isError(code); } +const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); } + +unsigned HUF_isError(size_t code) { return ERR_isError(code); } +const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } + + +/*-************************************************************** +* FSE NCount encoding-decoding +****************************************************************/ +size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + const BYTE* const istart = (const BYTE*) headerBuffer; + const BYTE* const iend = istart + hbSize; + const BYTE* ip = istart; + int nbBits; + int remaining; + int threshold; + U32 bitStream; + int bitCount; + unsigned charnum = 0; + int previous0 = 0; + + if (hbSize < 4) return ERROR(srcSize_wrong); + bitStream = MEM_readLE32(ip); + nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */ + if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); + bitStream >>= 4; + bitCount = 4; + *tableLogPtr = nbBits; + remaining = (1<1) & (charnum<=*maxSVPtr)) { + if (previous0) { + unsigned n0 = charnum; + while ((bitStream & 0xFFFF) == 0xFFFF) { + n0 += 24; + if (ip < iend-5) { + ip += 2; + bitStream = MEM_readLE32(ip) >> bitCount; + } else { + bitStream >>= 16; + bitCount += 16; + } } + while ((bitStream & 3) == 3) { + n0 += 3; + bitStream >>= 2; + bitCount += 2; + } + n0 += bitStream & 3; + bitCount += 2; + if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall); + while (charnum < n0) normalizedCounter[charnum++] = 0; + if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + ip += bitCount>>3; + bitCount &= 7; + bitStream = MEM_readLE32(ip) >> bitCount; + } else { + bitStream >>= 2; + } } + { int const max = (2*threshold-1) - remaining; + int count; + + if ((bitStream & (threshold-1)) < (U32)max) { + count = bitStream & (threshold-1); + bitCount += nbBits-1; + } else { + count = bitStream & (2*threshold-1); + if (count >= threshold) count -= max; + bitCount += nbBits; + } + + count--; /* extra accuracy */ + remaining -= count < 0 ? -count : count; /* -1 means +1 */ + normalizedCounter[charnum++] = (short)count; + previous0 = !count; + while (remaining < threshold) { + nbBits--; + threshold >>= 1; + } + + if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + ip += bitCount>>3; + bitCount &= 7; + } else { + bitCount -= (int)(8 * (iend - 4 - ip)); + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> (bitCount & 31); + } } /* while ((remaining>1) & (charnum<=*maxSVPtr)) */ + if (remaining != 1) return ERROR(corruption_detected); + if (bitCount > 32) return ERROR(corruption_detected); + *maxSVPtr = charnum-1; + + ip += (bitCount+7)>>3; + return ip-istart; +} + + +/*! HUF_readStats() : + Read compact Huffman tree, saved by HUF_writeCTable(). + `huffWeight` is destination buffer. + `rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32. + @return : size read from `src` , or an error Code . + Note : Needed by HUF_readCTable() and HUF_readDTableX?() . +*/ +size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize) +{ + U32 weightTotal; + const BYTE* ip = (const BYTE*) src; + size_t iSize; + size_t oSize; + + if (!srcSize) return ERROR(srcSize_wrong); + iSize = ip[0]; + /* memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */ + + if (iSize >= 128) { /* special header */ + oSize = iSize - 127; + iSize = ((oSize+1)/2); + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + if (oSize >= hwSize) return ERROR(corruption_detected); + ip += 1; + { U32 n; + for (n=0; n> 4; + huffWeight[n+1] = ip[n/2] & 15; + } } } + else { /* header compressed with FSE (normal case) */ + FSE_DTable fseWorkspace[FSE_DTABLE_SIZE_U32(6)]; /* 6 is max possible tableLog for HUF header (maybe even 5, to be tested) */ + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + oSize = FSE_decompress_wksp(huffWeight, hwSize-1, ip+1, iSize, fseWorkspace, 6); /* max (hwSize-1) values decoded, as last one is implied */ + if (FSE_isError(oSize)) return oSize; + } + + /* collect weight stats */ + memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32)); + weightTotal = 0; + { U32 n; for (n=0; n= HUF_TABLELOG_MAX) return ERROR(corruption_detected); + rankStats[huffWeight[n]]++; + weightTotal += (1 << huffWeight[n]) >> 1; + } } + if (weightTotal == 0) return ERROR(corruption_detected); + + /* get last non-null symbol weight (implied, total must be 2^n) */ + { U32 const tableLog = BIT_highbit32(weightTotal) + 1; + if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected); + *tableLogPtr = tableLog; + /* determine last weight */ + { U32 const total = 1 << tableLog; + U32 const rest = total - weightTotal; + U32 const verif = 1 << BIT_highbit32(rest); + U32 const lastWeight = BIT_highbit32(rest) + 1; + if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ + huffWeight[oSize] = (BYTE)lastWeight; + rankStats[lastWeight]++; + } } + + /* check tree construction validity */ + if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ + + /* results */ + *nbSymbolsPtr = (U32)(oSize+1); + return iSize+1; +} diff --git a/src/borg/algorithms/zstd/lib/common/error_private.c b/src/borg/algorithms/zstd/lib/common/error_private.c new file mode 100644 index 0000000000..11f7cdab1c --- /dev/null +++ b/src/borg/algorithms/zstd/lib/common/error_private.c @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* The purpose of this file is to have a single list of error strings embedded in binary */ + +#include "error_private.h" + +const char* ERR_getErrorString(ERR_enum code) +{ + static const char* const notErrorCode = "Unspecified error code"; + switch( code ) + { + case PREFIX(no_error): return "No error detected"; + case PREFIX(GENERIC): return "Error (generic)"; + case PREFIX(prefix_unknown): return "Unknown frame descriptor"; + case PREFIX(version_unsupported): return "Version not supported"; + case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; + case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding"; + case PREFIX(corruption_detected): return "Corrupted block detected"; + case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; + case PREFIX(parameter_unsupported): return "Unsupported parameter"; + case PREFIX(parameter_outOfBound): return "Parameter is out of bound"; + case PREFIX(init_missing): return "Context should be init first"; + case PREFIX(memory_allocation): return "Allocation error : not enough memory"; + case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; + case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; + case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; + case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; + case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; + case PREFIX(dictionary_wrong): return "Dictionary mismatch"; + case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples"; + case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; + case PREFIX(srcSize_wrong): return "Src size is incorrect"; + /* following error codes are not stable and may be removed or changed in a future version */ + case PREFIX(frameIndex_tooLarge): return "Frame index is too large"; + case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking"; + case PREFIX(maxCode): + default: return notErrorCode; + } +} diff --git a/src/borg/algorithms/zstd/lib/common/error_private.h b/src/borg/algorithms/zstd/lib/common/error_private.h new file mode 100644 index 0000000000..0d2fa7e34b --- /dev/null +++ b/src/borg/algorithms/zstd/lib/common/error_private.h @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* Note : this module is expected to remain private, do not expose it */ + +#ifndef ERROR_H_MODULE +#define ERROR_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* **************************************** +* Dependencies +******************************************/ +#include /* size_t */ +#include "zstd_errors.h" /* enum list */ + + +/* **************************************** +* Compiler-specific +******************************************/ +#if defined(__GNUC__) +# define ERR_STATIC static __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define ERR_STATIC static inline +#elif defined(_MSC_VER) +# define ERR_STATIC static __inline +#else +# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + + +/*-**************************************** +* Customization (error_public.h) +******************************************/ +typedef ZSTD_ErrorCode ERR_enum; +#define PREFIX(name) ZSTD_error_##name + + +/*-**************************************** +* Error codes handling +******************************************/ +#undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ +#define ERROR(name) ZSTD_ERROR(name) +#define ZSTD_ERROR(name) ((size_t)-PREFIX(name)) + +ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } + +ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } + + +/*-**************************************** +* Error Strings +******************************************/ + +const char* ERR_getErrorString(ERR_enum code); /* error_private.c */ + +ERR_STATIC const char* ERR_getErrorName(size_t code) +{ + return ERR_getErrorString(ERR_getErrorCode(code)); +} + +#if defined (__cplusplus) +} +#endif + +#endif /* ERROR_H_MODULE */ diff --git a/src/borg/algorithms/zstd/lib/common/fse.h b/src/borg/algorithms/zstd/lib/common/fse.h new file mode 100644 index 0000000000..afd7801963 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/common/fse.h @@ -0,0 +1,704 @@ +/* ****************************************************************** + FSE : Finite State Entropy codec + Public Prototypes declaration + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Source repository : https://github.com/Cyan4973/FiniteStateEntropy +****************************************************************** */ + +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef FSE_H +#define FSE_H + + +/*-***************************************** +* Dependencies +******************************************/ +#include /* size_t, ptrdiff_t */ + + +/*-***************************************** +* FSE_PUBLIC_API : control library symbols visibility +******************************************/ +#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4) +# define FSE_PUBLIC_API __attribute__ ((visibility ("default"))) +#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */ +# define FSE_PUBLIC_API __declspec(dllexport) +#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1) +# define FSE_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define FSE_PUBLIC_API +#endif + +/*------ Version ------*/ +#define FSE_VERSION_MAJOR 0 +#define FSE_VERSION_MINOR 9 +#define FSE_VERSION_RELEASE 0 + +#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE +#define FSE_QUOTE(str) #str +#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str) +#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION) + +#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE) +FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */ + +/*-**************************************** +* FSE simple functions +******************************************/ +/*! FSE_compress() : + Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'. + 'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize). + @return : size of compressed data (<= dstCapacity). + Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! + if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead. + if FSE_isError(return), compression failed (more details using FSE_getErrorName()) +*/ +FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + +/*! FSE_decompress(): + Decompress FSE data from buffer 'cSrc', of size 'cSrcSize', + into already allocated destination buffer 'dst', of size 'dstCapacity'. + @return : size of regenerated data (<= maxDstSize), + or an error code, which can be tested using FSE_isError() . + + ** Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!! + Why ? : making this distinction requires a header. + Header management is intentionally delegated to the user layer, which can better manage special cases. +*/ +FSE_PUBLIC_API size_t FSE_decompress(void* dst, size_t dstCapacity, + const void* cSrc, size_t cSrcSize); + + +/*-***************************************** +* Tool functions +******************************************/ +FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */ + +/* Error Management */ +FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */ +FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */ + + +/*-***************************************** +* FSE advanced functions +******************************************/ +/*! FSE_compress2() : + Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog' + Both parameters can be defined as '0' to mean : use default value + @return : size of compressed data + Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!! + if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression. + if FSE_isError(return), it's an error code. +*/ +FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); + + +/*-***************************************** +* FSE detailed API +******************************************/ +/*! +FSE_compress() does the following: +1. count symbol occurrence from source[] into table count[] +2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog) +3. save normalized counters to memory buffer using writeNCount() +4. build encoding table 'CTable' from normalized counters +5. encode the data stream using encoding table 'CTable' + +FSE_decompress() does the following: +1. read normalized counters with readNCount() +2. build decoding table 'DTable' from normalized counters +3. decode the data stream using decoding table 'DTable' + +The following API allows targeting specific sub-functions for advanced tasks. +For example, it's possible to compress several blocks using the same 'CTable', +or to save and provide normalized distribution using external method. +*/ + +/* *** COMPRESSION *** */ + +/*! FSE_count(): + Provides the precise count of each byte within a table 'count'. + 'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1). + *maxSymbolValuePtr will be updated if detected smaller than initial value. + @return : the count of the most frequent symbol (which is not identified). + if return == srcSize, there is only one symbol. + Can also return an error code, which can be tested with FSE_isError(). */ +FSE_PUBLIC_API size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); + +/*! FSE_optimalTableLog(): + dynamically downsize 'tableLog' when conditions are met. + It saves CPU time, by using smaller tables, while preserving or even improving compression ratio. + @return : recommended tableLog (necessarily <= 'maxTableLog') */ +FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); + +/*! FSE_normalizeCount(): + normalize counts so that sum(count[]) == Power_of_2 (2^tableLog) + 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1). + @return : tableLog, + or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue); + +/*! FSE_NCountWriteBound(): + Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'. + Typically useful for allocation purpose. */ +FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_writeNCount(): + Compactly save 'normalizedCounter' into 'buffer'. + @return : size of the compressed table, + or an errorCode, which can be tested using FSE_isError(). */ +FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + + +/*! Constructor and Destructor of FSE_CTable. + Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */ +typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */ +FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog); +FSE_PUBLIC_API void FSE_freeCTable (FSE_CTable* ct); + +/*! FSE_buildCTable(): + Builds `ct`, which must be already allocated, using FSE_createCTable(). + @return : 0, or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_compress_usingCTable(): + Compress `src` using `ct` into `dst` which must be already allocated. + @return : size of compressed data (<= `dstCapacity`), + or 0 if compressed data could not fit into `dst`, + or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct); + +/*! +Tutorial : +---------- +The first step is to count all symbols. FSE_count() does this job very fast. +Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells. +'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0] +maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value) +FSE_count() will return the number of occurrence of the most frequent symbol. +This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). + +The next step is to normalize the frequencies. +FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'. +It also guarantees a minimum of 1 to any Symbol with frequency >= 1. +You can use 'tableLog'==0 to mean "use default tableLog value". +If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(), +which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default"). + +The result of FSE_normalizeCount() will be saved into a table, +called 'normalizedCounter', which is a table of signed short. +'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells. +The return value is tableLog if everything proceeded as expected. +It is 0 if there is a single symbol within distribution. +If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()). + +'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount(). +'buffer' must be already allocated. +For guaranteed success, buffer size must be at least FSE_headerBound(). +The result of the function is the number of bytes written into 'buffer'. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small). + +'normalizedCounter' can then be used to create the compression table 'CTable'. +The space required by 'CTable' must be already allocated, using FSE_createCTable(). +You can then use FSE_buildCTable() to fill 'CTable'. +If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()). + +'CTable' can then be used to compress 'src', with FSE_compress_usingCTable(). +Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize' +The function returns the size of compressed data (without header), necessarily <= `dstCapacity`. +If it returns '0', compressed data could not fit into 'dst'. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). +*/ + + +/* *** DECOMPRESSION *** */ + +/*! FSE_readNCount(): + Read compactly saved 'normalizedCounter' from 'rBuffer'. + @return : size read from 'rBuffer', + or an errorCode, which can be tested using FSE_isError(). + maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ +FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize); + +/*! Constructor and Destructor of FSE_DTable. + Note that its size depends on 'tableLog' */ +typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ +FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog); +FSE_PUBLIC_API void FSE_freeDTable(FSE_DTable* dt); + +/*! FSE_buildDTable(): + Builds 'dt', which must be already allocated, using FSE_createDTable(). + return : 0, or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_decompress_usingDTable(): + Decompress compressed source `cSrc` of size `cSrcSize` using `dt` + into `dst` which must be already allocated. + @return : size of regenerated data (necessarily <= `dstCapacity`), + or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt); + +/*! +Tutorial : +---------- +(Note : these functions only decompress FSE-compressed blocks. + If block is uncompressed, use memcpy() instead + If block is a single repeated byte, use memset() instead ) + +The first step is to obtain the normalized frequencies of symbols. +This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount(). +'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short. +In practice, that means it's necessary to know 'maxSymbolValue' beforehand, +or size the table to handle worst case situations (typically 256). +FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'. +The result of FSE_readNCount() is the number of bytes read from 'rBuffer'. +Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that. +If there is an error, the function will return an error code, which can be tested using FSE_isError(). + +The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'. +This is performed by the function FSE_buildDTable(). +The space required by 'FSE_DTable' must be already allocated using FSE_createDTable(). +If there is an error, the function will return an error code, which can be tested using FSE_isError(). + +`FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable(). +`cSrcSize` must be strictly correct, otherwise decompression will fail. +FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`). +If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small) +*/ + +#endif /* FSE_H */ + +#if defined(FSE_STATIC_LINKING_ONLY) && !defined(FSE_H_FSE_STATIC_LINKING_ONLY) +#define FSE_H_FSE_STATIC_LINKING_ONLY + +/* *** Dependency *** */ +#include "bitstream.h" + + +/* ***************************************** +* Static allocation +*******************************************/ +/* FSE buffer bounds */ +#define FSE_NCOUNTBOUND 512 +#define FSE_BLOCKBOUND(size) (size + (size>>7)) +#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */ +#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2)) +#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<= `1024` unsigned + */ +size_t FSE_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, unsigned* workSpace); + +/** FSE_countFast() : + * same as FSE_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr + */ +size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); + +/* FSE_countFast_wksp() : + * Same as FSE_countFast(), but using an externally provided scratch buffer. + * `workSpace` must be a table of minimum `1024` unsigned + */ +size_t FSE_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned* workSpace); + +/*! FSE_count_simple + * Same as FSE_countFast(), but does not use any additional memory (not even on stack). + * This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr` (presuming it's also the size of `count`). +*/ +size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); + + + +unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); +/**< same as FSE_optimalTableLog(), which used `minus==2` */ + +/* FSE_compress_wksp() : + * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). + * FSE_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable. + */ +#define FSE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) ) +size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); + +size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits); +/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */ + +size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue); +/**< build a fake FSE_CTable, designed to compress always the same symbolValue */ + +/* FSE_buildCTable_wksp() : + * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). + * `wkspSize` must be >= `(1<= BIT_DStream_completed + +When it's done, verify decompression is fully completed, by checking both DStream and the relevant states. +Checking if DStream has reached its end is performed by : + BIT_endOfDStream(&DStream); +Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible. + FSE_endOfDState(&DState); +*/ + + +/* ***************************************** +* FSE unsafe API +*******************************************/ +static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD); +/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ + + +/* ***************************************** +* Implementation of inlined functions +*******************************************/ +typedef struct { + int deltaFindState; + U32 deltaNbBits; +} FSE_symbolCompressionTransform; /* total 8 bytes */ + +MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct) +{ + const void* ptr = ct; + const U16* u16ptr = (const U16*) ptr; + const U32 tableLog = MEM_read16(ptr); + statePtr->value = (ptrdiff_t)1<stateTable = u16ptr+2; + statePtr->symbolTT = ((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1)); + statePtr->stateLog = tableLog; +} + + +/*! FSE_initCState2() : +* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read) +* uses the smallest state value possible, saving the cost of this symbol */ +MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol) +{ + FSE_initCState(statePtr, ct); + { const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; + const U16* stateTable = (const U16*)(statePtr->stateTable); + U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16); + statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits; + statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; + } +} + +MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, U32 symbol) +{ + FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; + const U16* const stateTable = (const U16*)(statePtr->stateTable); + U32 const nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16); + BIT_addBits(bitC, statePtr->value, nbBitsOut); + statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; +} + +MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr) +{ + BIT_addBits(bitC, statePtr->value, statePtr->stateLog); + BIT_flushBits(bitC); +} + + +/* ====== Decompression ====== */ + +typedef struct { + U16 tableLog; + U16 fastMode; +} FSE_DTableHeader; /* sizeof U32 */ + +typedef struct +{ + unsigned short newState; + unsigned char symbol; + unsigned char nbBits; +} FSE_decode_t; /* size == U32 */ + +MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt) +{ + const void* ptr = dt; + const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr; + DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); + BIT_reloadDStream(bitD); + DStatePtr->table = dt + 1; +} + +MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + return DInfo.symbol; +} + +MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + size_t const lowBits = BIT_readBits(bitD, nbBits); + DStatePtr->state = DInfo.newState + lowBits; +} + +MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BIT_readBits(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +/*! FSE_decodeSymbolFast() : + unsafe, only works if no symbol has a probability > 50% */ +MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BIT_readBitsFast(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) +{ + return DStatePtr->state == 0; +} + + + +#ifndef FSE_COMMONDEFS_ONLY + +/* ************************************************************** +* Tuning parameters +****************************************************************/ +/*!MEMORY_USAGE : +* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) +* Increasing memory usage improves compression ratio +* Reduced memory usage can improve speed, due to cache effect +* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ +#ifndef FSE_MAX_MEMORY_USAGE +# define FSE_MAX_MEMORY_USAGE 14 +#endif +#ifndef FSE_DEFAULT_MEMORY_USAGE +# define FSE_DEFAULT_MEMORY_USAGE 13 +#endif + +/*!FSE_MAX_SYMBOL_VALUE : +* Maximum symbol value authorized. +* Required for proper stack allocation */ +#ifndef FSE_MAX_SYMBOL_VALUE +# define FSE_MAX_SYMBOL_VALUE 255 +#endif + +/* ************************************************************** +* template functions type & suffix +****************************************************************/ +#define FSE_FUNCTION_TYPE BYTE +#define FSE_FUNCTION_EXTENSION +#define FSE_DECODE_TYPE FSE_decode_t + + +#endif /* !FSE_COMMONDEFS_ONLY */ + + +/* *************************************************************** +* Constants +*****************************************************************/ +#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2) +#define FSE_MAX_TABLESIZE (1U< FSE_TABLELOG_ABSOLUTE_MAX +# error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported" +#endif + +#define FSE_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3) + + +#endif /* FSE_STATIC_LINKING_ONLY */ + + +#if defined (__cplusplus) +} +#endif diff --git a/src/borg/algorithms/zstd/lib/common/fse_decompress.c b/src/borg/algorithms/zstd/lib/common/fse_decompress.c new file mode 100644 index 0000000000..8e3f0035f6 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/common/fse_decompress.c @@ -0,0 +1,309 @@ +/* ****************************************************************** + FSE : Finite State Entropy decoder + Copyright (C) 2013-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + + +/* ************************************************************** +* Includes +****************************************************************/ +#include /* malloc, free, qsort */ +#include /* memcpy, memset */ +#include "bitstream.h" +#include "compiler.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#include "error_private.h" + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define FSE_isError ERR_isError +#define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + +/* check and forward error code */ +#define CHECK_F(f) { size_t const e = f; if (FSE_isError(e)) return e; } + + +/* ************************************************************** +* Templates +****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ + +/* safety checks */ +#ifndef FSE_FUNCTION_EXTENSION +# error "FSE_FUNCTION_EXTENSION must be defined" +#endif +#ifndef FSE_FUNCTION_TYPE +# error "FSE_FUNCTION_TYPE must be defined" +#endif + +/* Function names */ +#define FSE_CAT(X,Y) X##Y +#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) +#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) + + +/* Function templates */ +FSE_DTable* FSE_createDTable (unsigned tableLog) +{ + if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; + return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); +} + +void FSE_freeDTable (FSE_DTable* dt) +{ + free(dt); +} + +size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +{ + void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ + FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr); + U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1]; + + U32 const maxSV1 = maxSymbolValue + 1; + U32 const tableSize = 1 << tableLog; + U32 highThreshold = tableSize-1; + + /* Sanity Checks */ + if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); + + /* Init, lay down lowprob symbols */ + { FSE_DTableHeader DTableH; + DTableH.tableLog = (U16)tableLog; + DTableH.fastMode = 1; + { S16 const largeLimit= (S16)(1 << (tableLog-1)); + U32 s; + for (s=0; s= largeLimit) DTableH.fastMode=0; + symbolNext[s] = normalizedCounter[s]; + } } } + memcpy(dt, &DTableH, sizeof(DTableH)); + } + + /* Spread symbols */ + { U32 const tableMask = tableSize-1; + U32 const step = FSE_TABLESTEP(tableSize); + U32 s, position = 0; + for (s=0; s highThreshold) position = (position + step) & tableMask; /* lowprob area */ + } } + if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ + } + + /* Build Decoding table */ + { U32 u; + for (u=0; utableLog = 0; + DTableH->fastMode = 0; + + cell->newState = 0; + cell->symbol = symbolValue; + cell->nbBits = 0; + + return 0; +} + + +size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits) +{ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + void* dPtr = dt + 1; + FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr; + const unsigned tableSize = 1 << nbBits; + const unsigned tableMask = tableSize - 1; + const unsigned maxSV1 = tableMask+1; + unsigned s; + + /* Sanity checks */ + if (nbBits < 1) return ERROR(GENERIC); /* min size */ + + /* Build Decoding Table */ + DTableH->tableLog = (U16)nbBits; + DTableH->fastMode = 1; + for (s=0; s sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[1] = FSE_GETSYMBOL(&state2); + + if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + { if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } } + + op[2] = FSE_GETSYMBOL(&state1); + + if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[3] = FSE_GETSYMBOL(&state2); + } + + /* tail */ + /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ + while (1) { + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + *op++ = FSE_GETSYMBOL(&state1); + if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { + *op++ = FSE_GETSYMBOL(&state2); + break; + } + + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + *op++ = FSE_GETSYMBOL(&state2); + if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { + *op++ = FSE_GETSYMBOL(&state1); + break; + } } + + return op-ostart; +} + + +size_t FSE_decompress_usingDTable(void* dst, size_t originalSize, + const void* cSrc, size_t cSrcSize, + const FSE_DTable* dt) +{ + const void* ptr = dt; + const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; + const U32 fastMode = DTableH->fastMode; + + /* select fast mode (static) */ + if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); + return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); +} + + +size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, FSE_DTable* workSpace, unsigned maxLog) +{ + const BYTE* const istart = (const BYTE*)cSrc; + const BYTE* ip = istart; + short counting[FSE_MAX_SYMBOL_VALUE+1]; + unsigned tableLog; + unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; + + /* normal FSE decoding mode */ + size_t const NCountLength = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize); + if (FSE_isError(NCountLength)) return NCountLength; + //if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size; supposed to be already checked in NCountLength, only remaining case : NCountLength==cSrcSize */ + if (tableLog > maxLog) return ERROR(tableLog_tooLarge); + ip += NCountLength; + cSrcSize -= NCountLength; + + CHECK_F( FSE_buildDTable (workSpace, counting, maxSymbolValue, tableLog) ); + + return FSE_decompress_usingDTable (dst, dstCapacity, ip, cSrcSize, workSpace); /* always return, even if it is an error code */ +} + + +typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; + +size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize) +{ + DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */ + return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, dt, FSE_MAX_TABLELOG); +} + + + +#endif /* FSE_COMMONDEFS_ONLY */ diff --git a/src/borg/algorithms/zstd/lib/common/huf.h b/src/borg/algorithms/zstd/lib/common/huf.h new file mode 100644 index 0000000000..522bf9b6c0 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/common/huf.h @@ -0,0 +1,302 @@ +/* ****************************************************************** + Huffman coder, part of New Generation Entropy library + header file + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Source repository : https://github.com/Cyan4973/FiniteStateEntropy +****************************************************************** */ + +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef HUF_H_298734234 +#define HUF_H_298734234 + +/* *** Dependencies *** */ +#include /* size_t */ + + +/* *** library symbols visibility *** */ +/* Note : when linking with -fvisibility=hidden on gcc, or by default on Visual, + * HUF symbols remain "private" (internal symbols for library only). + * Set macro FSE_DLL_EXPORT to 1 if you want HUF symbols visible on DLL interface */ +#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4) +# define HUF_PUBLIC_API __attribute__ ((visibility ("default"))) +#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */ +# define HUF_PUBLIC_API __declspec(dllexport) +#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1) +# define HUF_PUBLIC_API __declspec(dllimport) /* not required, just to generate faster code (saves a function pointer load from IAT and an indirect jump) */ +#else +# define HUF_PUBLIC_API +#endif + + +/* *** simple functions *** */ +/** +HUF_compress() : + Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'. + 'dst' buffer must be already allocated. + Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize). + `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB. + @return : size of compressed data (<= `dstCapacity`). + Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! + if return == 1, srcData is a single repeated byte symbol (RLE compression). + if HUF_isError(return), compression failed (more details using HUF_getErrorName()) +*/ +HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + +/** +HUF_decompress() : + Decompress HUF data from buffer 'cSrc', of size 'cSrcSize', + into already allocated buffer 'dst', of minimum size 'dstSize'. + `originalSize` : **must** be the ***exact*** size of original (uncompressed) data. + Note : in contrast with FSE, HUF_decompress can regenerate + RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, + because it knows size to regenerate. + @return : size of regenerated data (== originalSize), + or an error code, which can be tested using HUF_isError() +*/ +HUF_PUBLIC_API size_t HUF_decompress(void* dst, size_t originalSize, + const void* cSrc, size_t cSrcSize); + + +/* *** Tool functions *** */ +#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ +HUF_PUBLIC_API size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ + +/* Error Management */ +HUF_PUBLIC_API unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ +HUF_PUBLIC_API const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ + + +/* *** Advanced function *** */ + +/** HUF_compress2() : + * Same as HUF_compress(), but offers direct control over `maxSymbolValue` and `tableLog`. + * `tableLog` must be `<= HUF_TABLELOG_MAX` . */ +HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); + +/** HUF_compress4X_wksp() : + * Same as HUF_compress2(), but uses externally allocated `workSpace`. + * `workspace` must have minimum alignment of 4, and be at least as large as following macro */ +#define HUF_WORKSPACE_SIZE (6 << 10) +#define HUF_WORKSPACE_SIZE_U32 (HUF_WORKSPACE_SIZE / sizeof(U32)) +HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); + +/** + * The minimum workspace size for the `workSpace` used in + * HUF_readDTableX2_wksp() and HUF_readDTableX4_wksp(). + * + * The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when + * HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15. + * Buffer overflow errors may potentially occur if code modifications result in + * a required workspace size greater than that specified in the following + * macro. + */ +#define HUF_DECOMPRESS_WORKSPACE_SIZE (2 << 10) +#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32)) + +#endif /* HUF_H_298734234 */ + +/* ****************************************************************** + * WARNING !! + * The following section contains advanced and experimental definitions + * which shall never be used in the context of dll + * because they are not guaranteed to remain stable in the future. + * Only consider them in association with static linking. + *******************************************************************/ +#if defined(HUF_STATIC_LINKING_ONLY) && !defined(HUF_H_HUF_STATIC_LINKING_ONLY) +#define HUF_H_HUF_STATIC_LINKING_ONLY + +/* *** Dependencies *** */ +#include "mem.h" /* U32 */ + + +/* *** Constants *** */ +#define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */ +#define HUF_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */ +#define HUF_SYMBOLVALUE_MAX 255 + +#define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ +#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX) +# error "HUF_TABLELOG_MAX is too large !" +#endif + + +/* **************************************** +* Static allocation +******************************************/ +/* HUF buffer bounds */ +#define HUF_CTABLEBOUND 129 +#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true when incompressible is pre-filtered with fast heuristic */ +#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* static allocation of HUF's Compression Table */ +#define HUF_CTABLE_SIZE_U32(maxSymbolValue) ((maxSymbolValue)+1) /* Use tables of U32, for proper alignment */ +#define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_U32(maxSymbolValue) * sizeof(U32)) +#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ + U32 name##hb[HUF_CTABLE_SIZE_U32(maxSymbolValue)]; \ + void* name##hv = &(name##hb); \ + HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */ + +/* static allocation of HUF's DTable */ +typedef U32 HUF_DTable; +#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog))) +#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \ + HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) } +#define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \ + HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) } + + +/* **************************************** +* Advanced decompression functions +******************************************/ +size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ +size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ + +size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */ +size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */ +size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< considers RLE and uncompressed as errors */ +size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ +size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */ +size_t HUF_decompress4X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ +size_t HUF_decompress4X4_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */ + + +/* **************************************** +* HUF detailed API +******************************************/ +/*! +HUF_compress() does the following: +1. count symbol occurrence from source[] into table count[] using FSE_count() +2. (optional) refine tableLog using HUF_optimalTableLog() +3. build Huffman table from count using HUF_buildCTable() +4. save Huffman table to memory buffer using HUF_writeCTable() +5. encode the data stream using HUF_compress4X_usingCTable() + +The following API allows targeting specific sub-functions for advanced tasks. +For example, it's possible to compress several blocks using the same 'CTable', +or to save and regenerate 'CTable' using external methods. +*/ +/* FSE_count() : find it within "fse.h" */ +unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); +typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */ +size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); +size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog); +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); + +typedef enum { + HUF_repeat_none, /**< Cannot use the previous table */ + HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */ + HUF_repeat_valid /**< Can use the previous table and it is asumed to be valid */ + } HUF_repeat; +/** HUF_compress4X_repeat() : +* Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. +* If it uses hufTable it does not modify hufTable or repeat. +* If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. +* If preferRepeat then the old table will always be used if valid. */ +size_t HUF_compress4X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */ + +/** HUF_buildCTable_wksp() : + * Same as HUF_buildCTable(), but using externally allocated scratch buffer. + * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned. + */ +size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize); + +/*! HUF_readStats() : + Read compact Huffman tree, saved by HUF_writeCTable(). + `huffWeight` is destination buffer. + @return : size read from `src` , or an error Code . + Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */ +size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize); + +/** HUF_readCTable() : +* Loading a CTable saved with HUF_writeCTable() */ +size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); + + +/* +HUF_decompress() does the following: +1. select the decompression algorithm (X2, X4) based on pre-computed heuristics +2. build Huffman table from save, using HUF_readDTableXn() +3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable +*/ + +/** HUF_selectDecoder() : +* Tells which decoder is likely to decode faster, +* based on a set of pre-determined metrics. +* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 . +* Assumption : 0 < cSrcSize < dstSize <= 128 KB */ +U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize); + +size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize); +size_t HUF_readDTableX2_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize); +size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize); +size_t HUF_readDTableX4_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize); + +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); + + +/* single stream variants */ + +size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); +size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */ +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); +/** HUF_compress1X_repeat() : +* Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. +* If it uses hufTable it does not modify hufTable or repeat. +* If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. +* If preferRepeat then the old table will always be used if valid. */ +size_t HUF_compress1X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */ + +size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ +size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ + +size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); +size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); +size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ +size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */ +size_t HUF_decompress1X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ +size_t HUF_decompress1X4_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */ + +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */ +size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); + +#endif /* HUF_STATIC_LINKING_ONLY */ + +#if defined (__cplusplus) +} +#endif diff --git a/src/borg/algorithms/zstd/lib/common/mem.h b/src/borg/algorithms/zstd/lib/common/mem.h new file mode 100644 index 0000000000..23335c3146 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/common/mem.h @@ -0,0 +1,360 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef MEM_H_MODULE +#define MEM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + +/*-**************************************** +* Dependencies +******************************************/ +#include /* size_t, ptrdiff_t */ +#include /* memcpy */ + + +/*-**************************************** +* Compiler specifics +******************************************/ +#if defined(_MSC_VER) /* Visual Studio */ +# include /* _byteswap_ulong */ +# include /* _byteswap_* */ +#endif +#if defined(__GNUC__) +# define MEM_STATIC static __inline __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define MEM_STATIC static inline +#elif defined(_MSC_VER) +# define MEM_STATIC static __inline +#else +# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + +/* code only tested on 32 and 64 bits systems */ +#define MEM_STATIC_ASSERT(c) { enum { MEM_static_assert = 1/(int)(!!(c)) }; } +MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } + + +/*-************************************************************** +* Basic Types +*****************************************************************/ +#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef int16_t S16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef int64_t S64; + typedef intptr_t iPtrDiff; + typedef uintptr_t uPtrDiff; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef signed short S16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; + typedef signed long long S64; + typedef ptrdiff_t iPtrDiff; + typedef size_t uPtrDiff; +#endif + + +/*-************************************************************** +* Memory I/O +*****************************************************************/ +/* MEM_FORCE_MEMORY_ACCESS : + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : `__packed` statement. It depends on compiler extension (i.e., not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. + * Method 2 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets depending on alignment. + * In some circumstances, it's the only known way to get the most performance (i.e. GCC + ARMv6) + * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1 > 2) + */ +#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +# define MEM_FORCE_MEMORY_ACCESS 2 +# elif defined(__INTEL_COMPILER) || defined(__GNUC__) +# define MEM_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; } +MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } + +MEM_STATIC unsigned MEM_isLittleEndian(void) +{ + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +} + +#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) + +/* violates C standard, by lying on structure alignment. +Only use if no other choice to achieve best performance on target platform */ +MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; } +MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; } +MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; } +MEM_STATIC size_t MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } + +#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32)) + __pragma( pack(push, 1) ) + typedef union { U16 u16; U32 u32; U64 u64; size_t st; } unalign; + __pragma( pack(pop) ) +#else + typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign; +#endif + +MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } +MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } +MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } +MEM_STATIC size_t MEM_readST(const void* ptr) { return ((const unalign*)ptr)->st; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; } + +#else + +/* default method, safe and standard. + can sometimes prove slower */ + +MEM_STATIC U16 MEM_read16(const void* memPtr) +{ + U16 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U32 MEM_read32(const void* memPtr) +{ + U32 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U64 MEM_read64(const void* memPtr) +{ + U64 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC size_t MEM_readST(const void* memPtr) +{ + size_t val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write32(void* memPtr, U32 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write64(void* memPtr, U64 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +#endif /* MEM_FORCE_MEMORY_ACCESS */ + +MEM_STATIC U32 MEM_swap32(U32 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_ulong(in); +#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403) + return __builtin_bswap32(in); +#else + return ((in << 24) & 0xff000000 ) | + ((in << 8) & 0x00ff0000 ) | + ((in >> 8) & 0x0000ff00 ) | + ((in >> 24) & 0x000000ff ); +#endif +} + +MEM_STATIC U64 MEM_swap64(U64 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_uint64(in); +#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403) + return __builtin_bswap64(in); +#else + return ((in << 56) & 0xff00000000000000ULL) | + ((in << 40) & 0x00ff000000000000ULL) | + ((in << 24) & 0x0000ff0000000000ULL) | + ((in << 8) & 0x000000ff00000000ULL) | + ((in >> 8) & 0x00000000ff000000ULL) | + ((in >> 24) & 0x0000000000ff0000ULL) | + ((in >> 40) & 0x000000000000ff00ULL) | + ((in >> 56) & 0x00000000000000ffULL); +#endif +} + +MEM_STATIC size_t MEM_swapST(size_t in) +{ + if (MEM_32bits()) + return (size_t)MEM_swap32((U32)in); + else + return (size_t)MEM_swap64((U64)in); +} + +/*=== Little endian r/w ===*/ + +MEM_STATIC U16 MEM_readLE16(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read16(memPtr); + else { + const BYTE* p = (const BYTE*)memPtr; + return (U16)(p[0] + (p[1]<<8)); + } +} + +MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) +{ + if (MEM_isLittleEndian()) { + MEM_write16(memPtr, val); + } else { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE)val; + p[1] = (BYTE)(val>>8); + } +} + +MEM_STATIC U32 MEM_readLE24(const void* memPtr) +{ + return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16); +} + +MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val) +{ + MEM_writeLE16(memPtr, (U16)val); + ((BYTE*)memPtr)[2] = (BYTE)(val>>16); +} + +MEM_STATIC U32 MEM_readLE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read32(memPtr); + else + return MEM_swap32(MEM_read32(memPtr)); +} + +MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, val32); + else + MEM_write32(memPtr, MEM_swap32(val32)); +} + +MEM_STATIC U64 MEM_readLE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read64(memPtr); + else + return MEM_swap64(MEM_read64(memPtr)); +} + +MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, val64); + else + MEM_write64(memPtr, MEM_swap64(val64)); +} + +MEM_STATIC size_t MEM_readLEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readLE32(memPtr); + else + return (size_t)MEM_readLE64(memPtr); +} + +MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeLE32(memPtr, (U32)val); + else + MEM_writeLE64(memPtr, (U64)val); +} + +/*=== Big endian r/w ===*/ + +MEM_STATIC U32 MEM_readBE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap32(MEM_read32(memPtr)); + else + return MEM_read32(memPtr); +} + +MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, MEM_swap32(val32)); + else + MEM_write32(memPtr, val32); +} + +MEM_STATIC U64 MEM_readBE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap64(MEM_read64(memPtr)); + else + return MEM_read64(memPtr); +} + +MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, MEM_swap64(val64)); + else + MEM_write64(memPtr, val64); +} + +MEM_STATIC size_t MEM_readBEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readBE32(memPtr); + else + return (size_t)MEM_readBE64(memPtr); +} + +MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeBE32(memPtr, (U32)val); + else + MEM_writeBE64(memPtr, (U64)val); +} + + +#if defined (__cplusplus) +} +#endif + +#endif /* MEM_H_MODULE */ diff --git a/src/borg/algorithms/zstd/lib/common/pool.c b/src/borg/algorithms/zstd/lib/common/pool.c new file mode 100644 index 0000000000..1b0fe1035d --- /dev/null +++ b/src/borg/algorithms/zstd/lib/common/pool.c @@ -0,0 +1,255 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* ====== Dependencies ======= */ +#include /* size_t */ +#include /* malloc, calloc, free */ +#include "pool.h" + +/* ====== Compiler specifics ====== */ +#if defined(_MSC_VER) +# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ +#endif + + +#ifdef ZSTD_MULTITHREAD + +#include "threading.h" /* pthread adaptation */ + +/* A job is a function and an opaque argument */ +typedef struct POOL_job_s { + POOL_function function; + void *opaque; +} POOL_job; + +struct POOL_ctx_s { + ZSTD_customMem customMem; + /* Keep track of the threads */ + ZSTD_pthread_t *threads; + size_t numThreads; + + /* The queue is a circular buffer */ + POOL_job *queue; + size_t queueHead; + size_t queueTail; + size_t queueSize; + + /* The number of threads working on jobs */ + size_t numThreadsBusy; + /* Indicates if the queue is empty */ + int queueEmpty; + + /* The mutex protects the queue */ + ZSTD_pthread_mutex_t queueMutex; + /* Condition variable for pushers to wait on when the queue is full */ + ZSTD_pthread_cond_t queuePushCond; + /* Condition variables for poppers to wait on when the queue is empty */ + ZSTD_pthread_cond_t queuePopCond; + /* Indicates if the queue is shutting down */ + int shutdown; +}; + +/* POOL_thread() : + Work thread for the thread pool. + Waits for jobs and executes them. + @returns : NULL on failure else non-null. +*/ +static void* POOL_thread(void* opaque) { + POOL_ctx* const ctx = (POOL_ctx*)opaque; + if (!ctx) { return NULL; } + for (;;) { + /* Lock the mutex and wait for a non-empty queue or until shutdown */ + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + + while (ctx->queueEmpty && !ctx->shutdown) { + ZSTD_pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex); + } + /* empty => shutting down: so stop */ + if (ctx->queueEmpty) { + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return opaque; + } + /* Pop a job off the queue */ + { POOL_job const job = ctx->queue[ctx->queueHead]; + ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize; + ctx->numThreadsBusy++; + ctx->queueEmpty = ctx->queueHead == ctx->queueTail; + /* Unlock the mutex, signal a pusher, and run the job */ + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + ZSTD_pthread_cond_signal(&ctx->queuePushCond); + + job.function(job.opaque); + + /* If the intended queue size was 0, signal after finishing job */ + if (ctx->queueSize == 1) { + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + ctx->numThreadsBusy--; + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + ZSTD_pthread_cond_signal(&ctx->queuePushCond); + } } + } /* for (;;) */ + /* Unreachable */ +} + +POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) { + return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem); +} + +POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem) { + POOL_ctx* ctx; + /* Check the parameters */ + if (!numThreads) { return NULL; } + /* Allocate the context and zero initialize */ + ctx = (POOL_ctx*)ZSTD_calloc(sizeof(POOL_ctx), customMem); + if (!ctx) { return NULL; } + /* Initialize the job queue. + * It needs one extra space since one space is wasted to differentiate empty + * and full queues. + */ + ctx->queueSize = queueSize + 1; + ctx->queue = (POOL_job*) malloc(ctx->queueSize * sizeof(POOL_job)); + ctx->queueHead = 0; + ctx->queueTail = 0; + ctx->numThreadsBusy = 0; + ctx->queueEmpty = 1; + (void)ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL); + (void)ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL); + (void)ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL); + ctx->shutdown = 0; + /* Allocate space for the thread handles */ + ctx->threads = (ZSTD_pthread_t*)ZSTD_malloc(numThreads * sizeof(ZSTD_pthread_t), customMem); + ctx->numThreads = 0; + ctx->customMem = customMem; + /* Check for errors */ + if (!ctx->threads || !ctx->queue) { POOL_free(ctx); return NULL; } + /* Initialize the threads */ + { size_t i; + for (i = 0; i < numThreads; ++i) { + if (ZSTD_pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) { + ctx->numThreads = i; + POOL_free(ctx); + return NULL; + } } + ctx->numThreads = numThreads; + } + return ctx; +} + +/*! POOL_join() : + Shutdown the queue, wake any sleeping threads, and join all of the threads. +*/ +static void POOL_join(POOL_ctx* ctx) { + /* Shut down the queue */ + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + ctx->shutdown = 1; + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + /* Wake up sleeping threads */ + ZSTD_pthread_cond_broadcast(&ctx->queuePushCond); + ZSTD_pthread_cond_broadcast(&ctx->queuePopCond); + /* Join all of the threads */ + { size_t i; + for (i = 0; i < ctx->numThreads; ++i) { + ZSTD_pthread_join(ctx->threads[i], NULL); + } } +} + +void POOL_free(POOL_ctx *ctx) { + if (!ctx) { return; } + POOL_join(ctx); + ZSTD_pthread_mutex_destroy(&ctx->queueMutex); + ZSTD_pthread_cond_destroy(&ctx->queuePushCond); + ZSTD_pthread_cond_destroy(&ctx->queuePopCond); + ZSTD_free(ctx->queue, ctx->customMem); + ZSTD_free(ctx->threads, ctx->customMem); + ZSTD_free(ctx, ctx->customMem); +} + +size_t POOL_sizeof(POOL_ctx *ctx) { + if (ctx==NULL) return 0; /* supports sizeof NULL */ + return sizeof(*ctx) + + ctx->queueSize * sizeof(POOL_job) + + ctx->numThreads * sizeof(ZSTD_pthread_t); +} + +/** + * Returns 1 if the queue is full and 0 otherwise. + * + * If the queueSize is 1 (the pool was created with an intended queueSize of 0), + * then a queue is empty if there is a thread free and no job is waiting. + */ +static int isQueueFull(POOL_ctx const* ctx) { + if (ctx->queueSize > 1) { + return ctx->queueHead == ((ctx->queueTail + 1) % ctx->queueSize); + } else { + return ctx->numThreadsBusy == ctx->numThreads || + !ctx->queueEmpty; + } +} + +void POOL_add(void* ctxVoid, POOL_function function, void *opaque) { + POOL_ctx* const ctx = (POOL_ctx*)ctxVoid; + if (!ctx) { return; } + + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + { POOL_job const job = {function, opaque}; + + /* Wait until there is space in the queue for the new job */ + while (isQueueFull(ctx) && !ctx->shutdown) { + ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex); + } + /* The queue is still going => there is space */ + if (!ctx->shutdown) { + ctx->queueEmpty = 0; + ctx->queue[ctx->queueTail] = job; + ctx->queueTail = (ctx->queueTail + 1) % ctx->queueSize; + } + } + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + ZSTD_pthread_cond_signal(&ctx->queuePopCond); +} + +#else /* ZSTD_MULTITHREAD not defined */ +/* No multi-threading support */ + +/* We don't need any data, but if it is empty malloc() might return NULL. */ +struct POOL_ctx_s { + int dummy; +}; +static POOL_ctx g_ctx; + +POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) { + return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem); +} + +POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem) { + (void)numThreads; + (void)queueSize; + (void)customMem; + return &g_ctx; +} + +void POOL_free(POOL_ctx* ctx) { + assert(!ctx || ctx == &g_ctx); + (void)ctx; +} + +void POOL_add(void* ctx, POOL_function function, void* opaque) { + (void)ctx; + function(opaque); +} + +size_t POOL_sizeof(POOL_ctx* ctx) { + if (ctx==NULL) return 0; /* supports sizeof NULL */ + assert(ctx == &g_ctx); + return sizeof(*ctx); +} + +#endif /* ZSTD_MULTITHREAD */ diff --git a/src/borg/algorithms/zstd/lib/common/pool.h b/src/borg/algorithms/zstd/lib/common/pool.h new file mode 100644 index 0000000000..08c63715aa --- /dev/null +++ b/src/borg/algorithms/zstd/lib/common/pool.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef POOL_H +#define POOL_H + +#if defined (__cplusplus) +extern "C" { +#endif + + +#include /* size_t */ +#include "zstd_internal.h" /* ZSTD_customMem */ + +typedef struct POOL_ctx_s POOL_ctx; + +/*! POOL_create() : + * Create a thread pool with at most `numThreads` threads. + * `numThreads` must be at least 1. + * The maximum number of queued jobs before blocking is `queueSize`. + * @return : POOL_ctx pointer on success, else NULL. +*/ +POOL_ctx *POOL_create(size_t numThreads, size_t queueSize); + +POOL_ctx *POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem); + +/*! POOL_free() : + Free a thread pool returned by POOL_create(). +*/ +void POOL_free(POOL_ctx *ctx); + +/*! POOL_sizeof() : + return memory usage of pool returned by POOL_create(). +*/ +size_t POOL_sizeof(POOL_ctx *ctx); + +/*! POOL_function : + The function type that can be added to a thread pool. +*/ +typedef void (*POOL_function)(void *); +/*! POOL_add_function : + The function type for a generic thread pool add function. +*/ +typedef void (*POOL_add_function)(void *, POOL_function, void *); + +/*! POOL_add() : + Add the job `function(opaque)` to the thread pool. + Possibly blocks until there is room in the queue. + Note : The function may be executed asynchronously, so `opaque` must live until the function has been completed. +*/ +void POOL_add(void *ctx, POOL_function function, void *opaque); + + +#if defined (__cplusplus) +} +#endif + +#endif diff --git a/src/borg/algorithms/zstd/lib/common/threading.c b/src/borg/algorithms/zstd/lib/common/threading.c new file mode 100644 index 0000000000..8be8c8da94 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/common/threading.c @@ -0,0 +1,75 @@ +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + */ + +/** + * This file will hold wrapper for systems, which do not support pthreads + */ + +/* create fake symbol to avoid empty trnaslation unit warning */ +int g_ZSTD_threading_useles_symbol; + +#if defined(ZSTD_MULTITHREAD) && defined(_WIN32) + +/** + * Windows minimalist Pthread Wrapper, based on : + * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html + */ + + +/* === Dependencies === */ +#include +#include +#include "threading.h" + + +/* === Implementation === */ + +static unsigned __stdcall worker(void *arg) +{ + ZSTD_pthread_t* const thread = (ZSTD_pthread_t*) arg; + thread->arg = thread->start_routine(thread->arg); + return 0; +} + +int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused, + void* (*start_routine) (void*), void* arg) +{ + (void)unused; + thread->arg = arg; + thread->start_routine = start_routine; + thread->handle = (HANDLE) _beginthreadex(NULL, 0, worker, thread, 0, NULL); + + if (!thread->handle) + return errno; + else + return 0; +} + +int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr) +{ + DWORD result; + + if (!thread.handle) return 0; + + result = WaitForSingleObject(thread.handle, INFINITE); + switch (result) { + case WAIT_OBJECT_0: + if (value_ptr) *value_ptr = thread.arg; + return 0; + case WAIT_ABANDONED: + return EINVAL; + default: + return GetLastError(); + } +} + +#endif /* ZSTD_MULTITHREAD */ diff --git a/src/borg/algorithms/zstd/lib/common/threading.h b/src/borg/algorithms/zstd/lib/common/threading.h new file mode 100644 index 0000000000..197770db27 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/common/threading.h @@ -0,0 +1,123 @@ +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + */ + +#ifndef THREADING_H_938743 +#define THREADING_H_938743 + +#if defined (__cplusplus) +extern "C" { +#endif + +#if defined(ZSTD_MULTITHREAD) && defined(_WIN32) + +/** + * Windows minimalist Pthread Wrapper, based on : + * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html + */ +#ifdef WINVER +# undef WINVER +#endif +#define WINVER 0x0600 + +#ifdef _WIN32_WINNT +# undef _WIN32_WINNT +#endif +#define _WIN32_WINNT 0x0600 + +#ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +#endif + +#undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ +#include +#undef ERROR +#define ERROR(name) ZSTD_ERROR(name) + + +/* mutex */ +#define ZSTD_pthread_mutex_t CRITICAL_SECTION +#define ZSTD_pthread_mutex_init(a, b) (InitializeCriticalSection((a)), 0) +#define ZSTD_pthread_mutex_destroy(a) DeleteCriticalSection((a)) +#define ZSTD_pthread_mutex_lock(a) EnterCriticalSection((a)) +#define ZSTD_pthread_mutex_unlock(a) LeaveCriticalSection((a)) + +/* condition variable */ +#define ZSTD_pthread_cond_t CONDITION_VARIABLE +#define ZSTD_pthread_cond_init(a, b) (InitializeConditionVariable((a)), 0) +#define ZSTD_pthread_cond_destroy(a) /* No delete */ +#define ZSTD_pthread_cond_wait(a, b) SleepConditionVariableCS((a), (b), INFINITE) +#define ZSTD_pthread_cond_signal(a) WakeConditionVariable((a)) +#define ZSTD_pthread_cond_broadcast(a) WakeAllConditionVariable((a)) + +/* ZSTD_pthread_create() and ZSTD_pthread_join() */ +typedef struct { + HANDLE handle; + void* (*start_routine)(void*); + void* arg; +} ZSTD_pthread_t; + +int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused, + void* (*start_routine) (void*), void* arg); + +int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr); + +/** + * add here more wrappers as required + */ + + +#elif defined(ZSTD_MULTITHREAD) /* posix assumed ; need a better detection method */ +/* === POSIX Systems === */ +# include + +#define ZSTD_pthread_mutex_t pthread_mutex_t +#define ZSTD_pthread_mutex_init(a, b) pthread_mutex_init((a), (b)) +#define ZSTD_pthread_mutex_destroy(a) pthread_mutex_destroy((a)) +#define ZSTD_pthread_mutex_lock(a) pthread_mutex_lock((a)) +#define ZSTD_pthread_mutex_unlock(a) pthread_mutex_unlock((a)) + +#define ZSTD_pthread_cond_t pthread_cond_t +#define ZSTD_pthread_cond_init(a, b) pthread_cond_init((a), (b)) +#define ZSTD_pthread_cond_destroy(a) pthread_cond_destroy((a)) +#define ZSTD_pthread_cond_wait(a, b) pthread_cond_wait((a), (b)) +#define ZSTD_pthread_cond_signal(a) pthread_cond_signal((a)) +#define ZSTD_pthread_cond_broadcast(a) pthread_cond_broadcast((a)) + +#define ZSTD_pthread_t pthread_t +#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d)) +#define ZSTD_pthread_join(a, b) pthread_join((a),(b)) + +#else /* ZSTD_MULTITHREAD not defined */ +/* No multithreading support */ + +typedef int ZSTD_pthread_mutex_t; +#define ZSTD_pthread_mutex_init(a, b) ((void)a, 0) +#define ZSTD_pthread_mutex_destroy(a) +#define ZSTD_pthread_mutex_lock(a) +#define ZSTD_pthread_mutex_unlock(a) + +typedef int ZSTD_pthread_cond_t; +#define ZSTD_pthread_cond_init(a, b) ((void)a, 0) +#define ZSTD_pthread_cond_destroy(a) +#define ZSTD_pthread_cond_wait(a, b) +#define ZSTD_pthread_cond_signal(a) +#define ZSTD_pthread_cond_broadcast(a) + +/* do not use ZSTD_pthread_t */ + +#endif /* ZSTD_MULTITHREAD */ + +#if defined (__cplusplus) +} +#endif + +#endif /* THREADING_H_938743 */ diff --git a/src/borg/algorithms/zstd/lib/common/xxhash.c b/src/borg/algorithms/zstd/lib/common/xxhash.c new file mode 100644 index 0000000000..9d9c0e963c --- /dev/null +++ b/src/borg/algorithms/zstd/lib/common/xxhash.c @@ -0,0 +1,875 @@ +/* +* xxHash - Fast Hash algorithm +* Copyright (C) 2012-2016, Yann Collet +* +* BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are +* met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above +* copyright notice, this list of conditions and the following disclaimer +* in the documentation and/or other materials provided with the +* distribution. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +* +* You can contact the author at : +* - xxHash homepage: http://www.xxhash.com +* - xxHash source repository : https://github.com/Cyan4973/xxHash +*/ + + +/* ************************************* +* Tuning parameters +***************************************/ +/*!XXH_FORCE_MEMORY_ACCESS : + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. + * Method 2 : direct access. This method doesn't depend on compiler but violate C standard. + * It can generate buggy code on targets which do not support unaligned memory accesses. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See http://stackoverflow.com/a/32095106/646947 for details. + * Prefer these methods in priority order (0 > 1 > 2) + */ +#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +# define XXH_FORCE_MEMORY_ACCESS 2 +# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \ + (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) +# define XXH_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +/*!XXH_ACCEPT_NULL_INPUT_POINTER : + * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. + * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. + * By default, this option is disabled. To enable it, uncomment below define : + */ +/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */ + +/*!XXH_FORCE_NATIVE_FORMAT : + * By default, xxHash library provides endian-independant Hash values, based on little-endian convention. + * Results are therefore identical for little-endian and big-endian CPU. + * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. + * Should endian-independance be of no importance for your application, you may set the #define below to 1, + * to improve speed for Big-endian CPU. + * This option has no impact on Little_Endian CPU. + */ +#ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */ +# define XXH_FORCE_NATIVE_FORMAT 0 +#endif + +/*!XXH_FORCE_ALIGN_CHECK : + * This is a minor performance trick, only useful with lots of very small keys. + * It means : check for aligned/unaligned input. + * The check costs one initial branch per hash; set to 0 when the input data + * is guaranteed to be aligned. + */ +#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ +# if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) +# define XXH_FORCE_ALIGN_CHECK 0 +# else +# define XXH_FORCE_ALIGN_CHECK 1 +# endif +#endif + + +/* ************************************* +* Includes & Memory related functions +***************************************/ +/* Modify the local functions below should you wish to use some other memory routines */ +/* for malloc(), free() */ +#include +static void* XXH_malloc(size_t s) { return malloc(s); } +static void XXH_free (void* p) { free(p); } +/* for memcpy() */ +#include +static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } + +#ifndef XXH_STATIC_LINKING_ONLY +# define XXH_STATIC_LINKING_ONLY +#endif +#include "xxhash.h" + + +/* ************************************* +* Compiler Specific Options +***************************************/ +#if defined (__GNUC__) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# define INLINE_KEYWORD inline +#else +# define INLINE_KEYWORD +#endif + +#if defined(__GNUC__) +# define FORCE_INLINE_ATTR __attribute__((always_inline)) +#elif defined(_MSC_VER) +# define FORCE_INLINE_ATTR __forceinline +#else +# define FORCE_INLINE_ATTR +#endif + +#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR + + +#ifdef _MSC_VER +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + + +/* ************************************* +* Basic Types +***************************************/ +#ifndef MEM_MODULE +# define MEM_MODULE +# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +# else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; /* if your compiler doesn't support unsigned long long, replace by another 64-bit type here. Note that xxhash.h will also need to be updated. */ +# endif +#endif + + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; } +static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; } + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign; + +static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } +static U64 XXH_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } + +#else + +/* portable and safe solution. Generally efficient. + * see : http://stackoverflow.com/a/32095106/646947 + */ + +static U32 XXH_read32(const void* memPtr) +{ + U32 val; + memcpy(&val, memPtr, sizeof(val)); + return val; +} + +static U64 XXH_read64(const void* memPtr) +{ + U64 val; + memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + + +/* **************************************** +* Compiler-specific Functions and Macros +******************************************/ +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */ +#if defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) +# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) +#endif + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +# define XXH_swap64 _byteswap_uint64 +#elif GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +# define XXH_swap64 __builtin_bswap64 +#else +static U32 XXH_swap32 (U32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +static U64 XXH_swap64 (U64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + + +/* ************************************* +* Architecture Macros +***************************************/ +typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; + +/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */ +#ifndef XXH_CPU_LITTLE_ENDIAN + static const int g_one = 1; +# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&g_one)) +#endif + + +/* *************************** +* Memory reads +*****************************/ +typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; + +FORCE_INLINE_TEMPLATE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); + else + return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); +} + +FORCE_INLINE_TEMPLATE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) +{ + return XXH_readLE32_align(ptr, endian, XXH_unaligned); +} + +static U32 XXH_readBE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); +} + +FORCE_INLINE_TEMPLATE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); + else + return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); +} + +FORCE_INLINE_TEMPLATE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) +{ + return XXH_readLE64_align(ptr, endian, XXH_unaligned); +} + +static U64 XXH_readBE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); +} + + +/* ************************************* +* Macros +***************************************/ +#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + + +/* ************************************* +* Constants +***************************************/ +static const U32 PRIME32_1 = 2654435761U; +static const U32 PRIME32_2 = 2246822519U; +static const U32 PRIME32_3 = 3266489917U; +static const U32 PRIME32_4 = 668265263U; +static const U32 PRIME32_5 = 374761393U; + +static const U64 PRIME64_1 = 11400714785074694791ULL; +static const U64 PRIME64_2 = 14029467366897019727ULL; +static const U64 PRIME64_3 = 1609587929392839161ULL; +static const U64 PRIME64_4 = 9650029242287828579ULL; +static const U64 PRIME64_5 = 2870177450012600261ULL; + +XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } + + +/* ************************** +* Utils +****************************/ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dstState, const XXH32_state_t* restrict srcState) +{ + memcpy(dstState, srcState, sizeof(*dstState)); +} + +XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dstState, const XXH64_state_t* restrict srcState) +{ + memcpy(dstState, srcState, sizeof(*dstState)); +} + + +/* *************************** +* Simple Hash Functions +*****************************/ + +static U32 XXH32_round(U32 seed, U32 input) +{ + seed += input * PRIME32_2; + seed = XXH_rotl32(seed, 13); + seed *= PRIME32_1; + return seed; +} + +FORCE_INLINE_TEMPLATE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* bEnd = p + len; + U32 h32; +#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p==NULL) { + len=0; + bEnd=p=(const BYTE*)(size_t)16; + } +#endif + + if (len>=16) { + const BYTE* const limit = bEnd - 16; + U32 v1 = seed + PRIME32_1 + PRIME32_2; + U32 v2 = seed + PRIME32_2; + U32 v3 = seed + 0; + U32 v4 = seed - PRIME32_1; + + do { + v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4; + v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4; + v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4; + v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4; + } while (p<=limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } else { + h32 = seed + PRIME32_5; + } + + h32 += (U32) len; + + while (p+4<=bEnd) { + h32 += XXH_get32bits(p) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; + p+=4; + } + + while (p> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + + +XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_CREATESTATE_STATIC(state); + XXH32_reset(state, seed); + XXH32_update(state, input, len); + return XXH32_digest(state); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } } + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + + +static U64 XXH64_round(U64 acc, U64 input) +{ + acc += input * PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= PRIME64_1; + return acc; +} + +static U64 XXH64_mergeRound(U64 acc, U64 val) +{ + val = XXH64_round(0, val); + acc ^= val; + acc = acc * PRIME64_1 + PRIME64_4; + return acc; +} + +FORCE_INLINE_TEMPLATE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + U64 h64; +#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p==NULL) { + len=0; + bEnd=p=(const BYTE*)(size_t)32; + } +#endif + + if (len>=32) { + const BYTE* const limit = bEnd - 32; + U64 v1 = seed + PRIME64_1 + PRIME64_2; + U64 v2 = seed + PRIME64_2; + U64 v3 = seed + 0; + U64 v4 = seed - PRIME64_1; + + do { + v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8; + v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8; + v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8; + v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8; + } while (p<=limit); + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + h64 = XXH64_mergeRound(h64, v1); + h64 = XXH64_mergeRound(h64, v2); + h64 = XXH64_mergeRound(h64, v3); + h64 = XXH64_mergeRound(h64, v4); + + } else { + h64 = seed + PRIME64_5; + } + + h64 += (U64) len; + + while (p+8<=bEnd) { + U64 const k1 = XXH64_round(0, XXH_get64bits(p)); + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; + p+=8; + } + + if (p+4<=bEnd) { + h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + p+=4; + } + + while (p> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + + return h64; +} + + +XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH64_CREATESTATE_STATIC(state); + XXH64_reset(state, seed); + XXH64_update(state, input, len); + return XXH64_digest(state); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } } + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + + +/* ************************************************** +* Advanced Hash Functions +****************************************************/ + +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) +{ + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) +{ + return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); +} +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + + +/*** Hash feed ***/ + +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed) +{ + XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ + memset(&state, 0, sizeof(state)-4); /* do not write into reserved, for future removal */ + state.v1 = seed + PRIME32_1 + PRIME32_2; + state.v2 = seed + PRIME32_2; + state.v3 = seed + 0; + state.v4 = seed - PRIME32_1; + memcpy(statePtr, &state, sizeof(state)); + return XXH_OK; +} + + +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed) +{ + XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ + memset(&state, 0, sizeof(state)-8); /* do not write into reserved, for future removal */ + state.v1 = seed + PRIME64_1 + PRIME64_2; + state.v2 = seed + PRIME64_2; + state.v3 = seed + 0; + state.v4 = seed - PRIME64_1; + memcpy(statePtr, &state, sizeof(state)); + return XXH_OK; +} + + +FORCE_INLINE_TEMPLATE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input==NULL) return XXH_ERROR; +#endif + + state->total_len_32 += (unsigned)len; + state->large_len |= (len>=16) | (state->total_len_32>=16); + + if (state->memsize + len < 16) { /* fill in tmp buffer */ + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); + state->memsize += (unsigned)len; + return XXH_OK; + } + + if (state->memsize) { /* some data left from previous update */ + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); + { const U32* p32 = state->mem32; + state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++; + state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++; + state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++; + state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++; + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) { + const BYTE* const limit = bEnd - 16; + U32 v1 = state->v1; + U32 v2 = state->v2; + U32 v3 = state->v3; + U32 v4 = state->v4; + + do { + v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4; + v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4; + v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4; + v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4; + } while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) { + XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + + return XXH_OK; +} + +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH32_update_endian(state_in, input, len, XXH_bigEndian); +} + + + +FORCE_INLINE_TEMPLATE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) +{ + const BYTE * p = (const BYTE*)state->mem32; + const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize; + U32 h32; + + if (state->large_len) { + h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); + } else { + h32 = state->v3 /* == seed */ + PRIME32_5; + } + + h32 += state->total_len_32; + + while (p+4<=bEnd) { + h32 += XXH_readLE32(p, endian) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4; + p+=4; + } + + while (p> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + + +XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_digest_endian(state_in, XXH_littleEndian); + else + return XXH32_digest_endian(state_in, XXH_bigEndian); +} + + + +/* **** XXH64 **** */ + +FORCE_INLINE_TEMPLATE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input==NULL) return XXH_ERROR; +#endif + + state->total_len += len; + + if (state->memsize + len < 32) { /* fill in tmp buffer */ + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); + state->memsize += (U32)len; + return XXH_OK; + } + + if (state->memsize) { /* tmp buffer is full */ + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); + state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian)); + state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian)); + state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian)); + state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian)); + p += 32-state->memsize; + state->memsize = 0; + } + + if (p+32 <= bEnd) { + const BYTE* const limit = bEnd - 32; + U64 v1 = state->v1; + U64 v2 = state->v2; + U64 v3 = state->v3; + U64 v4 = state->v4; + + do { + v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8; + v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8; + v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8; + v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8; + } while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) { + XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + + return XXH_OK; +} + +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH64_update_endian(state_in, input, len, XXH_bigEndian); +} + + + +FORCE_INLINE_TEMPLATE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) +{ + const BYTE * p = (const BYTE*)state->mem64; + const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize; + U64 h64; + + if (state->total_len >= 32) { + U64 const v1 = state->v1; + U64 const v2 = state->v2; + U64 const v3 = state->v3; + U64 const v4 = state->v4; + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + h64 = XXH64_mergeRound(h64, v1); + h64 = XXH64_mergeRound(h64, v2); + h64 = XXH64_mergeRound(h64, v3); + h64 = XXH64_mergeRound(h64, v4); + } else { + h64 = state->v3 + PRIME64_5; + } + + h64 += (U64) state->total_len; + + while (p+8<=bEnd) { + U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian)); + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; + p+=8; + } + + if (p+4<=bEnd) { + h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1; + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + p+=4; + } + + while (p> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + + return h64; +} + + +XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_digest_endian(state_in, XXH_littleEndian); + else + return XXH64_digest_endian(state_in, XXH_bigEndian); +} + + +/* ************************** +* Canonical representation +****************************/ + +/*! Default XXH result types are basic unsigned 32 and 64 bits. +* The canonical representation follows human-readable write convention, aka big-endian (large digits first). +* These functions allow transformation of hash result into and from its canonical format. +* This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs. +*/ + +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); + memcpy(dst, &hash, sizeof(*dst)); +} + +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); + memcpy(dst, &hash, sizeof(*dst)); +} + +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) +{ + return XXH_readBE32(src); +} + +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) +{ + return XXH_readBE64(src); +} diff --git a/src/borg/algorithms/zstd/lib/common/xxhash.h b/src/borg/algorithms/zstd/lib/common/xxhash.h new file mode 100644 index 0000000000..9bad1f59f6 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/common/xxhash.h @@ -0,0 +1,305 @@ +/* + xxHash - Extremely Fast Hash algorithm + Header File + Copyright (C) 2012-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - xxHash source repository : https://github.com/Cyan4973/xxHash +*/ + +/* Notice extracted from xxHash homepage : + +xxHash is an extremely fast Hash algorithm, running at RAM speed limits. +It also successfully passes all tests from the SMHasher suite. + +Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) + +Name Speed Q.Score Author +xxHash 5.4 GB/s 10 +CrapWow 3.2 GB/s 2 Andrew +MumurHash 3a 2.7 GB/s 10 Austin Appleby +SpookyHash 2.0 GB/s 10 Bob Jenkins +SBox 1.4 GB/s 9 Bret Mulvey +Lookup3 1.2 GB/s 9 Bob Jenkins +SuperFastHash 1.2 GB/s 1 Paul Hsieh +CityHash64 1.05 GB/s 10 Pike & Alakuijala +FNV 0.55 GB/s 5 Fowler, Noll, Vo +CRC32 0.43 GB/s 9 +MD5-32 0.33 GB/s 10 Ronald L. Rivest +SHA1-32 0.28 GB/s 10 + +Q.Score is a measure of quality of the hash function. +It depends on successfully passing SMHasher test set. +10 is a perfect score. + +A 64-bits version, named XXH64, is available since r35. +It offers much better speed, but for 64-bits applications only. +Name Speed on 64 bits Speed on 32 bits +XXH64 13.8 GB/s 1.9 GB/s +XXH32 6.8 GB/s 6.0 GB/s +*/ + +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef XXHASH_H_5627135585666179 +#define XXHASH_H_5627135585666179 1 + + +/* **************************** +* Definitions +******************************/ +#include /* size_t */ +typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; + + +/* **************************** +* API modifier +******************************/ +/** XXH_PRIVATE_API +* This is useful if you want to include xxhash functions in `static` mode +* in order to inline them, and remove their symbol from the public list. +* Methodology : +* #define XXH_PRIVATE_API +* #include "xxhash.h" +* `xxhash.c` is automatically included. +* It's not useful to compile and link it as a separate module anymore. +*/ +#ifdef XXH_PRIVATE_API +# ifndef XXH_STATIC_LINKING_ONLY +# define XXH_STATIC_LINKING_ONLY +# endif +# if defined(__GNUC__) +# define XXH_PUBLIC_API static __inline __attribute__((unused)) +# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define XXH_PUBLIC_API static inline +# elif defined(_MSC_VER) +# define XXH_PUBLIC_API static __inline +# else +# define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */ +# endif +#else +# define XXH_PUBLIC_API /* do nothing */ +#endif /* XXH_PRIVATE_API */ + +/*!XXH_NAMESPACE, aka Namespace Emulation : + +If you want to include _and expose_ xxHash functions from within your own library, +but also want to avoid symbol collisions with another library which also includes xxHash, + +you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library +with the value of XXH_NAMESPACE (so avoid to keep it NULL and avoid numeric values). + +Note that no change is required within the calling program as long as it includes `xxhash.h` : +regular symbol name will be automatically translated by this header. +*/ +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) +# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) +# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) +# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) +# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) +# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) +#endif + + +/* ************************************* +* Version +***************************************/ +#define XXH_VERSION_MAJOR 0 +#define XXH_VERSION_MINOR 6 +#define XXH_VERSION_RELEASE 2 +#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) +XXH_PUBLIC_API unsigned XXH_versionNumber (void); + + +/* **************************** +* Simple Hash Functions +******************************/ +typedef unsigned int XXH32_hash_t; +typedef unsigned long long XXH64_hash_t; + +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed); +XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed); + +/*! +XXH32() : + Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". + The memory between input & input+length must be valid (allocated and read-accessible). + "seed" can be used to alter the result predictably. + Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s +XXH64() : + Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". + "seed" can be used to alter the result predictably. + This function runs 2x faster on 64-bits systems, but slower on 32-bits systems (see benchmark). +*/ + + +/* **************************** +* Streaming Hash Functions +******************************/ +typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */ +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ + +/*! State allocation, compatible with dynamic libraries */ + +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); + +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); + + +/* hash streaming */ + +XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed); +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); + +XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); + +/* +These functions generate the xxHash of an input provided in multiple segments. +Note that, for small input, they are slower than single-call functions, due to state management. +For small input, prefer `XXH32()` and `XXH64()` . + +XXH state must first be allocated, using XXH*_createState() . + +Start a new hash by initializing state with a seed, using XXH*_reset(). + +Then, feed the hash state by calling XXH*_update() as many times as necessary. +Obviously, input must be allocated and read accessible. +The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. + +Finally, a hash value can be produced anytime, by using XXH*_digest(). +This function returns the nn-bits hash as an int or long long. + +It's still possible to continue inserting input into the hash state after a digest, +and generate some new hashes later on, by calling again XXH*_digest(). + +When done, free XXH state space if it was allocated dynamically. +*/ + + +/* ************************** +* Utils +****************************/ +#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* ! C99 */ +# define restrict /* disable restrict */ +#endif + +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dst_state, const XXH32_state_t* restrict src_state); +XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dst_state, const XXH64_state_t* restrict src_state); + + +/* ************************** +* Canonical representation +****************************/ +/* Default result type for XXH functions are primitive unsigned 32 and 64 bits. +* The canonical representation uses human-readable write convention, aka big-endian (large digits first). +* These functions allow transformation of hash result into and from its canonical format. +* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. +*/ +typedef struct { unsigned char digest[4]; } XXH32_canonical_t; +typedef struct { unsigned char digest[8]; } XXH64_canonical_t; + +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); + +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); + +#endif /* XXHASH_H_5627135585666179 */ + + + +/* ================================================================================================ + This section contains definitions which are not guaranteed to remain stable. + They may change in future versions, becoming incompatible with a different version of the library. + They shall only be used with static linking. + Never use these definitions in association with dynamic linking ! +=================================================================================================== */ +#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXH_STATIC_H_3543687687345) +#define XXH_STATIC_H_3543687687345 + +/* These definitions are only meant to allow allocation of XXH state + statically, on stack, or in a struct for example. + Do not use members directly. */ + + struct XXH32_state_s { + unsigned total_len_32; + unsigned large_len; + unsigned v1; + unsigned v2; + unsigned v3; + unsigned v4; + unsigned mem32[4]; /* buffer defined as U32 for alignment */ + unsigned memsize; + unsigned reserved; /* never read nor write, will be removed in a future version */ + }; /* typedef'd to XXH32_state_t */ + + struct XXH64_state_s { + unsigned long long total_len; + unsigned long long v1; + unsigned long long v2; + unsigned long long v3; + unsigned long long v4; + unsigned long long mem64[4]; /* buffer defined as U64 for alignment */ + unsigned memsize; + unsigned reserved[2]; /* never read nor write, will be removed in a future version */ + }; /* typedef'd to XXH64_state_t */ + + +# ifdef XXH_PRIVATE_API +# include "xxhash.c" /* include xxhash functions as `static`, for inlining */ +# endif + +#endif /* XXH_STATIC_LINKING_ONLY && XXH_STATIC_H_3543687687345 */ + + +#if defined (__cplusplus) +} +#endif diff --git a/src/borg/algorithms/zstd/lib/common/zstd_common.c b/src/borg/algorithms/zstd/lib/common/zstd_common.c new file mode 100644 index 0000000000..c2041053be --- /dev/null +++ b/src/borg/algorithms/zstd/lib/common/zstd_common.c @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + + +/*-************************************* +* Dependencies +***************************************/ +#include /* malloc, calloc, free */ +#include /* memset */ +#include "error_private.h" +#include "zstd_internal.h" + + +/*-**************************************** +* Version +******************************************/ +unsigned ZSTD_versionNumber(void) { return ZSTD_VERSION_NUMBER; } + +const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; } + + +/*-**************************************** +* ZSTD Error Management +******************************************/ +/*! ZSTD_isError() : +* tells if a return value is an error code */ +unsigned ZSTD_isError(size_t code) { return ERR_isError(code); } + +/*! ZSTD_getErrorName() : +* provides error code string from function result (useful for debugging) */ +const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); } + +/*! ZSTD_getError() : +* convert a `size_t` function result into a proper ZSTD_errorCode enum */ +ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); } + +/*! ZSTD_getErrorString() : +* provides error code string from enum */ +const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); } + + +/*=************************************************************** +* Custom allocator +****************************************************************/ +void* ZSTD_malloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) + return customMem.customAlloc(customMem.opaque, size); + return malloc(size); +} + +void* ZSTD_calloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) { + /* calloc implemented as malloc+memset; + * not as efficient as calloc, but next best guess for custom malloc */ + void* const ptr = customMem.customAlloc(customMem.opaque, size); + memset(ptr, 0, size); + return ptr; + } + return calloc(1, size); +} + +void ZSTD_free(void* ptr, ZSTD_customMem customMem) +{ + if (ptr!=NULL) { + if (customMem.customFree) + customMem.customFree(customMem.opaque, ptr); + else + free(ptr); + } +} diff --git a/src/borg/algorithms/zstd/lib/common/zstd_errors.h b/src/borg/algorithms/zstd/lib/common/zstd_errors.h new file mode 100644 index 0000000000..4bcb7769fe --- /dev/null +++ b/src/borg/algorithms/zstd/lib/common/zstd_errors.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_ERRORS_H_398273423 +#define ZSTD_ERRORS_H_398273423 + +#if defined (__cplusplus) +extern "C" { +#endif + +/*===== dependency =====*/ +#include /* size_t */ + + +/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */ +#ifndef ZSTDERRORLIB_VISIBILITY +# if defined(__GNUC__) && (__GNUC__ >= 4) +# define ZSTDERRORLIB_VISIBILITY __attribute__ ((visibility ("default"))) +# else +# define ZSTDERRORLIB_VISIBILITY +# endif +#endif +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBILITY +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY +#endif + +/*-**************************************** + * error codes list + * note : this API is still considered unstable + * and shall not be used with a dynamic library. + * only static linking is allowed + ******************************************/ +typedef enum { + ZSTD_error_no_error = 0, + ZSTD_error_GENERIC = 1, + ZSTD_error_prefix_unknown = 10, + ZSTD_error_version_unsupported = 12, + ZSTD_error_frameParameter_unsupported = 14, + ZSTD_error_frameParameter_windowTooLarge = 16, + ZSTD_error_corruption_detected = 20, + ZSTD_error_checksum_wrong = 22, + ZSTD_error_dictionary_corrupted = 30, + ZSTD_error_dictionary_wrong = 32, + ZSTD_error_dictionaryCreation_failed = 34, + ZSTD_error_parameter_unsupported = 40, + ZSTD_error_parameter_outOfBound = 42, + ZSTD_error_tableLog_tooLarge = 44, + ZSTD_error_maxSymbolValue_tooLarge = 46, + ZSTD_error_maxSymbolValue_tooSmall = 48, + ZSTD_error_stage_wrong = 60, + ZSTD_error_init_missing = 62, + ZSTD_error_memory_allocation = 64, + ZSTD_error_dstSize_tooSmall = 70, + ZSTD_error_srcSize_wrong = 72, + /* following error codes are not stable and may be removed or changed in a future version */ + ZSTD_error_frameIndex_tooLarge = 100, + ZSTD_error_seekableIO = 102, + ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */ +} ZSTD_ErrorCode; + +/*! ZSTD_getErrorCode() : + convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, + which can be used to compare with enum list published above */ +ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); +ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /**< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */ + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_ERRORS_H_398273423 */ diff --git a/src/borg/algorithms/zstd/lib/common/zstd_internal.h b/src/borg/algorithms/zstd/lib/common/zstd_internal.h new file mode 100644 index 0000000000..e91cd20baa --- /dev/null +++ b/src/borg/algorithms/zstd/lib/common/zstd_internal.h @@ -0,0 +1,409 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_CCOMMON_H_MODULE +#define ZSTD_CCOMMON_H_MODULE + + +/*-************************************* +* Dependencies +***************************************/ +#include "compiler.h" +#include "mem.h" +#include "error_private.h" +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" +#ifndef XXH_STATIC_LINKING_ONLY +# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ +#endif +#include "xxhash.h" /* XXH_reset, update, digest */ + + +#if defined (__cplusplus) +extern "C" { +#endif + + +/*-************************************* +* Debug +***************************************/ +#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1) +# include +#else +# ifndef assert +# define assert(condition) ((void)0) +# endif +#endif + +#define ZSTD_STATIC_ASSERT(c) { enum { ZSTD_static_assert = 1/(int)(!!(c)) }; } + +#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2) +# include +/* recommended values for ZSTD_DEBUG display levels : + * 1 : no display, enables assert() only + * 2 : reserved for currently active debugging path + * 3 : events once per object lifetime (CCtx, CDict) + * 4 : events once per frame + * 5 : events once per block + * 6 : events once per sequence (*very* verbose) */ +# define DEBUGLOG(l, ...) { \ + if (l<=ZSTD_DEBUG) { \ + fprintf(stderr, __FILE__ ": "); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, " \n"); \ + } } +#else +# define DEBUGLOG(l, ...) {} /* disabled */ +#endif + + +/*-************************************* +* shared macros +***************************************/ +#undef MIN +#undef MAX +#define MIN(a,b) ((a)<(b) ? (a) : (b)) +#define MAX(a,b) ((a)>(b) ? (a) : (b)) +#define CHECK_F(f) { size_t const errcod = f; if (ERR_isError(errcod)) return errcod; } /* check and Forward error code */ +#define CHECK_E(f, e) { size_t const errcod = f; if (ERR_isError(errcod)) return ERROR(e); } /* check and send Error code */ + + +/*-************************************* +* Common constants +***************************************/ +#define ZSTD_OPT_NUM (1<<12) + +#define ZSTD_REP_NUM 3 /* number of repcodes */ +#define ZSTD_REP_CHECK (ZSTD_REP_NUM) /* number of repcodes to check by the optimal parser */ +#define ZSTD_REP_MOVE (ZSTD_REP_NUM-1) +#define ZSTD_REP_MOVE_OPT (ZSTD_REP_NUM) +static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define BIT7 128 +#define BIT6 64 +#define BIT5 32 +#define BIT4 16 +#define BIT1 2 +#define BIT0 1 + +#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10 +#define ZSTD_WINDOWLOG_DEFAULTMAX 27 /* Default maximum allowed window log */ +static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 }; +static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; + +#define ZSTD_FRAMEIDSIZE 4 +static const size_t ZSTD_frameIdSize = ZSTD_FRAMEIDSIZE; /* magic number size */ + +#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ +static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; +typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; + +#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ +#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */ + +#define HufLog 12 +typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e; + +#define LONGNBSEQ 0x7F00 + +#define MINMATCH 3 + +#define Litbits 8 +#define MaxLit ((1<= 3) /* GCC Intrinsic */ + return 31 - __builtin_clz(val); +# else /* Software version */ + static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; + U32 v = val; + int r; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27]; + return r; +# endif + } +} + + +/* hidden functions */ + +/* ZSTD_invalidateRepCodes() : + * ensures next compression will not use repcodes from previous block. + * Note : only works with regular variant; + * do not use with extDict variant ! */ +void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx); + + +/*! ZSTD_initCStream_internal() : + * Private use only. Init streaming operation. + * expects params to be valid. + * must receive dict, or cdict, or none, but not both. + * @return : 0, or an error code */ +size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, unsigned long long pledgedSrcSize); + +/*! ZSTD_compressStream_generic() : + * Private use only. To be called from zstdmt_compress.c in single-thread mode. */ +size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective const flushMode); + +/*! ZSTD_getCParamsFromCDict() : + * as the name implies */ +ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict); + +/* ZSTD_compressBegin_advanced_internal() : + * Private use only. To be called from zstdmt_compress.c. */ +size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_dictMode_e dictMode, + ZSTD_CCtx_params params, + unsigned long long pledgedSrcSize); + +/* ZSTD_compress_advanced_internal() : + * Private use only. To be called from zstdmt_compress.c. */ +size_t ZSTD_compress_advanced_internal(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_CCtx_params params); + +typedef struct { + blockType_e blockType; + U32 lastBlock; + U32 origSize; +} blockProperties_t; + +/*! ZSTD_getcBlockSize() : +* Provides the size of compressed block from block header `src` */ +size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, + blockProperties_t* bpPtr); + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/src/borg/algorithms/zstd/lib/compress/fse_compress.c b/src/borg/algorithms/zstd/lib/compress/fse_compress.c new file mode 100644 index 0000000000..549c115d42 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/compress/fse_compress.c @@ -0,0 +1,841 @@ +/* ****************************************************************** + FSE : Finite State Entropy encoder + Copyright (C) 2013-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + +/* ************************************************************** +* Includes +****************************************************************/ +#include /* malloc, free, qsort */ +#include /* memcpy, memset */ +#include /* printf (debug) */ +#include "bitstream.h" +#include "compiler.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#include "error_private.h" + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define FSE_isError ERR_isError +#define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + + +/* ************************************************************** +* Templates +****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ + +/* safety checks */ +#ifndef FSE_FUNCTION_EXTENSION +# error "FSE_FUNCTION_EXTENSION must be defined" +#endif +#ifndef FSE_FUNCTION_TYPE +# error "FSE_FUNCTION_TYPE must be defined" +#endif + +/* Function names */ +#define FSE_CAT(X,Y) X##Y +#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) +#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) + + +/* Function templates */ + +/* FSE_buildCTable_wksp() : + * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). + * wkspSize should be sized to handle worst case situation, which is `1<>1 : 1) ; + FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); + U32 const step = FSE_TABLESTEP(tableSize); + U32 cumul[FSE_MAX_SYMBOL_VALUE+2]; + + FSE_FUNCTION_TYPE* const tableSymbol = (FSE_FUNCTION_TYPE*)workSpace; + U32 highThreshold = tableSize-1; + + /* CTable header */ + if (((size_t)1 << tableLog) * sizeof(FSE_FUNCTION_TYPE) > wkspSize) return ERROR(tableLog_tooLarge); + tableU16[-2] = (U16) tableLog; + tableU16[-1] = (U16) maxSymbolValue; + + /* For explanations on how to distribute symbol values over the table : + * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ + + /* symbol start positions */ + { U32 u; + cumul[0] = 0; + for (u=1; u<=maxSymbolValue+1; u++) { + if (normalizedCounter[u-1]==-1) { /* Low proba symbol */ + cumul[u] = cumul[u-1] + 1; + tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1); + } else { + cumul[u] = cumul[u-1] + normalizedCounter[u-1]; + } } + cumul[maxSymbolValue+1] = tableSize+1; + } + + /* Spread symbols */ + { U32 position = 0; + U32 symbol; + for (symbol=0; symbol<=maxSymbolValue; symbol++) { + int nbOccurences; + for (nbOccurences=0; nbOccurences highThreshold) position = (position + step) & tableMask; /* Low proba area */ + } } + + if (position!=0) return ERROR(GENERIC); /* Must have gone through all positions */ + } + + /* Build table */ + { U32 u; for (u=0; u> 3) + 3; + return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */ +} + +static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize, + const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, + unsigned writeIsSafe) +{ + BYTE* const ostart = (BYTE*) header; + BYTE* out = ostart; + BYTE* const oend = ostart + headerBufferSize; + int nbBits; + const int tableSize = 1 << tableLog; + int remaining; + int threshold; + U32 bitStream; + int bitCount; + unsigned charnum = 0; + int previous0 = 0; + + bitStream = 0; + bitCount = 0; + /* Table Size */ + bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount; + bitCount += 4; + + /* Init */ + remaining = tableSize+1; /* +1 for extra accuracy */ + threshold = tableSize; + nbBits = tableLog+1; + + while (remaining>1) { /* stops at 1 */ + if (previous0) { + unsigned start = charnum; + while (!normalizedCounter[charnum]) charnum++; + while (charnum >= start+24) { + start+=24; + bitStream += 0xFFFFU << bitCount; + if ((!writeIsSafe) && (out > oend-2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE) bitStream; + out[1] = (BYTE)(bitStream>>8); + out+=2; + bitStream>>=16; + } + while (charnum >= start+3) { + start+=3; + bitStream += 3 << bitCount; + bitCount += 2; + } + bitStream += (charnum-start) << bitCount; + bitCount += 2; + if (bitCount>16) { + if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out += 2; + bitStream >>= 16; + bitCount -= 16; + } } + { int count = normalizedCounter[charnum++]; + int const max = (2*threshold-1)-remaining; + remaining -= count < 0 ? -count : count; + count++; /* +1 for extra accuracy */ + if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ + bitStream += count << bitCount; + bitCount += nbBits; + bitCount -= (count>=1; + } + if (bitCount>16) { + if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out += 2; + bitStream >>= 16; + bitCount -= 16; + } } + + /* flush remaining bitStream */ + if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out+= (bitCount+7) /8; + + if (charnum > maxSymbolValue + 1) return ERROR(GENERIC); + + return (out-ostart); +} + + +size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +{ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported */ + if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */ + + if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog)) + return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0); + + return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1); +} + + + +/*-************************************************************** +* Counting histogram +****************************************************************/ +/*! FSE_count_simple + This function counts byte values within `src`, and store the histogram into table `count`. + It doesn't use any additional memory. + But this function is unsafe : it doesn't check that all values within `src` can fit into `count`. + For this reason, prefer using a table `count` with 256 elements. + @return : count of most numerous element +*/ +size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + const BYTE* const end = ip + srcSize; + unsigned maxSymbolValue = *maxSymbolValuePtr; + unsigned max=0; + + memset(count, 0, (maxSymbolValue+1)*sizeof(*count)); + if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; } + + while (ip max) max = count[s]; } + + return (size_t)max; +} + + +/* FSE_count_parallel_wksp() : + * Same as FSE_count_parallel(), but using an externally provided scratch buffer. + * `workSpace` size must be a minimum of `1024 * sizeof(unsigned)`` */ +static size_t FSE_count_parallel_wksp( + unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, + unsigned checkMax, unsigned* const workSpace) +{ + const BYTE* ip = (const BYTE*)source; + const BYTE* const iend = ip+sourceSize; + unsigned maxSymbolValue = *maxSymbolValuePtr; + unsigned max=0; + U32* const Counting1 = workSpace; + U32* const Counting2 = Counting1 + 256; + U32* const Counting3 = Counting2 + 256; + U32* const Counting4 = Counting3 + 256; + + memset(Counting1, 0, 4*256*sizeof(unsigned)); + + /* safety checks */ + if (!sourceSize) { + memset(count, 0, maxSymbolValue + 1); + *maxSymbolValuePtr = 0; + return 0; + } + if (!maxSymbolValue) maxSymbolValue = 255; /* 0 == default */ + + /* by stripes of 16 bytes */ + { U32 cached = MEM_read32(ip); ip += 4; + while (ip < iend-15) { + U32 c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + } + ip-=4; + } + + /* finish last symbols */ + while (ipmaxSymbolValue; s--) { + Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s]; + if (Counting1[s]) return ERROR(maxSymbolValue_tooSmall); + } } + + { U32 s; for (s=0; s<=maxSymbolValue; s++) { + count[s] = Counting1[s] + Counting2[s] + Counting3[s] + Counting4[s]; + if (count[s] > max) max = count[s]; + } } + + while (!count[maxSymbolValue]) maxSymbolValue--; + *maxSymbolValuePtr = maxSymbolValue; + return (size_t)max; +} + +/* FSE_countFast_wksp() : + * Same as FSE_countFast(), but using an externally provided scratch buffer. + * `workSpace` size must be table of >= `1024` unsigned */ +size_t FSE_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, unsigned* workSpace) +{ + if (sourceSize < 1500) return FSE_count_simple(count, maxSymbolValuePtr, source, sourceSize); + return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 0, workSpace); +} + +/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */ +size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize) +{ + unsigned tmpCounters[1024]; + return FSE_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters); +} + +/* FSE_count_wksp() : + * Same as FSE_count(), but using an externally provided scratch buffer. + * `workSpace` size must be table of >= `1024` unsigned */ +size_t FSE_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, unsigned* workSpace) +{ + if (*maxSymbolValuePtr < 255) + return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 1, workSpace); + *maxSymbolValuePtr = 255; + return FSE_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace); +} + +size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize) +{ + unsigned tmpCounters[1024]; + return FSE_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters); +} + + + +/*-************************************************************** +* FSE Compression Code +****************************************************************/ +/*! FSE_sizeof_CTable() : + FSE_CTable is a variable size structure which contains : + `U16 tableLog;` + `U16 maxSymbolValue;` + `U16 nextStateNumber[1 << tableLog];` // This size is variable + `FSE_symbolCompressionTransform symbolTT[maxSymbolValue+1];` // This size is variable +Allocation is manual (C standard does not support variable-size structures). +*/ +size_t FSE_sizeof_CTable (unsigned maxSymbolValue, unsigned tableLog) +{ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); + return FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32); +} + +FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog) +{ + size_t size; + if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; + size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32); + return (FSE_CTable*)malloc(size); +} + +void FSE_freeCTable (FSE_CTable* ct) { free(ct); } + +/* provides the minimum logSize to safely represent a distribution */ +static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) +{ + U32 minBitsSrc = BIT_highbit32((U32)(srcSize - 1)) + 1; + U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2; + U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; + assert(srcSize > 1); /* Not supported, RLE should be used instead */ + return minBits; +} + +unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus) +{ + U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus; + U32 tableLog = maxTableLog; + U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue); + assert(srcSize > 1); /* Not supported, RLE should be used instead */ + if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; + if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */ + if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */ + if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG; + if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG; + return tableLog; +} + +unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) +{ + return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2); +} + + +/* Secondary normalization method. + To be used when primary method fails. */ + +static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue) +{ + short const NOT_YET_ASSIGNED = -2; + U32 s; + U32 distributed = 0; + U32 ToDistribute; + + /* Init */ + U32 const lowThreshold = (U32)(total >> tableLog); + U32 lowOne = (U32)((total * 3) >> (tableLog + 1)); + + for (s=0; s<=maxSymbolValue; s++) { + if (count[s] == 0) { + norm[s]=0; + continue; + } + if (count[s] <= lowThreshold) { + norm[s] = -1; + distributed++; + total -= count[s]; + continue; + } + if (count[s] <= lowOne) { + norm[s] = 1; + distributed++; + total -= count[s]; + continue; + } + + norm[s]=NOT_YET_ASSIGNED; + } + ToDistribute = (1 << tableLog) - distributed; + + if ((total / ToDistribute) > lowOne) { + /* risk of rounding to zero */ + lowOne = (U32)((total * 3) / (ToDistribute * 2)); + for (s=0; s<=maxSymbolValue; s++) { + if ((norm[s] == NOT_YET_ASSIGNED) && (count[s] <= lowOne)) { + norm[s] = 1; + distributed++; + total -= count[s]; + continue; + } } + ToDistribute = (1 << tableLog) - distributed; + } + + if (distributed == maxSymbolValue+1) { + /* all values are pretty poor; + probably incompressible data (should have already been detected); + find max, then give all remaining points to max */ + U32 maxV = 0, maxC = 0; + for (s=0; s<=maxSymbolValue; s++) + if (count[s] > maxC) maxV=s, maxC=count[s]; + norm[maxV] += (short)ToDistribute; + return 0; + } + + if (total == 0) { + /* all of the symbols were low enough for the lowOne or lowThreshold */ + for (s=0; ToDistribute > 0; s = (s+1)%(maxSymbolValue+1)) + if (norm[s] > 0) ToDistribute--, norm[s]++; + return 0; + } + + { U64 const vStepLog = 62 - tableLog; + U64 const mid = (1ULL << (vStepLog-1)) - 1; + U64 const rStep = ((((U64)1<> vStepLog); + U32 const sEnd = (U32)(end >> vStepLog); + U32 const weight = sEnd - sStart; + if (weight < 1) + return ERROR(GENERIC); + norm[s] = (short)weight; + tmpTotal = end; + } } } + + return 0; +} + + +size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog, + const unsigned* count, size_t total, + unsigned maxSymbolValue) +{ + /* Sanity checks */ + if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; + if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported size */ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported size */ + if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */ + + { static U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 }; + U64 const scale = 62 - tableLog; + U64 const step = ((U64)1<<62) / total; /* <== here, one division ! */ + U64 const vStep = 1ULL<<(scale-20); + int stillToDistribute = 1<> tableLog); + + for (s=0; s<=maxSymbolValue; s++) { + if (count[s] == total) return 0; /* rle special case */ + if (count[s] == 0) { normalizedCounter[s]=0; continue; } + if (count[s] <= lowThreshold) { + normalizedCounter[s] = -1; + stillToDistribute--; + } else { + short proba = (short)((count[s]*step) >> scale); + if (proba<8) { + U64 restToBeat = vStep * rtbTable[proba]; + proba += (count[s]*step) - ((U64)proba< restToBeat; + } + if (proba > largestP) largestP=proba, largest=s; + normalizedCounter[s] = proba; + stillToDistribute -= proba; + } } + if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) { + /* corner case, need another normalization method */ + size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue); + if (FSE_isError(errorCode)) return errorCode; + } + else normalizedCounter[largest] += (short)stillToDistribute; + } + +#if 0 + { /* Print Table (debug) */ + U32 s; + U32 nTotal = 0; + for (s=0; s<=maxSymbolValue; s++) + printf("%3i: %4i \n", s, normalizedCounter[s]); + for (s=0; s<=maxSymbolValue; s++) + nTotal += abs(normalizedCounter[s]); + if (nTotal != (1U<>1); /* assumption : tableLog >= 1 */ + FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); + unsigned s; + + /* Sanity checks */ + if (nbBits < 1) return ERROR(GENERIC); /* min size */ + + /* header */ + tableU16[-2] = (U16) nbBits; + tableU16[-1] = (U16) maxSymbolValue; + + /* Build table */ + for (s=0; s FSE_MAX_TABLELOG*4+7 ) && (srcSize & 2)) { /* test bit 2 */ + FSE_encodeSymbol(&bitC, &CState2, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); + FSE_FLUSHBITS(&bitC); + } + + /* 2 or 4 encoding per loop */ + while ( ip>istart ) { + + FSE_encodeSymbol(&bitC, &CState2, *--ip); + + if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */ + FSE_FLUSHBITS(&bitC); + + FSE_encodeSymbol(&bitC, &CState1, *--ip); + + if (sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) { /* this test must be static */ + FSE_encodeSymbol(&bitC, &CState2, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); + } + + FSE_FLUSHBITS(&bitC); + } + + FSE_flushCState(&bitC, &CState2); + FSE_flushCState(&bitC, &CState1); + return BIT_closeCStream(&bitC); +} + +size_t FSE_compress_usingCTable (void* dst, size_t dstSize, + const void* src, size_t srcSize, + const FSE_CTable* ct) +{ + unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize)); + + if (fast) + return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1); + else + return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0); +} + + +size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); } + +#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e +#define CHECK_F(f) { CHECK_V_F(_var_err__, f); } + +/* FSE_compress_wksp() : + * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). + * `wkspSize` size must be `(1< not compressible */ + if (maxCount < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */ + } + + tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue); + CHECK_F( FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue) ); + + /* Write table description header */ + { CHECK_V_F(nc_err, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) ); + op += nc_err; + } + + /* Compress */ + CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, scratchBufferSize) ); + { CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, src, srcSize, CTable) ); + if (cSize == 0) return 0; /* not enough space for compressed data */ + op += cSize; + } + + /* check compressibility */ + if ( (size_t)(op-ostart) >= srcSize-1 ) return 0; + + return op-ostart; +} + +typedef struct { + FSE_CTable CTable_max[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)]; + BYTE scratchBuffer[1 << FSE_MAX_TABLELOG]; +} fseWkspMax_t; + +size_t FSE_compress2 (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog) +{ + fseWkspMax_t scratchBuffer; + FSE_STATIC_ASSERT(sizeof(scratchBuffer) >= FSE_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)); /* compilation failures here means scratchBuffer is not large enough */ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); + return FSE_compress_wksp(dst, dstCapacity, src, srcSize, maxSymbolValue, tableLog, &scratchBuffer, sizeof(scratchBuffer)); +} + +size_t FSE_compress (void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + return FSE_compress2(dst, dstCapacity, src, srcSize, FSE_MAX_SYMBOL_VALUE, FSE_DEFAULT_TABLELOG); +} + + +#endif /* FSE_COMMONDEFS_ONLY */ diff --git a/src/borg/algorithms/zstd/lib/compress/huf_compress.c b/src/borg/algorithms/zstd/lib/compress/huf_compress.c new file mode 100644 index 0000000000..5692d56e00 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/compress/huf_compress.c @@ -0,0 +1,690 @@ +/* ****************************************************************** + Huffman encoder, part of New Generation Entropy library + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + +/* ************************************************************** +* Compiler specifics +****************************************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + + +/* ************************************************************** +* Includes +****************************************************************/ +#include /* memcpy, memset */ +#include /* printf (debug) */ +#include "bitstream.h" +#define FSE_STATIC_LINKING_ONLY /* FSE_optimalTableLog_internal */ +#include "fse.h" /* header compression */ +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" +#include "error_private.h" + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define HUF_isError ERR_isError +#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ +#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e +#define CHECK_F(f) { CHECK_V_F(_var_err__, f); } + + +/* ************************************************************** +* Utils +****************************************************************/ +unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) +{ + return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); +} + + +/* ******************************************************* +* HUF : Huffman block compression +*********************************************************/ +/* HUF_compressWeights() : + * Same as FSE_compress(), but dedicated to huff0's weights compression. + * The use case needs much less stack memory. + * Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX. + */ +#define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6 +size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weightTable, size_t wtSize) +{ + BYTE* const ostart = (BYTE*) dst; + BYTE* op = ostart; + BYTE* const oend = ostart + dstSize; + + U32 maxSymbolValue = HUF_TABLELOG_MAX; + U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER; + + FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)]; + BYTE scratchBuffer[1< not compressible */ + } + + tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue); + CHECK_F( FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue) ); + + /* Write table description header */ + { CHECK_V_F(hSize, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) ); + op += hSize; + } + + /* Compress */ + CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, sizeof(scratchBuffer)) ); + { CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, weightTable, wtSize, CTable) ); + if (cSize == 0) return 0; /* not enough space for compressed data */ + op += cSize; + } + + return op-ostart; +} + + +struct HUF_CElt_s { + U16 val; + BYTE nbBits; +}; /* typedef'd to HUF_CElt within "huf.h" */ + +/*! HUF_writeCTable() : + `CTable` : Huffman tree to save, using huf representation. + @return : size of saved CTable */ +size_t HUF_writeCTable (void* dst, size_t maxDstSize, + const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog) +{ + BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */ + BYTE huffWeight[HUF_SYMBOLVALUE_MAX]; + BYTE* op = (BYTE*)dst; + U32 n; + + /* check conditions */ + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge); + + /* convert to weight */ + bitsToWeight[0] = 0; + for (n=1; n1) & (hSize < maxSymbolValue/2)) { /* FSE compressed */ + op[0] = (BYTE)hSize; + return hSize+1; + } } + + /* write raw values as 4-bits (max : 15) */ + if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */ + if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */ + op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1)); + huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */ + for (n=0; n HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + if (nbSymbols > *maxSymbolValuePtr+1) return ERROR(maxSymbolValue_tooSmall); + + /* Prepare base value per rank */ + { U32 n, nextRankStart = 0; + for (n=1; n<=tableLog; n++) { + U32 current = nextRankStart; + nextRankStart += (rankVal[n] << (n-1)); + rankVal[n] = current; + } } + + /* fill nbBits */ + { U32 n; for (n=0; nn=tableLog+1 */ + U16 valPerRank[HUF_TABLELOG_MAX+2] = {0}; + { U32 n; for (n=0; n0; n--) { /* start at n=tablelog <-> w=1 */ + valPerRank[n] = min; /* get starting value within each rank */ + min += nbPerRank[n]; + min >>= 1; + } } + /* assign value within rank, symbol order */ + { U32 n; for (n=0; n maxNbBits */ + + /* there are several too large elements (at least >= 2) */ + { int totalCost = 0; + const U32 baseCost = 1 << (largestBits - maxNbBits); + U32 n = lastNonNull; + + while (huffNode[n].nbBits > maxNbBits) { + totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)); + huffNode[n].nbBits = (BYTE)maxNbBits; + n --; + } /* n stops at huffNode[n].nbBits <= maxNbBits */ + while (huffNode[n].nbBits == maxNbBits) n--; /* n end at index of smallest symbol using < maxNbBits */ + + /* renorm totalCost */ + totalCost >>= (largestBits - maxNbBits); /* note : totalCost is necessarily a multiple of baseCost */ + + /* repay normalized cost */ + { U32 const noSymbol = 0xF0F0F0F0; + U32 rankLast[HUF_TABLELOG_MAX+2]; + int pos; + + /* Get pos of last (smallest) symbol per rank */ + memset(rankLast, 0xF0, sizeof(rankLast)); + { U32 currentNbBits = maxNbBits; + for (pos=n ; pos >= 0; pos--) { + if (huffNode[pos].nbBits >= currentNbBits) continue; + currentNbBits = huffNode[pos].nbBits; /* < maxNbBits */ + rankLast[maxNbBits-currentNbBits] = pos; + } } + + while (totalCost > 0) { + U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1; + for ( ; nBitsToDecrease > 1; nBitsToDecrease--) { + U32 highPos = rankLast[nBitsToDecrease]; + U32 lowPos = rankLast[nBitsToDecrease-1]; + if (highPos == noSymbol) continue; + if (lowPos == noSymbol) break; + { U32 const highTotal = huffNode[highPos].count; + U32 const lowTotal = 2 * huffNode[lowPos].count; + if (highTotal <= lowTotal) break; + } } + /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */ + /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ + while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) + nBitsToDecrease ++; + totalCost -= 1 << (nBitsToDecrease-1); + if (rankLast[nBitsToDecrease-1] == noSymbol) + rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; /* this rank is no longer empty */ + huffNode[rankLast[nBitsToDecrease]].nbBits ++; + if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */ + rankLast[nBitsToDecrease] = noSymbol; + else { + rankLast[nBitsToDecrease]--; + if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease) + rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */ + } } /* while (totalCost > 0) */ + + while (totalCost < 0) { /* Sometimes, cost correction overshoot */ + if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */ + while (huffNode[n].nbBits == maxNbBits) n--; + huffNode[n+1].nbBits--; + rankLast[1] = n+1; + totalCost++; + continue; + } + huffNode[ rankLast[1] + 1 ].nbBits--; + rankLast[1]++; + totalCost ++; + } } } /* there are several too large elements (at least >= 2) */ + + return maxNbBits; +} + + +typedef struct { + U32 base; + U32 current; +} rankPos; + +static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue) +{ + rankPos rank[32]; + U32 n; + + memset(rank, 0, sizeof(rank)); + for (n=0; n<=maxSymbolValue; n++) { + U32 r = BIT_highbit32(count[n] + 1); + rank[r].base ++; + } + for (n=30; n>0; n--) rank[n-1].base += rank[n].base; + for (n=0; n<32; n++) rank[n].current = rank[n].base; + for (n=0; n<=maxSymbolValue; n++) { + U32 const c = count[n]; + U32 const r = BIT_highbit32(c+1) + 1; + U32 pos = rank[r].current++; + while ((pos > rank[r].base) && (c > huffNode[pos-1].count)) huffNode[pos]=huffNode[pos-1], pos--; + huffNode[pos].count = c; + huffNode[pos].byte = (BYTE)n; + } +} + + +/** HUF_buildCTable_wksp() : + * Same as HUF_buildCTable(), but using externally allocated scratch buffer. + * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned. + */ +#define STARTNODE (HUF_SYMBOLVALUE_MAX+1) +typedef nodeElt huffNodeTable[2*HUF_SYMBOLVALUE_MAX+1 +1]; +size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize) +{ + nodeElt* const huffNode0 = (nodeElt*)workSpace; + nodeElt* const huffNode = huffNode0+1; + U32 n, nonNullRank; + int lowS, lowN; + U16 nodeNb = STARTNODE; + U32 nodeRoot; + + /* safety checks */ + if (wkspSize < sizeof(huffNodeTable)) return ERROR(GENERIC); /* workSpace is not large enough */ + if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT; + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(GENERIC); + memset(huffNode0, 0, sizeof(huffNodeTable)); + + /* sort, decreasing order */ + HUF_sort(huffNode, count, maxSymbolValue); + + /* init for parents */ + nonNullRank = maxSymbolValue; + while(huffNode[nonNullRank].count == 0) nonNullRank--; + lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb; + huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count; + huffNode[lowS].parent = huffNode[lowS-1].parent = nodeNb; + nodeNb++; lowS-=2; + for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30); + huffNode0[0].count = (U32)(1U<<31); /* fake entry, strong barrier */ + + /* create parents */ + while (nodeNb <= nodeRoot) { + U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; + U32 n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; + huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count; + huffNode[n1].parent = huffNode[n2].parent = nodeNb; + nodeNb++; + } + + /* distribute weights (unlimited tree height) */ + huffNode[nodeRoot].nbBits = 0; + for (n=nodeRoot-1; n>=STARTNODE; n--) + huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; + for (n=0; n<=nonNullRank; n++) + huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; + + /* enforce maxTableLog */ + maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits); + + /* fill result into tree (val, nbBits) */ + { U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0}; + U16 valPerRank[HUF_TABLELOG_MAX+1] = {0}; + if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */ + for (n=0; n<=nonNullRank; n++) + nbPerRank[huffNode[n].nbBits]++; + /* determine stating value per rank */ + { U16 min = 0; + for (n=maxNbBits; n>0; n--) { + valPerRank[n] = min; /* get starting value within each rank */ + min += nbPerRank[n]; + min >>= 1; + } } + for (n=0; n<=maxSymbolValue; n++) + tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */ + for (n=0; n<=maxSymbolValue; n++) + tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */ + } + + return maxNbBits; +} + +/** HUF_buildCTable() : + * Note : count is used before tree is written, so they can safely overlap + */ +size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits) +{ + huffNodeTable nodeTable; + return HUF_buildCTable_wksp(tree, count, maxSymbolValue, maxNbBits, nodeTable, sizeof(nodeTable)); +} + +static size_t HUF_estimateCompressedSize(HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) +{ + size_t nbBits = 0; + int s; + for (s = 0; s <= (int)maxSymbolValue; ++s) { + nbBits += CTable[s].nbBits * count[s]; + } + return nbBits >> 3; +} + +static int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) { + int bad = 0; + int s; + for (s = 0; s <= (int)maxSymbolValue; ++s) { + bad |= (count[s] != 0) & (CTable[s].nbBits == 0); + } + return !bad; +} + +static void HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable) +{ + BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits); +} + +size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); } + +#define HUF_FLUSHBITS(s) BIT_flushBits(s) + +#define HUF_FLUSHBITS_1(stream) \ + if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*2+7) HUF_FLUSHBITS(stream) + +#define HUF_FLUSHBITS_2(stream) \ + if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*4+7) HUF_FLUSHBITS(stream) + +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) +{ + const BYTE* ip = (const BYTE*) src; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + size_t n; + BIT_CStream_t bitC; + + /* init */ + if (dstSize < 8) return 0; /* not enough space to compress */ + { size_t const initErr = BIT_initCStream(&bitC, op, oend-op); + if (HUF_isError(initErr)) return 0; } + + n = srcSize & ~3; /* join to mod 4 */ + switch (srcSize & 3) + { + case 3 : HUF_encodeSymbol(&bitC, ip[n+ 2], CTable); + HUF_FLUSHBITS_2(&bitC); + /* fall-through */ + case 2 : HUF_encodeSymbol(&bitC, ip[n+ 1], CTable); + HUF_FLUSHBITS_1(&bitC); + /* fall-through */ + case 1 : HUF_encodeSymbol(&bitC, ip[n+ 0], CTable); + HUF_FLUSHBITS(&bitC); + /* fall-through */ + case 0 : /* fall-through */ + default: break; + } + + for (; n>0; n-=4) { /* note : n&3==0 at this stage */ + HUF_encodeSymbol(&bitC, ip[n- 1], CTable); + HUF_FLUSHBITS_1(&bitC); + HUF_encodeSymbol(&bitC, ip[n- 2], CTable); + HUF_FLUSHBITS_2(&bitC); + HUF_encodeSymbol(&bitC, ip[n- 3], CTable); + HUF_FLUSHBITS_1(&bitC); + HUF_encodeSymbol(&bitC, ip[n- 4], CTable); + HUF_FLUSHBITS(&bitC); + } + + return BIT_closeCStream(&bitC); +} + + +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) +{ + size_t const segmentSize = (srcSize+3)/4; /* first 3 segments */ + const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + + if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */ + if (srcSize < 12) return 0; /* no saving possible : too small input */ + op += 6; /* jumpTable */ + + { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) ); + if (cSize==0) return 0; + MEM_writeLE16(ostart, (U16)cSize); + op += cSize; + } + + ip += segmentSize; + { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) ); + if (cSize==0) return 0; + MEM_writeLE16(ostart+2, (U16)cSize); + op += cSize; + } + + ip += segmentSize; + { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) ); + if (cSize==0) return 0; + MEM_writeLE16(ostart+4, (U16)cSize); + op += cSize; + } + + ip += segmentSize; + { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable) ); + if (cSize==0) return 0; + op += cSize; + } + + return op-ostart; +} + + +static size_t HUF_compressCTable_internal( + BYTE* const ostart, BYTE* op, BYTE* const oend, + const void* src, size_t srcSize, + unsigned singleStream, const HUF_CElt* CTable) +{ + size_t const cSize = singleStream ? + HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) : + HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable); + if (HUF_isError(cSize)) { return cSize; } + if (cSize==0) { return 0; } /* uncompressible */ + op += cSize; + /* check compressibility */ + if ((size_t)(op-ostart) >= srcSize-1) { return 0; } + return op-ostart; +} + + +/* `workSpace` must a table of at least 1024 unsigned */ +static size_t HUF_compress_internal ( + void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + unsigned singleStream, + void* workSpace, size_t wkspSize, + HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat) +{ + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + + U32* count; + size_t const countSize = sizeof(U32) * (HUF_SYMBOLVALUE_MAX + 1); + HUF_CElt* CTable; + size_t const CTableSize = sizeof(HUF_CElt) * (HUF_SYMBOLVALUE_MAX + 1); + + /* checks & inits */ + if (wkspSize < sizeof(huffNodeTable) + countSize + CTableSize) return ERROR(GENERIC); + if (!srcSize) return 0; /* Uncompressed (note : 1 means rle, so first byte must be correct) */ + if (!dstSize) return 0; /* cannot fit within dst budget */ + if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */ + if (huffLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX; + if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT; + + count = (U32*)workSpace; + workSpace = (BYTE*)workSpace + countSize; + wkspSize -= countSize; + CTable = (HUF_CElt*)workSpace; + workSpace = (BYTE*)workSpace + CTableSize; + wkspSize -= CTableSize; + + /* Heuristic : If we don't need to check the validity of the old table use the old table for small inputs */ + if (preferRepeat && repeat && *repeat == HUF_repeat_valid) { + return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable); + } + + /* Scan input and build symbol stats */ + { CHECK_V_F(largest, FSE_count_wksp (count, &maxSymbolValue, (const BYTE*)src, srcSize, (U32*)workSpace) ); + if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */ + if (largest <= (srcSize >> 7)+1) return 0; /* Fast heuristic : not compressible enough */ + } + + /* Check validity of previous table */ + if (repeat && *repeat == HUF_repeat_check && !HUF_validateCTable(oldHufTable, count, maxSymbolValue)) { + *repeat = HUF_repeat_none; + } + /* Heuristic : use existing table for small inputs */ + if (preferRepeat && repeat && *repeat != HUF_repeat_none) { + return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable); + } + + /* Build Huffman Tree */ + huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); + { CHECK_V_F(maxBits, HUF_buildCTable_wksp (CTable, count, maxSymbolValue, huffLog, workSpace, wkspSize) ); + huffLog = (U32)maxBits; + /* Zero the unused symbols so we can check it for validity */ + memset(CTable + maxSymbolValue + 1, 0, CTableSize - (maxSymbolValue + 1) * sizeof(HUF_CElt)); + } + + /* Write table description header */ + { CHECK_V_F(hSize, HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog) ); + /* Check if using the previous table will be beneficial */ + if (repeat && *repeat != HUF_repeat_none) { + size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, count, maxSymbolValue); + size_t const newSize = HUF_estimateCompressedSize(CTable, count, maxSymbolValue); + if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) { + return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable); + } + } + /* Use the new table */ + if (hSize + 12ul >= srcSize) { return 0; } + op += hSize; + if (repeat) { *repeat = HUF_repeat_none; } + if (oldHufTable) { memcpy(oldHufTable, CTable, CTableSize); } /* Save the new table */ + } + return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, CTable); +} + + +size_t HUF_compress1X_wksp (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + void* workSpace, size_t wkspSize) +{ + return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, NULL, NULL, 0); +} + +size_t HUF_compress1X_repeat (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + void* workSpace, size_t wkspSize, + HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat) +{ + return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, hufTable, repeat, preferRepeat); +} + +size_t HUF_compress1X (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog) +{ + unsigned workSpace[1024]; + return HUF_compress1X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace)); +} + +size_t HUF_compress4X_wksp (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + void* workSpace, size_t wkspSize) +{ + return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, NULL, NULL, 0); +} + +size_t HUF_compress4X_repeat (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + void* workSpace, size_t wkspSize, + HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat) +{ + return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, hufTable, repeat, preferRepeat); +} + +size_t HUF_compress2 (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog) +{ + unsigned workSpace[1024]; + return HUF_compress4X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace)); +} + +size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize) +{ + return HUF_compress2(dst, maxDstSize, src, (U32)srcSize, 255, HUF_TABLELOG_DEFAULT); +} diff --git a/src/borg/algorithms/zstd/lib/compress/zstd_compress.c b/src/borg/algorithms/zstd/lib/compress/zstd_compress.c new file mode 100644 index 0000000000..2c46c79f1c --- /dev/null +++ b/src/borg/algorithms/zstd/lib/compress/zstd_compress.c @@ -0,0 +1,3023 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/*-************************************* +* Tuning parameters +***************************************/ +#ifndef ZSTD_CLEVEL_DEFAULT +# define ZSTD_CLEVEL_DEFAULT 3 +#endif + + +/*-************************************* +* Dependencies +***************************************/ +#include /* memset */ +#include "mem.h" +#define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */ +#include "fse.h" +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" +#include "zstd_compress.h" +#include "zstd_fast.h" +#include "zstd_double_fast.h" +#include "zstd_lazy.h" +#include "zstd_opt.h" +#include "zstd_ldm.h" + + +/*-************************************* +* Helper functions +***************************************/ +size_t ZSTD_compressBound(size_t srcSize) { + return ZSTD_COMPRESSBOUND(srcSize); +} + + +/*-************************************* +* Sequence storage +***************************************/ +static void ZSTD_resetSeqStore(seqStore_t* ssPtr) +{ + ssPtr->lit = ssPtr->litStart; + ssPtr->sequences = ssPtr->sequencesStart; + ssPtr->longLengthID = 0; +} + + +/*-************************************* +* Context memory management +***************************************/ +struct ZSTD_CDict_s { + void* dictBuffer; + const void* dictContent; + size_t dictContentSize; + ZSTD_CCtx* refContext; +}; /* typedef'd to ZSTD_CDict within "zstd.h" */ + +ZSTD_CCtx* ZSTD_createCCtx(void) +{ + return ZSTD_createCCtx_advanced(ZSTD_defaultCMem); +} + +ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) +{ + ZSTD_CCtx* cctx; + + if (!customMem.customAlloc ^ !customMem.customFree) return NULL; + + cctx = (ZSTD_CCtx*) ZSTD_calloc(sizeof(ZSTD_CCtx), customMem); + if (!cctx) return NULL; + cctx->customMem = customMem; + cctx->requestedParams.compressionLevel = ZSTD_CLEVEL_DEFAULT; + ZSTD_STATIC_ASSERT(zcss_init==0); + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN==(0ULL - 1)); + return cctx; +} + +ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize) +{ + ZSTD_CCtx* const cctx = (ZSTD_CCtx*) workspace; + if (workspaceSize <= sizeof(ZSTD_CCtx)) return NULL; /* minimum size */ + if ((size_t)workspace & 7) return NULL; /* must be 8-aligned */ + memset(workspace, 0, workspaceSize); /* may be a bit generous, could memset be smaller ? */ + cctx->staticSize = workspaceSize; + cctx->workSpace = (void*)(cctx+1); + cctx->workSpaceSize = workspaceSize - sizeof(ZSTD_CCtx); + + /* entropy space (never moves) */ + if (cctx->workSpaceSize < sizeof(ZSTD_entropyCTables_t)) return NULL; + assert(((size_t)cctx->workSpace & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ + cctx->entropy = (ZSTD_entropyCTables_t*)cctx->workSpace; + + return cctx; +} + +size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) +{ + if (cctx==NULL) return 0; /* support free on NULL */ + if (cctx->staticSize) return ERROR(memory_allocation); /* not compatible with static CCtx */ + ZSTD_free(cctx->workSpace, cctx->customMem); + cctx->workSpace = NULL; + ZSTD_freeCDict(cctx->cdictLocal); + cctx->cdictLocal = NULL; +#ifdef ZSTD_MULTITHREAD + ZSTDMT_freeCCtx(cctx->mtctx); + cctx->mtctx = NULL; +#endif + ZSTD_free(cctx, cctx->customMem); + return 0; /* reserved as a potential error code in the future */ +} + + +static size_t ZSTD_sizeof_mtctx(const ZSTD_CCtx* cctx) +{ +#ifdef ZSTD_MULTITHREAD + return ZSTDMT_sizeof_CCtx(cctx->mtctx); +#else + (void) cctx; + return 0; +#endif +} + + +size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) +{ + if (cctx==NULL) return 0; /* support sizeof on NULL */ + DEBUGLOG(3, "sizeof(*cctx) : %u", (U32)sizeof(*cctx)); + DEBUGLOG(3, "workSpaceSize (including streaming buffers): %u", (U32)cctx->workSpaceSize); + DEBUGLOG(3, "inner cdict : %u", (U32)ZSTD_sizeof_CDict(cctx->cdictLocal)); + DEBUGLOG(3, "inner MTCTX : %u", (U32)ZSTD_sizeof_mtctx(cctx)); + return sizeof(*cctx) + cctx->workSpaceSize + + ZSTD_sizeof_CDict(cctx->cdictLocal) + + ZSTD_sizeof_mtctx(cctx); +} + +size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) +{ + return ZSTD_sizeof_CCtx(zcs); /* same object */ +} + +/* private API call, for dictBuilder only */ +const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); } + +#define ZSTD_CLEVEL_CUSTOM 999 + +static ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( + ZSTD_CCtx_params params, U64 srcSizeHint, size_t dictSize) +{ + return (params.compressionLevel == ZSTD_CLEVEL_CUSTOM ? + params.cParams : + ZSTD_getCParams(params.compressionLevel, srcSizeHint, dictSize)); +} + +static void ZSTD_cLevelToCCtxParams_srcSize(ZSTD_CCtx_params* params, U64 srcSize) +{ + params->cParams = ZSTD_getCParamsFromCCtxParams(*params, srcSize, 0); + params->compressionLevel = ZSTD_CLEVEL_CUSTOM; +} + +static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx) +{ + ZSTD_cLevelToCCtxParams_srcSize( + &cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1); +} + +static void ZSTD_cLevelToCCtxParams(ZSTD_CCtx_params* params) +{ + ZSTD_cLevelToCCtxParams_srcSize(params, 0); +} + +static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams( + ZSTD_compressionParameters cParams) +{ + ZSTD_CCtx_params cctxParams; + memset(&cctxParams, 0, sizeof(cctxParams)); + cctxParams.cParams = cParams; + cctxParams.compressionLevel = ZSTD_CLEVEL_CUSTOM; + return cctxParams; +} + +static ZSTD_CCtx_params* ZSTD_createCCtxParams_advanced( + ZSTD_customMem customMem) +{ + ZSTD_CCtx_params* params; + if (!customMem.customAlloc ^ !customMem.customFree) return NULL; + params = (ZSTD_CCtx_params*)ZSTD_calloc( + sizeof(ZSTD_CCtx_params), customMem); + if (!params) { return NULL; } + params->customMem = customMem; + params->compressionLevel = ZSTD_CLEVEL_DEFAULT; + return params; +} + +ZSTD_CCtx_params* ZSTD_createCCtxParams(void) +{ + return ZSTD_createCCtxParams_advanced(ZSTD_defaultCMem); +} + +size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params) +{ + if (params == NULL) { return 0; } + ZSTD_free(params, params->customMem); + return 0; +} + +size_t ZSTD_resetCCtxParams(ZSTD_CCtx_params* params) +{ + return ZSTD_initCCtxParams(params, ZSTD_CLEVEL_DEFAULT); +} + +size_t ZSTD_initCCtxParams(ZSTD_CCtx_params* cctxParams, int compressionLevel) { + if (!cctxParams) { return ERROR(GENERIC); } + memset(cctxParams, 0, sizeof(*cctxParams)); + cctxParams->compressionLevel = compressionLevel; + return 0; +} + +size_t ZSTD_initCCtxParams_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params) +{ + if (!cctxParams) { return ERROR(GENERIC); } + CHECK_F( ZSTD_checkCParams(params.cParams) ); + memset(cctxParams, 0, sizeof(*cctxParams)); + cctxParams->cParams = params.cParams; + cctxParams->fParams = params.fParams; + cctxParams->compressionLevel = ZSTD_CLEVEL_CUSTOM; + return 0; +} + +static ZSTD_CCtx_params ZSTD_assignParamsToCCtxParams( + ZSTD_CCtx_params cctxParams, ZSTD_parameters params) +{ + ZSTD_CCtx_params ret = cctxParams; + ret.cParams = params.cParams; + ret.fParams = params.fParams; + ret.compressionLevel = ZSTD_CLEVEL_CUSTOM; + return ret; +} + +#define CLAMPCHECK(val,min,max) { \ + if (((val)<(min)) | ((val)>(max))) { \ + return ERROR(parameter_outOfBound); \ +} } + +size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value) +{ + if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); + + switch(param) + { + case ZSTD_p_format : + return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); + + case ZSTD_p_compressionLevel: + if (value == 0) return 0; /* special value : 0 means "don't change anything" */ + if (cctx->cdict) return ERROR(stage_wrong); + return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); + + case ZSTD_p_windowLog: + case ZSTD_p_hashLog: + case ZSTD_p_chainLog: + case ZSTD_p_searchLog: + case ZSTD_p_minMatch: + case ZSTD_p_targetLength: + case ZSTD_p_compressionStrategy: + if (value == 0) return 0; /* special value : 0 means "don't change anything" */ + if (cctx->cdict) return ERROR(stage_wrong); + ZSTD_cLevelToCParams(cctx); /* Can optimize if srcSize is known */ + return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); + + case ZSTD_p_contentSizeFlag: + case ZSTD_p_checksumFlag: + case ZSTD_p_dictIDFlag: + return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); + + case ZSTD_p_forceMaxWindow : /* Force back-references to remain < windowSize, + * even when referencing into Dictionary content + * default : 0 when using a CDict, 1 when using a Prefix */ + cctx->loadedDictEnd = 0; + return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); + + case ZSTD_p_nbThreads: + if (value==0) return 0; + DEBUGLOG(5, " setting nbThreads : %u", value); + if (value > 1 && cctx->staticSize) { + return ERROR(parameter_unsupported); /* MT not compatible with static alloc */ + } + return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); + + case ZSTD_p_jobSize: + return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); + + case ZSTD_p_overlapSizeLog: + DEBUGLOG(5, " setting overlap with nbThreads == %u", cctx->requestedParams.nbThreads); + return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); + + case ZSTD_p_enableLongDistanceMatching: + if (cctx->cdict) return ERROR(stage_wrong); + if (value != 0) { + ZSTD_cLevelToCParams(cctx); + } + return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); + + case ZSTD_p_ldmHashLog: + case ZSTD_p_ldmMinMatch: + if (value == 0) return 0; /* special value : 0 means "don't change anything" */ + if (cctx->cdict) return ERROR(stage_wrong); + return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); + + case ZSTD_p_ldmBucketSizeLog: + case ZSTD_p_ldmHashEveryLog: + if (cctx->cdict) return ERROR(stage_wrong); + return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); + + default: return ERROR(parameter_unsupported); + } +} + +size_t ZSTD_CCtxParam_setParameter( + ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned value) +{ + switch(param) + { + case ZSTD_p_format : + if (value > (unsigned)ZSTD_f_zstd1_magicless) + return ERROR(parameter_unsupported); + params->format = (ZSTD_format_e)value; + return 0; + + case ZSTD_p_compressionLevel : + if ((int)value > ZSTD_maxCLevel()) value = ZSTD_maxCLevel(); + if (value == 0) return 0; + params->compressionLevel = value; + return 0; + + case ZSTD_p_windowLog : + if (value == 0) return 0; + CLAMPCHECK(value, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); + ZSTD_cLevelToCCtxParams(params); + params->cParams.windowLog = value; + return 0; + + case ZSTD_p_hashLog : + if (value == 0) return 0; + CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); + ZSTD_cLevelToCCtxParams(params); + params->cParams.hashLog = value; + return 0; + + case ZSTD_p_chainLog : + if (value == 0) return 0; + CLAMPCHECK(value, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); + ZSTD_cLevelToCCtxParams(params); + params->cParams.chainLog = value; + return 0; + + case ZSTD_p_searchLog : + if (value == 0) return 0; + CLAMPCHECK(value, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); + ZSTD_cLevelToCCtxParams(params); + params->cParams.searchLog = value; + return 0; + + case ZSTD_p_minMatch : + if (value == 0) return 0; + CLAMPCHECK(value, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); + ZSTD_cLevelToCCtxParams(params); + params->cParams.searchLength = value; + return 0; + + case ZSTD_p_targetLength : + if (value == 0) return 0; + CLAMPCHECK(value, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX); + ZSTD_cLevelToCCtxParams(params); + params->cParams.targetLength = value; + return 0; + + case ZSTD_p_compressionStrategy : + if (value == 0) return 0; + CLAMPCHECK(value, (unsigned)ZSTD_fast, (unsigned)ZSTD_btultra); + ZSTD_cLevelToCCtxParams(params); + params->cParams.strategy = (ZSTD_strategy)value; + return 0; + + case ZSTD_p_contentSizeFlag : + /* Content size written in frame header _when known_ (default:1) */ + DEBUGLOG(5, "set content size flag = %u", (value>0)); + params->fParams.contentSizeFlag = value > 0; + return 0; + + case ZSTD_p_checksumFlag : + /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */ + params->fParams.checksumFlag = value > 0; + return 0; + + case ZSTD_p_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */ + DEBUGLOG(5, "set dictIDFlag = %u", (value>0)); + params->fParams.noDictIDFlag = (value == 0); + return 0; + + case ZSTD_p_forceMaxWindow : + params->forceWindow = value > 0; + return 0; + + case ZSTD_p_nbThreads : + if (value == 0) return 0; +#ifndef ZSTD_MULTITHREAD + if (value > 1) return ERROR(parameter_unsupported); + return 0; +#else + return ZSTDMT_initializeCCtxParameters(params, value); +#endif + + case ZSTD_p_jobSize : +#ifndef ZSTD_MULTITHREAD + return ERROR(parameter_unsupported); +#else + if (params->nbThreads <= 1) return ERROR(parameter_unsupported); + return ZSTDMT_CCtxParam_setMTCtxParameter(params, ZSTDMT_p_sectionSize, value); +#endif + + case ZSTD_p_overlapSizeLog : +#ifndef ZSTD_MULTITHREAD + return ERROR(parameter_unsupported); +#else + if (params->nbThreads <= 1) return ERROR(parameter_unsupported); + return ZSTDMT_CCtxParam_setMTCtxParameter(params, ZSTDMT_p_overlapSectionLog, value); +#endif + + case ZSTD_p_enableLongDistanceMatching : + if (value != 0) { + ZSTD_cLevelToCCtxParams(params); + params->cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG; + } + return ZSTD_ldm_initializeParameters(¶ms->ldmParams, value); + + case ZSTD_p_ldmHashLog : + if (value == 0) return 0; + CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); + params->ldmParams.hashLog = value; + return 0; + + case ZSTD_p_ldmMinMatch : + if (value == 0) return 0; + CLAMPCHECK(value, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX); + params->ldmParams.minMatchLength = value; + return 0; + + case ZSTD_p_ldmBucketSizeLog : + if (value > ZSTD_LDM_BUCKETSIZELOG_MAX) { + return ERROR(parameter_outOfBound); + } + params->ldmParams.bucketSizeLog = value; + return 0; + + case ZSTD_p_ldmHashEveryLog : + if (value > ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN) { + return ERROR(parameter_outOfBound); + } + params->ldmParams.hashEveryLog = value; + return 0; + + default: return ERROR(parameter_unsupported); + } +} + +/** + * This function should be updated whenever ZSTD_CCtx_params is updated. + * Parameters are copied manually before the dictionary is loaded. + * The multithreading parameters jobSize and overlapSizeLog are set only if + * nbThreads > 1. + * + * Pledged srcSize is treated as unknown. + */ +size_t ZSTD_CCtx_setParametersUsingCCtxParams( + ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params) +{ + if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); + if (cctx->cdict) return ERROR(stage_wrong); + + /* Assume the compression and frame parameters are validated */ + cctx->requestedParams.cParams = params->cParams; + cctx->requestedParams.fParams = params->fParams; + cctx->requestedParams.compressionLevel = params->compressionLevel; + + /* Set force window explicitly since it sets cctx->loadedDictEnd */ + CHECK_F( ZSTD_CCtx_setParameter( + cctx, ZSTD_p_forceMaxWindow, params->forceWindow) ); + + /* Set multithreading parameters explicitly */ + CHECK_F( ZSTD_CCtx_setParameter(cctx, ZSTD_p_nbThreads, params->nbThreads) ); + if (params->nbThreads > 1) { + CHECK_F( ZSTD_CCtx_setParameter(cctx, ZSTD_p_jobSize, params->jobSize) ); + CHECK_F( ZSTD_CCtx_setParameter( + cctx, ZSTD_p_overlapSizeLog, params->overlapSizeLog) ); + } + + /* Copy long distance matching parameters */ + cctx->requestedParams.ldmParams = params->ldmParams; + + /* customMem is used only for create/free params and can be ignored */ + return 0; +} + +ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, " setting pledgedSrcSize to %u", (U32)pledgedSrcSize); + if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); + cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1; + return 0; +} + +size_t ZSTD_CCtx_loadDictionary_advanced( + ZSTD_CCtx* cctx, const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode) +{ + if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); + if (cctx->staticSize) return ERROR(memory_allocation); /* no malloc for static CCtx */ + DEBUGLOG(4, "load dictionary of size %u", (U32)dictSize); + ZSTD_freeCDict(cctx->cdictLocal); /* in case one already exists */ + if (dict==NULL || dictSize==0) { /* no dictionary mode */ + cctx->cdictLocal = NULL; + cctx->cdict = NULL; + } else { + ZSTD_compressionParameters const cParams = + ZSTD_getCParamsFromCCtxParams(cctx->requestedParams, 0, dictSize); + cctx->cdictLocal = ZSTD_createCDict_advanced( + dict, dictSize, + dictLoadMethod, dictMode, + cParams, cctx->customMem); + cctx->cdict = cctx->cdictLocal; + if (cctx->cdictLocal == NULL) + return ERROR(memory_allocation); + } + return 0; +} + +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference( + ZSTD_CCtx* cctx, const void* dict, size_t dictSize) +{ + return ZSTD_CCtx_loadDictionary_advanced( + cctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dm_auto); +} + +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize) +{ + return ZSTD_CCtx_loadDictionary_advanced( + cctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dm_auto); +} + + +size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) +{ + if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); + cctx->cdict = cdict; + memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* exclusive */ + return 0; +} + +size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize) +{ + return ZSTD_CCtx_refPrefix_advanced(cctx, prefix, prefixSize, ZSTD_dm_rawContent); +} + +size_t ZSTD_CCtx_refPrefix_advanced( + ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictMode_e dictMode) +{ + if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); + cctx->cdict = NULL; /* prefix discards any prior cdict */ + cctx->prefixDict.dict = prefix; + cctx->prefixDict.dictSize = prefixSize; + cctx->prefixDict.dictMode = dictMode; + return 0; +} + +static void ZSTD_startNewCompression(ZSTD_CCtx* cctx) +{ + cctx->streamStage = zcss_init; + cctx->pledgedSrcSizePlusOne = 0; +} + +/*! ZSTD_CCtx_reset() : + * Also dumps dictionary */ +void ZSTD_CCtx_reset(ZSTD_CCtx* cctx) +{ + ZSTD_startNewCompression(cctx); + cctx->cdict = NULL; +} + +/** ZSTD_checkCParams() : + control CParam values remain within authorized range. + @return : 0, or an error code if one value is beyond authorized range */ +size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) +{ + CLAMPCHECK(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); + CLAMPCHECK(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); + CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); + CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); + CLAMPCHECK(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); + CLAMPCHECK(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX); + if ((U32)(cParams.strategy) > (U32)ZSTD_btultra) + return ERROR(parameter_unsupported); + return 0; +} + +/** ZSTD_clampCParams() : + * make CParam values within valid range. + * @return : valid CParams */ +static ZSTD_compressionParameters ZSTD_clampCParams(ZSTD_compressionParameters cParams) +{ +# define CLAMP(val,min,max) { \ + if (valmax) val=max; \ + } + CLAMP(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); + CLAMP(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); + CLAMP(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); + CLAMP(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); + CLAMP(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); + CLAMP(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX); + if ((U32)(cParams.strategy) > (U32)ZSTD_btultra) cParams.strategy = ZSTD_btultra; + return cParams; +} + +/** ZSTD_cycleLog() : + * condition for correct operation : hashLog > 1 */ +static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) +{ + U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2); + return hashLog - btScale; +} + +/** ZSTD_adjustCParams_internal() : + optimize `cPar` for a given input (`srcSize` and `dictSize`). + mostly downsizing to reduce memory consumption and initialization latency. + Both `srcSize` and `dictSize` are optional (use 0 if unknown). + Note : cPar is considered validated at this stage. Use ZSTD_checkCParams() to ensure that condition. */ +ZSTD_compressionParameters ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize) +{ + static const U64 minSrcSize = 513; /* (1<<9) + 1 */ + static const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1); + assert(ZSTD_checkCParams(cPar)==0); + + if (dictSize && (srcSize+1<2) /* srcSize unknown */ ) + srcSize = minSrcSize; /* presumed small when there is a dictionary */ + else if (srcSize == 0) + srcSize = ZSTD_CONTENTSIZE_UNKNOWN; /* 0 == unknown : presumed large */ + + /* resize windowLog if input is small enough, to use less memory */ + if ( (srcSize < maxWindowResize) + && (dictSize < maxWindowResize) ) { + U32 const tSize = (U32)(srcSize + dictSize); + static U32 const hashSizeMin = 1 << ZSTD_HASHLOG_MIN; + U32 const srcLog = (tSize < hashSizeMin) ? ZSTD_HASHLOG_MIN : + ZSTD_highbit32(tSize-1) + 1; + if (cPar.windowLog > srcLog) cPar.windowLog = srcLog; + } + if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog; + { U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy); + if (cycleLog > cPar.windowLog) + cPar.chainLog -= (cycleLog - cPar.windowLog); + } + + if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) + cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */ + + return cPar; +} + +ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize) +{ + cPar = ZSTD_clampCParams(cPar); + return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize); +} + +size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) +{ + /* Estimate CCtx size is supported for single-threaded compression only. */ + if (params->nbThreads > 1) { return ERROR(GENERIC); } + { ZSTD_compressionParameters const cParams = + ZSTD_getCParamsFromCCtxParams(*params, 0, 0); + size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); + U32 const divider = (cParams.searchLength==3) ? 3 : 4; + size_t const maxNbSeq = blockSize / divider; + size_t const tokenSpace = blockSize + 11*maxNbSeq; + size_t const chainSize = + (cParams.strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams.chainLog); + size_t const hSize = ((size_t)1) << cParams.hashLog; + U32 const hashLog3 = (cParams.searchLength>3) ? + 0 : MIN(ZSTD_HASHLOG3_MAX, cParams.windowLog); + size_t const h3Size = ((size_t)1) << hashLog3; + size_t const entropySpace = sizeof(ZSTD_entropyCTables_t); + size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); + + size_t const optBudget = + ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<ldmParams.enableLdm ? + ZSTD_ldm_getTableSize(params->ldmParams.hashLog, + params->ldmParams.bucketSizeLog) : 0; + + size_t const neededSpace = entropySpace + tableSpace + tokenSpace + + optSpace + ldmSpace; + + DEBUGLOG(5, "sizeof(ZSTD_CCtx) : %u", (U32)sizeof(ZSTD_CCtx)); + DEBUGLOG(5, "estimate workSpace : %u", (U32)neededSpace); + return sizeof(ZSTD_CCtx) + neededSpace; + } +} + +size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams) +{ + ZSTD_CCtx_params const params = ZSTD_makeCCtxParamsFromCParams(cParams); + return ZSTD_estimateCCtxSize_usingCCtxParams(¶ms); +} + +size_t ZSTD_estimateCCtxSize(int compressionLevel) +{ + ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, 0, 0); + return ZSTD_estimateCCtxSize_usingCParams(cParams); +} + +size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params) +{ + if (params->nbThreads > 1) { return ERROR(GENERIC); } + { size_t const CCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params); + size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params->cParams.windowLog); + size_t const inBuffSize = ((size_t)1 << params->cParams.windowLog) + blockSize; + size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1; + size_t const streamingSize = inBuffSize + outBuffSize; + + return CCtxSize + streamingSize; + } +} + +size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams) +{ + ZSTD_CCtx_params const params = ZSTD_makeCCtxParamsFromCParams(cParams); + return ZSTD_estimateCStreamSize_usingCCtxParams(¶ms); +} + +size_t ZSTD_estimateCStreamSize(int compressionLevel) { + ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, 0, 0); + return ZSTD_estimateCStreamSize_usingCParams(cParams); +} + +static U32 ZSTD_equivalentCParams(ZSTD_compressionParameters cParams1, + ZSTD_compressionParameters cParams2) +{ + U32 bslog1 = MIN(cParams1.windowLog, ZSTD_BLOCKSIZELOG_MAX); + U32 bslog2 = MIN(cParams2.windowLog, ZSTD_BLOCKSIZELOG_MAX); + return (bslog1 == bslog2) /* same block size */ + & (cParams1.hashLog == cParams2.hashLog) + & (cParams1.chainLog == cParams2.chainLog) + & (cParams1.strategy == cParams2.strategy) /* opt parser space */ + & ((cParams1.searchLength==3) == (cParams2.searchLength==3)); /* hashlog3 space */ +} + +/** The parameters are equivalent if ldm is not enabled in both sets or + * all the parameters are equivalent. */ +static U32 ZSTD_equivalentLdmParams(ldmParams_t ldmParams1, + ldmParams_t ldmParams2) +{ + return (!ldmParams1.enableLdm && !ldmParams2.enableLdm) || + (ldmParams1.enableLdm == ldmParams2.enableLdm && + ldmParams1.hashLog == ldmParams2.hashLog && + ldmParams1.bucketSizeLog == ldmParams2.bucketSizeLog && + ldmParams1.minMatchLength == ldmParams2.minMatchLength && + ldmParams1.hashEveryLog == ldmParams2.hashEveryLog); +} + +/** Equivalence for resetCCtx purposes */ +static U32 ZSTD_equivalentParams(ZSTD_CCtx_params params1, + ZSTD_CCtx_params params2) +{ + return ZSTD_equivalentCParams(params1.cParams, params2.cParams) && + ZSTD_equivalentLdmParams(params1.ldmParams, params2.ldmParams); +} + +/*! ZSTD_continueCCtx() : + * reuse CCtx without reset (note : requires no dictionary) */ +static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_CCtx_params params, U64 pledgedSrcSize) +{ + U32 const end = (U32)(cctx->nextSrc - cctx->base); + DEBUGLOG(4, "continue mode"); + cctx->appliedParams = params; + cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1; + cctx->consumedSrcSize = 0; + if (pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN) + cctx->appliedParams.fParams.contentSizeFlag = 0; + DEBUGLOG(4, "pledged content size : %u ; flag : %u", + (U32)pledgedSrcSize, cctx->appliedParams.fParams.contentSizeFlag); + cctx->lowLimit = end; + cctx->dictLimit = end; + cctx->nextToUpdate = end+1; + cctx->stage = ZSTDcs_init; + cctx->dictID = 0; + cctx->loadedDictEnd = 0; + { int i; for (i=0; iseqStore.rep[i] = repStartValue[i]; } + cctx->optState.litLengthSum = 0; /* force reset of btopt stats */ + XXH64_reset(&cctx->xxhState, 0); + return 0; +} + +typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset } ZSTD_compResetPolicy_e; +typedef enum { ZSTDb_not_buffered, ZSTDb_buffered } ZSTD_buffered_policy_e; + +/*! ZSTD_resetCCtx_internal() : + note : `params` are assumed fully validated at this stage */ +static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, + ZSTD_CCtx_params params, U64 pledgedSrcSize, + ZSTD_compResetPolicy_e const crp, + ZSTD_buffered_policy_e const zbuff) +{ + DEBUGLOG(4, "ZSTD_resetCCtx_internal"); + assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + DEBUGLOG(4, "pledgedSrcSize: %u", (U32)pledgedSrcSize); + + if (crp == ZSTDcrp_continue) { + if (ZSTD_equivalentParams(params, zc->appliedParams)) { + DEBUGLOG(4, "ZSTD_equivalentParams()==1"); + assert(!(params.ldmParams.enableLdm && + params.ldmParams.hashEveryLog == ZSTD_LDM_HASHEVERYLOG_NOTSET)); + zc->entropy->hufCTable_repeatMode = HUF_repeat_none; + zc->entropy->offcode_repeatMode = FSE_repeat_none; + zc->entropy->matchlength_repeatMode = FSE_repeat_none; + zc->entropy->litlength_repeatMode = FSE_repeat_none; + return ZSTD_continueCCtx(zc, params, pledgedSrcSize); + } } + + if (params.ldmParams.enableLdm) { + /* Adjust long distance matching parameters */ + ZSTD_ldm_adjustParameters(¶ms.ldmParams, params.cParams.windowLog); + assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog); + assert(params.ldmParams.hashEveryLog < 32); + zc->ldmState.hashPower = + ZSTD_ldm_getHashPower(params.ldmParams.minMatchLength); + } + + { size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.cParams.windowLog); + U32 const divider = (params.cParams.searchLength==3) ? 3 : 4; + size_t const maxNbSeq = blockSize / divider; + size_t const tokenSpace = blockSize + 11*maxNbSeq; + size_t const chainSize = (params.cParams.strategy == ZSTD_fast) ? + 0 : ((size_t)1 << params.cParams.chainLog); + size_t const hSize = ((size_t)1) << params.cParams.hashLog; + U32 const hashLog3 = (params.cParams.searchLength>3) ? + 0 : MIN(ZSTD_HASHLOG3_MAX, params.cParams.windowLog); + size_t const h3Size = ((size_t)1) << hashLog3; + size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); + size_t const buffOutSize = (zbuff==ZSTDb_buffered) ? ZSTD_compressBound(blockSize)+1 : 0; + size_t const buffInSize = (zbuff==ZSTDb_buffered) ? ((size_t)1 << params.cParams.windowLog) + blockSize : 0; + void* ptr; + + /* Check if workSpace is large enough, alloc a new one if needed */ + { size_t const entropySpace = sizeof(ZSTD_entropyCTables_t); + size_t const optPotentialSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<workSpaceSize < neededSpace) { /* too small : resize */ + DEBUGLOG(5, "Need to update workSpaceSize from %uK to %uK \n", + (unsigned)zc->workSpaceSize>>10, + (unsigned)neededSpace>>10); + /* static cctx : no resize, error out */ + if (zc->staticSize) return ERROR(memory_allocation); + + zc->workSpaceSize = 0; + ZSTD_free(zc->workSpace, zc->customMem); + zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem); + if (zc->workSpace == NULL) return ERROR(memory_allocation); + zc->workSpaceSize = neededSpace; + ptr = zc->workSpace; + + /* entropy space */ + assert(((size_t)zc->workSpace & 3) == 0); /* ensure correct alignment */ + assert(zc->workSpaceSize >= sizeof(ZSTD_entropyCTables_t)); + zc->entropy = (ZSTD_entropyCTables_t*)zc->workSpace; + } } + + /* init params */ + zc->appliedParams = params; + zc->pledgedSrcSizePlusOne = pledgedSrcSize+1; + zc->consumedSrcSize = 0; + if (pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN) + zc->appliedParams.fParams.contentSizeFlag = 0; + DEBUGLOG(5, "pledged content size : %u ; flag : %u", + (U32)pledgedSrcSize, zc->appliedParams.fParams.contentSizeFlag); + zc->blockSize = blockSize; + + XXH64_reset(&zc->xxhState, 0); + zc->stage = ZSTDcs_init; + zc->dictID = 0; + zc->loadedDictEnd = 0; + zc->entropy->hufCTable_repeatMode = HUF_repeat_none; + zc->entropy->offcode_repeatMode = FSE_repeat_none; + zc->entropy->matchlength_repeatMode = FSE_repeat_none; + zc->entropy->litlength_repeatMode = FSE_repeat_none; + zc->nextToUpdate = 1; + zc->nextSrc = NULL; + zc->base = NULL; + zc->dictBase = NULL; + zc->dictLimit = 0; + zc->lowLimit = 0; + { int i; for (i=0; iseqStore.rep[i] = repStartValue[i]; } + zc->hashLog3 = hashLog3; + zc->optState.litLengthSum = 0; + + ptr = zc->entropy + 1; + + /* opt parser space */ + if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btultra)) { + DEBUGLOG(5, "reserving optimal parser space"); + assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ + zc->optState.litFreq = (U32*)ptr; + zc->optState.litLengthFreq = zc->optState.litFreq + (1<optState.matchLengthFreq = zc->optState.litLengthFreq + (MaxLL+1); + zc->optState.offCodeFreq = zc->optState.matchLengthFreq + (MaxML+1); + ptr = zc->optState.offCodeFreq + (MaxOff+1); + zc->optState.matchTable = (ZSTD_match_t*)ptr; + ptr = zc->optState.matchTable + ZSTD_OPT_NUM+1; + zc->optState.priceTable = (ZSTD_optimal_t*)ptr; + ptr = zc->optState.priceTable + ZSTD_OPT_NUM+1; + } + + /* ldm hash table */ + /* initialize bucketOffsets table later for pointer alignment */ + if (params.ldmParams.enableLdm) { + size_t const ldmHSize = ((size_t)1) << params.ldmParams.hashLog; + memset(ptr, 0, ldmHSize * sizeof(ldmEntry_t)); + assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ + zc->ldmState.hashTable = (ldmEntry_t*)ptr; + ptr = zc->ldmState.hashTable + ldmHSize; + } + + /* table Space */ + if (crp!=ZSTDcrp_noMemset) memset(ptr, 0, tableSpace); /* reset tables only */ + assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ + zc->hashTable = (U32*)(ptr); + zc->chainTable = zc->hashTable + hSize; + zc->hashTable3 = zc->chainTable + chainSize; + ptr = zc->hashTable3 + h3Size; + + /* sequences storage */ + zc->seqStore.sequencesStart = (seqDef*)ptr; + ptr = zc->seqStore.sequencesStart + maxNbSeq; + zc->seqStore.llCode = (BYTE*) ptr; + zc->seqStore.mlCode = zc->seqStore.llCode + maxNbSeq; + zc->seqStore.ofCode = zc->seqStore.mlCode + maxNbSeq; + zc->seqStore.litStart = zc->seqStore.ofCode + maxNbSeq; + ptr = zc->seqStore.litStart + blockSize; + + /* ldm bucketOffsets table */ + if (params.ldmParams.enableLdm) { + size_t const ldmBucketSize = + ((size_t)1) << (params.ldmParams.hashLog - + params.ldmParams.bucketSizeLog); + memset(ptr, 0, ldmBucketSize); + zc->ldmState.bucketOffsets = (BYTE*)ptr; + ptr = zc->ldmState.bucketOffsets + ldmBucketSize; + } + + /* buffers */ + zc->inBuffSize = buffInSize; + zc->inBuff = (char*)ptr; + zc->outBuffSize = buffOutSize; + zc->outBuff = zc->inBuff + buffInSize; + + return 0; + } +} + +/* ZSTD_invalidateRepCodes() : + * ensures next compression will not use repcodes from previous block. + * Note : only works with regular variant; + * do not use with extDict variant ! */ +void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) { + int i; + for (i=0; iseqStore.rep[i] = 0; +} + + +/*! ZSTD_copyCCtx_internal() : + * Duplicate an existing context `srcCCtx` into another one `dstCCtx`. + * The "context", in this case, refers to the hash and chain tables, entropy + * tables, and dictionary offsets. + * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). + * pledgedSrcSize=0 means "empty" if fParams.contentSizeFlag=1 + * @return : 0, or an error code */ +static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, + const ZSTD_CCtx* srcCCtx, + ZSTD_frameParameters fParams, + unsigned long long pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) +{ + DEBUGLOG(5, "ZSTD_copyCCtx_internal"); + if (srcCCtx->stage!=ZSTDcs_init) return ERROR(stage_wrong); + + memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); + { ZSTD_CCtx_params params = dstCCtx->requestedParams; + /* Copy only compression parameters related to tables. */ + params.cParams = srcCCtx->appliedParams.cParams; + params.fParams = fParams; + ZSTD_resetCCtx_internal(dstCCtx, params, pledgedSrcSize, + ZSTDcrp_noMemset, zbuff); + } + + /* copy tables */ + { size_t const chainSize = (srcCCtx->appliedParams.cParams.strategy == ZSTD_fast) ? 0 : ((size_t)1 << srcCCtx->appliedParams.cParams.chainLog); + size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog; + size_t const h3Size = (size_t)1 << srcCCtx->hashLog3; + size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); + assert((U32*)dstCCtx->chainTable == (U32*)dstCCtx->hashTable + hSize); /* chainTable must follow hashTable */ + assert((U32*)dstCCtx->hashTable3 == (U32*)dstCCtx->chainTable + chainSize); + memcpy(dstCCtx->hashTable, srcCCtx->hashTable, tableSpace); /* presumes all tables follow each other */ + } + + /* copy dictionary offsets */ + dstCCtx->nextToUpdate = srcCCtx->nextToUpdate; + dstCCtx->nextToUpdate3= srcCCtx->nextToUpdate3; + dstCCtx->nextSrc = srcCCtx->nextSrc; + dstCCtx->base = srcCCtx->base; + dstCCtx->dictBase = srcCCtx->dictBase; + dstCCtx->dictLimit = srcCCtx->dictLimit; + dstCCtx->lowLimit = srcCCtx->lowLimit; + dstCCtx->loadedDictEnd= srcCCtx->loadedDictEnd; + dstCCtx->dictID = srcCCtx->dictID; + + /* copy entropy tables */ + memcpy(dstCCtx->entropy, srcCCtx->entropy, sizeof(ZSTD_entropyCTables_t)); + + return 0; +} + +/*! ZSTD_copyCCtx() : + * Duplicate an existing context `srcCCtx` into another one `dstCCtx`. + * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). + * pledgedSrcSize==0 means "unknown". +* @return : 0, or an error code */ +size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long long pledgedSrcSize) +{ + ZSTD_frameParameters fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; + ZSTD_buffered_policy_e const zbuff = (ZSTD_buffered_policy_e)(srcCCtx->inBuffSize>0); + ZSTD_STATIC_ASSERT((U32)ZSTDb_buffered==1); + fParams.contentSizeFlag = pledgedSrcSize>0; + + return ZSTD_copyCCtx_internal(dstCCtx, srcCCtx, fParams, pledgedSrcSize, zbuff); +} + + +/*! ZSTD_reduceTable() : + * reduce table indexes by `reducerValue` */ +static void ZSTD_reduceTable (U32* const table, U32 const size, U32 const reducerValue) +{ + U32 u; + for (u=0 ; u < size ; u++) { + if (table[u] < reducerValue) table[u] = 0; + else table[u] -= reducerValue; + } +} + +/*! ZSTD_ldm_reduceTable() : + * reduce table indexes by `reducerValue` */ +static void ZSTD_ldm_reduceTable(ldmEntry_t* const table, U32 const size, + U32 const reducerValue) +{ + U32 u; + for (u = 0; u < size; u++) { + if (table[u].offset < reducerValue) table[u].offset = 0; + else table[u].offset -= reducerValue; + } +} + +/*! ZSTD_reduceIndex() : +* rescale all indexes to avoid future overflow (indexes are U32) */ +static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue) +{ + { U32 const hSize = (U32)1 << zc->appliedParams.cParams.hashLog; + ZSTD_reduceTable(zc->hashTable, hSize, reducerValue); } + + { U32 const chainSize = (zc->appliedParams.cParams.strategy == ZSTD_fast) ? 0 : ((U32)1 << zc->appliedParams.cParams.chainLog); + ZSTD_reduceTable(zc->chainTable, chainSize, reducerValue); } + + { U32 const h3Size = (zc->hashLog3) ? (U32)1 << zc->hashLog3 : 0; + ZSTD_reduceTable(zc->hashTable3, h3Size, reducerValue); } + + { if (zc->appliedParams.ldmParams.enableLdm) { + U32 const ldmHSize = (U32)1 << zc->appliedParams.ldmParams.hashLog; + ZSTD_ldm_reduceTable(zc->ldmState.hashTable, ldmHSize, reducerValue); + } + } +} + + +/*-******************************************************* +* Block entropic compression +*********************************************************/ + +/* See doc/zstd_compression_format.md for detailed format description */ + +size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + if (srcSize + ZSTD_blockHeaderSize > dstCapacity) return ERROR(dstSize_tooSmall); + memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize); + MEM_writeLE24(dst, (U32)(srcSize << 2) + (U32)bt_raw); + return ZSTD_blockHeaderSize+srcSize; +} + + +static size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + BYTE* const ostart = (BYTE* const)dst; + U32 const flSize = 1 + (srcSize>31) + (srcSize>4095); + + if (srcSize + flSize > dstCapacity) return ERROR(dstSize_tooSmall); + + switch(flSize) + { + case 1: /* 2 - 1 - 5 */ + ostart[0] = (BYTE)((U32)set_basic + (srcSize<<3)); + break; + case 2: /* 2 - 2 - 12 */ + MEM_writeLE16(ostart, (U16)((U32)set_basic + (1<<2) + (srcSize<<4))); + break; + case 3: /* 2 - 2 - 20 */ + MEM_writeLE32(ostart, (U32)((U32)set_basic + (3<<2) + (srcSize<<4))); + break; + default: /* not necessary : flSize is {1,2,3} */ + assert(0); + } + + memcpy(ostart + flSize, src, srcSize); + return srcSize + flSize; +} + +static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + BYTE* const ostart = (BYTE* const)dst; + U32 const flSize = 1 + (srcSize>31) + (srcSize>4095); + + (void)dstCapacity; /* dstCapacity already guaranteed to be >=4, hence large enough */ + + switch(flSize) + { + case 1: /* 2 - 1 - 5 */ + ostart[0] = (BYTE)((U32)set_rle + (srcSize<<3)); + break; + case 2: /* 2 - 2 - 12 */ + MEM_writeLE16(ostart, (U16)((U32)set_rle + (1<<2) + (srcSize<<4))); + break; + case 3: /* 2 - 2 - 20 */ + MEM_writeLE32(ostart, (U32)((U32)set_rle + (3<<2) + (srcSize<<4))); + break; + default: /* not necessary : flSize is {1,2,3} */ + assert(0); + } + + ostart[flSize] = *(const BYTE*)src; + return flSize+1; +} + + +static size_t ZSTD_minGain(size_t srcSize) { return (srcSize >> 6) + 2; } + +static size_t ZSTD_compressLiterals (ZSTD_entropyCTables_t * entropy, + ZSTD_strategy strategy, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t const minGain = ZSTD_minGain(srcSize); + size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB); + BYTE* const ostart = (BYTE*)dst; + U32 singleStream = srcSize < 256; + symbolEncodingType_e hType = set_compressed; + size_t cLitSize; + + + /* small ? don't even attempt compression (speed opt) */ +# define LITERAL_NOENTROPY 63 + { size_t const minLitSize = entropy->hufCTable_repeatMode == HUF_repeat_valid ? 6 : LITERAL_NOENTROPY; + if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + } + + if (dstCapacity < lhSize+1) return ERROR(dstSize_tooSmall); /* not enough space for compression */ + { HUF_repeat repeat = entropy->hufCTable_repeatMode; + int const preferRepeat = strategy < ZSTD_lazy ? srcSize <= 1024 : 0; + if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1; + cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, + entropy->workspace, sizeof(entropy->workspace), (HUF_CElt*)entropy->hufCTable, &repeat, preferRepeat) + : HUF_compress4X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, + entropy->workspace, sizeof(entropy->workspace), (HUF_CElt*)entropy->hufCTable, &repeat, preferRepeat); + if (repeat != HUF_repeat_none) { hType = set_repeat; } /* reused the existing table */ + else { entropy->hufCTable_repeatMode = HUF_repeat_check; } /* now have a table to reuse */ + } + + if ((cLitSize==0) | (cLitSize >= srcSize - minGain) | ERR_isError(cLitSize)) { + entropy->hufCTable_repeatMode = HUF_repeat_none; + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + } + if (cLitSize==1) { + entropy->hufCTable_repeatMode = HUF_repeat_none; + return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize); + } + + /* Build header */ + switch(lhSize) + { + case 3: /* 2 - 2 - 10 - 10 */ + { U32 const lhc = hType + ((!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14); + MEM_writeLE24(ostart, lhc); + break; + } + case 4: /* 2 - 2 - 14 - 14 */ + { U32 const lhc = hType + (2 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<18); + MEM_writeLE32(ostart, lhc); + break; + } + case 5: /* 2 - 2 - 18 - 18 */ + { U32 const lhc = hType + (3 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<22); + MEM_writeLE32(ostart, lhc); + ostart[4] = (BYTE)(cLitSize >> 10); + break; + } + default: /* not possible : lhSize is {3,4,5} */ + assert(0); + } + return lhSize+cLitSize; +} + + +void ZSTD_seqToCodes(const seqStore_t* seqStorePtr) +{ + BYTE const LL_deltaCode = 19; + BYTE const ML_deltaCode = 36; + const seqDef* const sequences = seqStorePtr->sequencesStart; + BYTE* const llCodeTable = seqStorePtr->llCode; + BYTE* const ofCodeTable = seqStorePtr->ofCode; + BYTE* const mlCodeTable = seqStorePtr->mlCode; + U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + U32 u; + for (u=0; u 63) ? (BYTE)ZSTD_highbit32(llv) + LL_deltaCode : LL_Code[llv]; + ofCodeTable[u] = (BYTE)ZSTD_highbit32(sequences[u].offset); + mlCodeTable[u] = (mlv>127) ? (BYTE)ZSTD_highbit32(mlv) + ML_deltaCode : ML_Code[mlv]; + } + if (seqStorePtr->longLengthID==1) + llCodeTable[seqStorePtr->longLengthPos] = MaxLL; + if (seqStorePtr->longLengthID==2) + mlCodeTable[seqStorePtr->longLengthPos] = MaxML; +} + +typedef enum { + ZSTD_defaultDisallowed = 0, + ZSTD_defaultAllowed = 1 +} ZSTD_defaultPolicy_e; + +MEM_STATIC symbolEncodingType_e ZSTD_selectEncodingType( + FSE_repeat* repeatMode, size_t const mostFrequent, size_t nbSeq, + U32 defaultNormLog, ZSTD_defaultPolicy_e const isDefaultAllowed) +{ +#define MIN_SEQ_FOR_DYNAMIC_FSE 64 +#define MAX_SEQ_FOR_STATIC_FSE 1000 + ZSTD_STATIC_ASSERT(ZSTD_defaultDisallowed == 0 && ZSTD_defaultAllowed != 0); + if ((mostFrequent == nbSeq) && (!isDefaultAllowed || nbSeq > 2)) { + /* Prefer set_basic over set_rle when there are 2 or less symbols, + * since RLE uses 1 byte, but set_basic uses 5-6 bits per symbol. + * If basic encoding isn't possible, always choose RLE. + */ + *repeatMode = FSE_repeat_check; + return set_rle; + } + if (isDefaultAllowed && (*repeatMode == FSE_repeat_valid) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) { + return set_repeat; + } + if (isDefaultAllowed && ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (defaultNormLog-1))))) { + *repeatMode = FSE_repeat_valid; + return set_basic; + } + *repeatMode = FSE_repeat_check; + return set_compressed; +} + +MEM_STATIC size_t ZSTD_buildCTable(void* dst, size_t dstCapacity, + FSE_CTable* CTable, U32 FSELog, symbolEncodingType_e type, + U32* count, U32 max, + BYTE const* codeTable, size_t nbSeq, + S16 const* defaultNorm, U32 defaultNormLog, U32 defaultMax, + void* workspace, size_t workspaceSize) +{ + BYTE* op = (BYTE*)dst; + BYTE const* const oend = op + dstCapacity; + + switch (type) { + case set_rle: + *op = codeTable[0]; + CHECK_F(FSE_buildCTable_rle(CTable, (BYTE)max)); + return 1; + case set_repeat: + return 0; + case set_basic: + CHECK_F(FSE_buildCTable_wksp(CTable, defaultNorm, defaultMax, defaultNormLog, workspace, workspaceSize)); + return 0; + case set_compressed: { + S16 norm[MaxSeq + 1]; + size_t nbSeq_1 = nbSeq; + const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); + if (count[codeTable[nbSeq-1]] > 1) { + count[codeTable[nbSeq-1]]--; + nbSeq_1--; + } + assert(nbSeq_1 > 1); + CHECK_F(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max)); + { size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */ + if (FSE_isError(NCountSize)) return NCountSize; + CHECK_F(FSE_buildCTable_wksp(CTable, norm, max, tableLog, workspace, workspaceSize)); + return NCountSize; + } + } + default: return assert(0), ERROR(GENERIC); + } +} + +MEM_STATIC size_t ZSTD_encodeSequences(void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets) +{ + BIT_CStream_t blockStream; + FSE_CState_t stateMatchLength; + FSE_CState_t stateOffsetBits; + FSE_CState_t stateLitLength; + + CHECK_E(BIT_initCStream(&blockStream, dst, dstCapacity), dstSize_tooSmall); /* not enough space remaining */ + + /* first symbols */ + FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]); + FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq-1]); + FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq-1]); + BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]); + if (MEM_32bits()) BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[nbSeq-1].matchLength, ML_bits[mlCodeTable[nbSeq-1]]); + if (MEM_32bits()) BIT_flushBits(&blockStream); + if (longOffsets) { + U32 const ofBits = ofCodeTable[nbSeq-1]; + int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); + if (extraBits) { + BIT_addBits(&blockStream, sequences[nbSeq-1].offset, extraBits); + BIT_flushBits(&blockStream); + } + BIT_addBits(&blockStream, sequences[nbSeq-1].offset >> extraBits, + ofBits - extraBits); + } else { + BIT_addBits(&blockStream, sequences[nbSeq-1].offset, ofCodeTable[nbSeq-1]); + } + BIT_flushBits(&blockStream); + + { size_t n; + for (n=nbSeq-2 ; n= 64-7-(LLFSELog+MLFSELog+OffFSELog))) + BIT_flushBits(&blockStream); /* (7)*/ + BIT_addBits(&blockStream, sequences[n].litLength, llBits); + if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[n].matchLength, mlBits); + if (MEM_32bits() || (ofBits+mlBits+llBits > 56)) BIT_flushBits(&blockStream); + if (longOffsets) { + int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); + if (extraBits) { + BIT_addBits(&blockStream, sequences[n].offset, extraBits); + BIT_flushBits(&blockStream); /* (7)*/ + } + BIT_addBits(&blockStream, sequences[n].offset >> extraBits, + ofBits - extraBits); /* 31 */ + } else { + BIT_addBits(&blockStream, sequences[n].offset, ofBits); /* 31 */ + } + BIT_flushBits(&blockStream); /* (7)*/ + } } + + FSE_flushCState(&blockStream, &stateMatchLength); + FSE_flushCState(&blockStream, &stateOffsetBits); + FSE_flushCState(&blockStream, &stateLitLength); + + { size_t const streamSize = BIT_closeCStream(&blockStream); + if (streamSize==0) return ERROR(dstSize_tooSmall); /* not enough space */ + return streamSize; + } +} + +MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, + ZSTD_entropyCTables_t* entropy, + ZSTD_compressionParameters const* cParams, + void* dst, size_t dstCapacity) +{ + const int longOffsets = cParams->windowLog > STREAM_ACCUMULATOR_MIN; + U32 count[MaxSeq+1]; + FSE_CTable* CTable_LitLength = entropy->litlengthCTable; + FSE_CTable* CTable_OffsetBits = entropy->offcodeCTable; + FSE_CTable* CTable_MatchLength = entropy->matchlengthCTable; + U32 LLtype, Offtype, MLtype; /* compressed, raw or rle */ + const seqDef* const sequences = seqStorePtr->sequencesStart; + const BYTE* const ofCodeTable = seqStorePtr->ofCode; + const BYTE* const llCodeTable = seqStorePtr->llCode; + const BYTE* const mlCodeTable = seqStorePtr->mlCode; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart; + size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart; + BYTE* seqHead; + + ZSTD_STATIC_ASSERT(sizeof(entropy->workspace) >= (1<litStart; + size_t const litSize = seqStorePtr->lit - literals; + size_t const cSize = ZSTD_compressLiterals( + entropy, cParams->strategy, op, dstCapacity, literals, litSize); + if (ZSTD_isError(cSize)) + return cSize; + op += cSize; + } + + /* Sequences Header */ + if ((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead */) return ERROR(dstSize_tooSmall); + if (nbSeq < 0x7F) *op++ = (BYTE)nbSeq; + else if (nbSeq < LONGNBSEQ) op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2; + else op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3; + if (nbSeq==0) return op - ostart; + + /* seqHead : flags for FSE encoding type */ + seqHead = op++; + + /* convert length/distances into codes */ + ZSTD_seqToCodes(seqStorePtr); + /* CTable for Literal Lengths */ + { U32 max = MaxLL; + size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, entropy->workspace); + LLtype = ZSTD_selectEncodingType(&entropy->litlength_repeatMode, mostFrequent, nbSeq, LL_defaultNormLog, ZSTD_defaultAllowed); + { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype, + count, max, llCodeTable, nbSeq, LL_defaultNorm, LL_defaultNormLog, MaxLL, + entropy->workspace, sizeof(entropy->workspace)); + if (ZSTD_isError(countSize)) return countSize; + op += countSize; + } } + /* CTable for Offsets */ + { U32 max = MaxOff; + size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, entropy->workspace); + /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */ + ZSTD_defaultPolicy_e const defaultPolicy = max <= DefaultMaxOff ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed; + Offtype = ZSTD_selectEncodingType(&entropy->offcode_repeatMode, mostFrequent, nbSeq, OF_defaultNormLog, defaultPolicy); + { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype, + count, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, + entropy->workspace, sizeof(entropy->workspace)); + if (ZSTD_isError(countSize)) return countSize; + op += countSize; + } } + /* CTable for MatchLengths */ + { U32 max = MaxML; + size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, entropy->workspace); + MLtype = ZSTD_selectEncodingType(&entropy->matchlength_repeatMode, mostFrequent, nbSeq, ML_defaultNormLog, ZSTD_defaultAllowed); + { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype, + count, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML, + entropy->workspace, sizeof(entropy->workspace)); + if (ZSTD_isError(countSize)) return countSize; + op += countSize; + } } + + *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2)); + + { size_t const streamSize = ZSTD_encodeSequences(op, oend - op, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); + if (ZSTD_isError(streamSize)) return streamSize; + op += streamSize; + } + + return op - ostart; +} + +MEM_STATIC size_t ZSTD_compressSequences(seqStore_t* seqStorePtr, + ZSTD_entropyCTables_t* entropy, + ZSTD_compressionParameters const* cParams, + void* dst, size_t dstCapacity, + size_t srcSize) +{ + size_t const cSize = ZSTD_compressSequences_internal(seqStorePtr, entropy, cParams, + dst, dstCapacity); + size_t const minGain = ZSTD_minGain(srcSize); + size_t const maxCSize = srcSize - minGain; + /* If the srcSize <= dstCapacity, then there is enough space to write a + * raw uncompressed block. Since we ran out of space, the block must not + * be compressible, so fall back to a raw uncompressed block. + */ + int const uncompressibleError = cSize == ERROR(dstSize_tooSmall) && srcSize <= dstCapacity; + + if (ZSTD_isError(cSize) && !uncompressibleError) + return cSize; + /* Check compressibility */ + if (cSize >= maxCSize || uncompressibleError) { + entropy->hufCTable_repeatMode = HUF_repeat_none; + entropy->offcode_repeatMode = FSE_repeat_none; + entropy->matchlength_repeatMode = FSE_repeat_none; + entropy->litlength_repeatMode = FSE_repeat_none; + return 0; + } + assert(!ZSTD_isError(cSize)); + + /* confirm repcodes */ + { int i; for (i=0; irep[i] = seqStorePtr->repToConfirm[i]; } + return cSize; +} + +/* ZSTD_selectBlockCompressor() : + * Not static, but internal use only (used by long distance matcher) + * assumption : strat is a valid strategy */ +typedef size_t (*ZSTD_blockCompressor) (ZSTD_CCtx* ctx, const void* src, size_t srcSize); +ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict) +{ + static const ZSTD_blockCompressor blockCompressor[2][(unsigned)ZSTD_btultra+1] = { + { ZSTD_compressBlock_fast /* default for 0 */, + ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy, + ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2, + ZSTD_compressBlock_btopt, ZSTD_compressBlock_btultra }, + { ZSTD_compressBlock_fast_extDict /* default for 0 */, + ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict, + ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, + ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btultra_extDict } + }; + ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1); + assert((U32)strat >= (U32)ZSTD_fast); + assert((U32)strat <= (U32)ZSTD_btultra); + + return blockCompressor[extDict!=0][(U32)strat]; +} + +static void ZSTD_storeLastLiterals(seqStore_t* seqStorePtr, + const BYTE* anchor, size_t lastLLSize) +{ + memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; +} + +static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + const BYTE* const base = zc->base; + const BYTE* const istart = (const BYTE*)src; + const U32 current = (U32)(istart-base); + size_t lastLLSize; + const BYTE* anchor; + U32 const extDict = zc->lowLimit < zc->dictLimit; + const ZSTD_blockCompressor blockCompressor = + zc->appliedParams.ldmParams.enableLdm + ? (extDict ? ZSTD_compressBlock_ldm_extDict : ZSTD_compressBlock_ldm) + : ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, extDict); + + if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) return 0; /* don't even attempt compression below a certain srcSize */ + ZSTD_resetSeqStore(&(zc->seqStore)); + if (current > zc->nextToUpdate + 384) + zc->nextToUpdate = current - MIN(192, (U32)(current - zc->nextToUpdate - 384)); /* limited update after finding a very long match */ + + lastLLSize = blockCompressor(zc, src, srcSize); + + /* Last literals */ + anchor = (const BYTE*)src + srcSize - lastLLSize; + ZSTD_storeLastLiterals(&zc->seqStore, anchor, lastLLSize); + + return ZSTD_compressSequences(&zc->seqStore, zc->entropy, &zc->appliedParams.cParams, dst, dstCapacity, srcSize); +} + + +/*! ZSTD_compress_frameChunk() : +* Compress a chunk of data into one or multiple blocks. +* All blocks will be terminated, all input will be consumed. +* Function will issue an error if there is not enough `dstCapacity` to hold the compressed content. +* Frame is supposed already started (header already produced) +* @return : compressed size, or an error code +*/ +static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + U32 lastFrameChunk) +{ + size_t blockSize = cctx->blockSize; + size_t remaining = srcSize; + const BYTE* ip = (const BYTE*)src; + BYTE* const ostart = (BYTE*)dst; + BYTE* op = ostart; + U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog; + + if (cctx->appliedParams.fParams.checksumFlag && srcSize) + XXH64_update(&cctx->xxhState, src, srcSize); + + while (remaining) { + U32 const lastBlock = lastFrameChunk & (blockSize >= remaining); + size_t cSize; + + if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) + return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ + if (remaining < blockSize) blockSize = remaining; + + /* preemptive overflow correction: + * 1. correction is large enough: + * lowLimit > (3<<29) ==> current > 3<<29 + 1< (3<<29 + 1< (3<<29 - blockSize) - (1< (3<<29 - blockSize) - (1<<30) (NOTE: chainLog <= 30) + * > 1<<29 - 1<<17 + * + * 2. (ip+blockSize - cctx->base) doesn't overflow: + * In 32 bit mode we limit windowLog to 30 so we don't get + * differences larger than 1<<31-1. + * 3. cctx->lowLimit < 1<<32: + * windowLog <= 31 ==> 3<<29 + 1<lowLimit > (3U<<29)) { + U32 const cycleMask = ((U32)1 << ZSTD_cycleLog(cctx->appliedParams.cParams.chainLog, cctx->appliedParams.cParams.strategy)) - 1; + U32 const current = (U32)(ip - cctx->base); + U32 const newCurrent = (current & cycleMask) + ((U32)1 << cctx->appliedParams.cParams.windowLog); + U32 const correction = current - newCurrent; + ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30); + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30); + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); + assert(current > newCurrent); + assert(correction > 1<<28); /* Loose bound, should be about 1<<29 */ + ZSTD_reduceIndex(cctx, correction); + cctx->base += correction; + cctx->dictBase += correction; + cctx->lowLimit -= correction; + cctx->dictLimit -= correction; + if (cctx->nextToUpdate < correction) cctx->nextToUpdate = 0; + else cctx->nextToUpdate -= correction; + DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x\n", correction, cctx->lowLimit); + } + + if ((U32)(ip+blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) { + /* enforce maxDist */ + U32 const newLowLimit = (U32)(ip+blockSize - cctx->base) - maxDist; + if (cctx->lowLimit < newLowLimit) cctx->lowLimit = newLowLimit; + if (cctx->dictLimit < cctx->lowLimit) cctx->dictLimit = cctx->lowLimit; + } + + cSize = ZSTD_compressBlock_internal(cctx, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize); + if (ZSTD_isError(cSize)) return cSize; + + if (cSize == 0) { /* block is not compressible */ + U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(blockSize << 3); + if (blockSize + ZSTD_blockHeaderSize > dstCapacity) return ERROR(dstSize_tooSmall); + MEM_writeLE32(op, cBlockHeader24); /* no pb, 4th byte will be overwritten */ + memcpy(op + ZSTD_blockHeaderSize, ip, blockSize); + cSize = ZSTD_blockHeaderSize+blockSize; + } else { + U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); + MEM_writeLE24(op, cBlockHeader24); + cSize += ZSTD_blockHeaderSize; + } + + remaining -= blockSize; + dstCapacity -= cSize; + ip += blockSize; + op += cSize; + } + + if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending; + return op-ostart; +} + + +static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, + ZSTD_CCtx_params params, U64 pledgedSrcSize, U32 dictID) +{ BYTE* const op = (BYTE*)dst; + U32 const dictIDSizeCodeLength = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */ + U32 const dictIDSizeCode = params.fParams.noDictIDFlag ? 0 : dictIDSizeCodeLength; /* 0-3 */ + U32 const checksumFlag = params.fParams.checksumFlag>0; + U32 const windowSize = (U32)1 << params.cParams.windowLog; + U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize >= pledgedSrcSize); + BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); + U32 const fcsCode = params.fParams.contentSizeFlag ? + (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : 0; /* 0-3 */ + BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) ); + size_t pos=0; + + if (dstCapacity < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall); + DEBUGLOG(4, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u", + !params.fParams.noDictIDFlag, dictID, dictIDSizeCode); + + if (params.format == ZSTD_f_zstd1) { + DEBUGLOG(4, "writing zstd magic number"); + MEM_writeLE32(dst, ZSTD_MAGICNUMBER); + pos = 4; + } + op[pos++] = frameHeaderDecriptionByte; + if (!singleSegment) op[pos++] = windowLogByte; + switch(dictIDSizeCode) + { + default: assert(0); /* impossible */ + case 0 : break; + case 1 : op[pos] = (BYTE)(dictID); pos++; break; + case 2 : MEM_writeLE16(op+pos, (U16)dictID); pos+=2; break; + case 3 : MEM_writeLE32(op+pos, dictID); pos+=4; break; + } + switch(fcsCode) + { + default: assert(0); /* impossible */ + case 0 : if (singleSegment) op[pos++] = (BYTE)(pledgedSrcSize); break; + case 1 : MEM_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break; + case 2 : MEM_writeLE32(op+pos, (U32)(pledgedSrcSize)); pos+=4; break; + case 3 : MEM_writeLE64(op+pos, (U64)(pledgedSrcSize)); pos+=8; break; + } + return pos; +} + + +static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + U32 frame, U32 lastFrameChunk) +{ + const BYTE* const ip = (const BYTE*) src; + size_t fhSize = 0; + + DEBUGLOG(5, "ZSTD_compressContinue_internal"); + DEBUGLOG(5, "stage: %u", cctx->stage); + if (cctx->stage==ZSTDcs_created) return ERROR(stage_wrong); /* missing init (ZSTD_compressBegin) */ + + if (frame && (cctx->stage==ZSTDcs_init)) { + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->appliedParams, + cctx->pledgedSrcSizePlusOne-1, cctx->dictID); + if (ZSTD_isError(fhSize)) return fhSize; + dstCapacity -= fhSize; + dst = (char*)dst + fhSize; + cctx->stage = ZSTDcs_ongoing; + } + + /* Check if blocks follow each other */ + if (src != cctx->nextSrc) { + /* not contiguous */ + ptrdiff_t const delta = cctx->nextSrc - ip; + cctx->lowLimit = cctx->dictLimit; + cctx->dictLimit = (U32)(cctx->nextSrc - cctx->base); + cctx->dictBase = cctx->base; + cctx->base -= delta; + cctx->nextToUpdate = cctx->dictLimit; + if (cctx->dictLimit - cctx->lowLimit < HASH_READ_SIZE) cctx->lowLimit = cctx->dictLimit; /* too small extDict */ + } + + /* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */ + if ((ip+srcSize > cctx->dictBase + cctx->lowLimit) & (ip < cctx->dictBase + cctx->dictLimit)) { + ptrdiff_t const highInputIdx = (ip + srcSize) - cctx->dictBase; + U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)cctx->dictLimit) ? cctx->dictLimit : (U32)highInputIdx; + cctx->lowLimit = lowLimitMax; + } + + cctx->nextSrc = ip + srcSize; + + if (srcSize) { + size_t const cSize = frame ? + ZSTD_compress_frameChunk (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) : + ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize); + if (ZSTD_isError(cSize)) return cSize; + cctx->consumedSrcSize += srcSize; + return cSize + fhSize; + } else + return fhSize; +} + +size_t ZSTD_compressContinue (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 0 /* last chunk */); +} + + +size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx) +{ + ZSTD_compressionParameters const cParams = + ZSTD_getCParamsFromCCtxParams(cctx->appliedParams, 0, 0); + return MIN (ZSTD_BLOCKSIZE_MAX, (U32)1 << cParams.windowLog); +} + +size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const blockSizeMax = ZSTD_getBlockSize(cctx); + if (srcSize > blockSizeMax) return ERROR(srcSize_wrong); + return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */); +} + +/*! ZSTD_loadDictionaryContent() : + * @return : 0, or an error code + */ +static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t srcSize) +{ + const BYTE* const ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; + + /* input becomes current prefix */ + zc->lowLimit = zc->dictLimit; + zc->dictLimit = (U32)(zc->nextSrc - zc->base); + zc->dictBase = zc->base; + zc->base += ip - zc->nextSrc; + zc->nextToUpdate = zc->dictLimit; + zc->loadedDictEnd = zc->appliedParams.forceWindow ? 0 : (U32)(iend - zc->base); + + zc->nextSrc = iend; + if (srcSize <= HASH_READ_SIZE) return 0; + + switch(zc->appliedParams.cParams.strategy) + { + case ZSTD_fast: + ZSTD_fillHashTable (zc, iend, zc->appliedParams.cParams.searchLength); + break; + case ZSTD_dfast: + ZSTD_fillDoubleHashTable (zc, iend, zc->appliedParams.cParams.searchLength); + break; + + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + if (srcSize >= HASH_READ_SIZE) + ZSTD_insertAndFindFirstIndex(zc, iend-HASH_READ_SIZE, zc->appliedParams.cParams.searchLength); + break; + + case ZSTD_btlazy2: + case ZSTD_btopt: + case ZSTD_btultra: + if (srcSize >= HASH_READ_SIZE) + ZSTD_updateTree(zc, iend-HASH_READ_SIZE, iend, (U32)1 << zc->appliedParams.cParams.searchLog, zc->appliedParams.cParams.searchLength); + break; + + default: + assert(0); /* not possible : not a valid strategy id */ + } + + zc->nextToUpdate = (U32)(iend - zc->base); + return 0; +} + + +/* Dictionaries that assign zero probability to symbols that show up causes problems + when FSE encoding. Refuse dictionaries that assign zero probability to symbols + that we may encounter during compression. + NOTE: This behavior is not standard and could be improved in the future. */ +static size_t ZSTD_checkDictNCount(short* normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue) { + U32 s; + if (dictMaxSymbolValue < maxSymbolValue) return ERROR(dictionary_corrupted); + for (s = 0; s <= maxSymbolValue; ++s) { + if (normalizedCounter[s] == 0) return ERROR(dictionary_corrupted); + } + return 0; +} + + +/* Dictionary format : + * See : + * https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#dictionary-format + */ +/*! ZSTD_loadZstdDictionary() : + * @return : 0, or an error code + * assumptions : magic number supposed already checked + * dictSize supposed > 8 + */ +static size_t ZSTD_loadZstdDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize) +{ + const BYTE* dictPtr = (const BYTE*)dict; + const BYTE* const dictEnd = dictPtr + dictSize; + short offcodeNCount[MaxOff+1]; + unsigned offcodeMaxValue = MaxOff; + + ZSTD_STATIC_ASSERT(sizeof(cctx->entropy->workspace) >= (1<dictID = cctx->appliedParams.fParams.noDictIDFlag ? 0 : MEM_readLE32(dictPtr); + dictPtr += 4; + + { unsigned maxSymbolValue = 255; + size_t const hufHeaderSize = HUF_readCTable((HUF_CElt*)cctx->entropy->hufCTable, &maxSymbolValue, dictPtr, dictEnd-dictPtr); + if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted); + if (maxSymbolValue < 255) return ERROR(dictionary_corrupted); + dictPtr += hufHeaderSize; + } + + { unsigned offcodeLog; + size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); + if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); + if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted); + /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */ + CHECK_E( FSE_buildCTable_wksp(cctx->entropy->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog, cctx->entropy->workspace, sizeof(cctx->entropy->workspace)), + dictionary_corrupted); + dictPtr += offcodeHeaderSize; + } + + { short matchlengthNCount[MaxML+1]; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; + size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); + if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted); + /* Every match length code must have non-zero probability */ + CHECK_F( ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML)); + CHECK_E( FSE_buildCTable_wksp(cctx->entropy->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, cctx->entropy->workspace, sizeof(cctx->entropy->workspace)), + dictionary_corrupted); + dictPtr += matchlengthHeaderSize; + } + + { short litlengthNCount[MaxLL+1]; + unsigned litlengthMaxValue = MaxLL, litlengthLog; + size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); + if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted); + /* Every literal length code must have non-zero probability */ + CHECK_F( ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL)); + CHECK_E( FSE_buildCTable_wksp(cctx->entropy->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog, cctx->entropy->workspace, sizeof(cctx->entropy->workspace)), + dictionary_corrupted); + dictPtr += litlengthHeaderSize; + } + + if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted); + cctx->seqStore.rep[0] = MEM_readLE32(dictPtr+0); + cctx->seqStore.rep[1] = MEM_readLE32(dictPtr+4); + cctx->seqStore.rep[2] = MEM_readLE32(dictPtr+8); + dictPtr += 12; + + { size_t const dictContentSize = (size_t)(dictEnd - dictPtr); + U32 offcodeMax = MaxOff; + if (dictContentSize <= ((U32)-1) - 128 KB) { + U32 const maxOffset = (U32)dictContentSize + 128 KB; /* The maximum offset that must be supported */ + offcodeMax = ZSTD_highbit32(maxOffset); /* Calculate minimum offset code required to represent maxOffset */ + } + /* All offset values <= dictContentSize + 128 KB must be representable */ + CHECK_F (ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff))); + /* All repCodes must be <= dictContentSize and != 0*/ + { U32 u; + for (u=0; u<3; u++) { + if (cctx->seqStore.rep[u] == 0) return ERROR(dictionary_corrupted); + if (cctx->seqStore.rep[u] > dictContentSize) return ERROR(dictionary_corrupted); + } } + + cctx->entropy->hufCTable_repeatMode = HUF_repeat_valid; + cctx->entropy->offcode_repeatMode = FSE_repeat_valid; + cctx->entropy->matchlength_repeatMode = FSE_repeat_valid; + cctx->entropy->litlength_repeatMode = FSE_repeat_valid; + return ZSTD_loadDictionaryContent(cctx, dictPtr, dictContentSize); + } +} + +/** ZSTD_compress_insertDictionary() : +* @return : 0, or an error code */ +static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_dictMode_e dictMode) +{ + DEBUGLOG(5, "ZSTD_compress_insertDictionary"); + if ((dict==NULL) || (dictSize<=8)) return 0; + + /* dict restricted modes */ + if (dictMode==ZSTD_dm_rawContent) + return ZSTD_loadDictionaryContent(cctx, dict, dictSize); + + if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) { + if (dictMode == ZSTD_dm_auto) { + DEBUGLOG(5, "raw content dictionary detected"); + return ZSTD_loadDictionaryContent(cctx, dict, dictSize); + } + if (dictMode == ZSTD_dm_fullDict) + return ERROR(dictionary_wrong); + assert(0); /* impossible */ + } + + /* dict as full zstd dictionary */ + return ZSTD_loadZstdDictionary(cctx, dict, dictSize); +} + +/*! ZSTD_compressBegin_internal() : + * @return : 0, or an error code */ +static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_dictMode_e dictMode, + const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) +{ + DEBUGLOG(4, "ZSTD_compressBegin_internal"); + /* params are supposed to be fully validated at this point */ + assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + assert(!((dict) && (cdict))); /* either dict or cdict, not both */ + + if (cdict && cdict->dictContentSize>0) { + return ZSTD_copyCCtx_internal(cctx, cdict->refContext, + params.fParams, pledgedSrcSize, + zbuff); + } + + CHECK_F( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, + ZSTDcrp_continue, zbuff) ); + return ZSTD_compress_insertDictionary(cctx, dict, dictSize, dictMode); +} + +size_t ZSTD_compressBegin_advanced_internal( + ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_dictMode_e dictMode, + ZSTD_CCtx_params params, + unsigned long long pledgedSrcSize) +{ + /* compression parameters verification and optimization */ + CHECK_F( ZSTD_checkCParams(params.cParams) ); + return ZSTD_compressBegin_internal(cctx, dict, dictSize, dictMode, NULL, + params, pledgedSrcSize, + ZSTDb_not_buffered); +} + +/*! ZSTD_compressBegin_advanced() : +* @return : 0, or an error code */ +size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pledgedSrcSize) +{ + ZSTD_CCtx_params const cctxParams = + ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); + return ZSTD_compressBegin_advanced_internal(cctx, dict, dictSize, ZSTD_dm_auto, + cctxParams, + pledgedSrcSize); +} + +size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); + ZSTD_CCtx_params const cctxParams = + ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); + return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL, + cctxParams, 0, ZSTDb_not_buffered); +} + +size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel) +{ + return ZSTD_compressBegin_usingDict(cctx, NULL, 0, compressionLevel); +} + + +/*! ZSTD_writeEpilogue() : +* Ends a frame. +* @return : nb of bytes written into dst (or an error code) */ +static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) +{ + BYTE* const ostart = (BYTE*)dst; + BYTE* op = ostart; + size_t fhSize = 0; + + DEBUGLOG(5, "ZSTD_writeEpilogue"); + if (cctx->stage == ZSTDcs_created) return ERROR(stage_wrong); /* init missing */ + + /* special case : empty frame */ + if (cctx->stage == ZSTDcs_init) { + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->appliedParams, 0, 0); + if (ZSTD_isError(fhSize)) return fhSize; + dstCapacity -= fhSize; + op += fhSize; + cctx->stage = ZSTDcs_ongoing; + } + + if (cctx->stage != ZSTDcs_ending) { + /* write one last empty block, make it the "last" block */ + U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1) + 0; + if (dstCapacity<4) return ERROR(dstSize_tooSmall); + MEM_writeLE32(op, cBlockHeader24); + op += ZSTD_blockHeaderSize; + dstCapacity -= ZSTD_blockHeaderSize; + } + + if (cctx->appliedParams.fParams.checksumFlag) { + U32 const checksum = (U32) XXH64_digest(&cctx->xxhState); + if (dstCapacity<4) return ERROR(dstSize_tooSmall); + MEM_writeLE32(op, checksum); + op += 4; + } + + cctx->stage = ZSTDcs_created; /* return to "created but no init" status */ + return op-ostart; +} + + +size_t ZSTD_compressEnd (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t endResult; + size_t const cSize = ZSTD_compressContinue_internal(cctx, + dst, dstCapacity, src, srcSize, + 1 /* frame mode */, 1 /* last chunk */); + if (ZSTD_isError(cSize)) return cSize; + endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize); + if (ZSTD_isError(endResult)) return endResult; + if (cctx->appliedParams.fParams.contentSizeFlag) { /* control src size */ + DEBUGLOG(4, "end of frame : controlling src size"); + if (cctx->pledgedSrcSizePlusOne != cctx->consumedSrcSize+1) { + DEBUGLOG(4, "error : pledgedSrcSize = %u, while realSrcSize = %u", + (U32)cctx->pledgedSrcSizePlusOne-1, (U32)cctx->consumedSrcSize); + return ERROR(srcSize_wrong); + } } + return cSize + endResult; +} + + +static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params) +{ + ZSTD_CCtx_params const cctxParams = + ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); + return ZSTD_compress_advanced_internal(cctx, + dst, dstCapacity, + src, srcSize, + dict, dictSize, + cctxParams); +} + +size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params) +{ + CHECK_F(ZSTD_checkCParams(params.cParams)); + return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params); +} + +/* Internal */ +size_t ZSTD_compress_advanced_internal( + ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_CCtx_params params) +{ + CHECK_F( ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dm_auto, NULL, + params, srcSize, ZSTDb_not_buffered) ); + return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); +} + +size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, + const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, dict ? dictSize : 0); + params.fParams.contentSizeFlag = 1; + return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params); +} + +size_t ZSTD_compressCCtx (ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel) +{ + return ZSTD_compress_usingDict(ctx, dst, dstCapacity, src, srcSize, NULL, 0, compressionLevel); +} + +size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel) +{ + size_t result; + ZSTD_CCtx ctxBody; + memset(&ctxBody, 0, sizeof(ctxBody)); + ctxBody.customMem = ZSTD_defaultCMem; + result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel); + ZSTD_free(ctxBody.workSpace, ZSTD_defaultCMem); /* can't free ctxBody itself, as it's on stack; free only heap content */ + return result; +} + + +/* ===== Dictionary API ===== */ + +/*! ZSTD_estimateCDictSize_advanced() : + * Estimate amount of memory that will be needed to create a dictionary with following arguments */ +size_t ZSTD_estimateCDictSize_advanced( + size_t dictSize, ZSTD_compressionParameters cParams, + ZSTD_dictLoadMethod_e dictLoadMethod) +{ + DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (U32)sizeof(ZSTD_CDict)); + DEBUGLOG(5, "CCtx estimate : %u", + (U32)ZSTD_estimateCCtxSize_usingCParams(cParams)); + return sizeof(ZSTD_CDict) + ZSTD_estimateCCtxSize_usingCParams(cParams) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); +} + +size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel) +{ + ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); + return ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); +} + +size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) +{ + if (cdict==NULL) return 0; /* support sizeof on NULL */ + DEBUGLOG(5, "sizeof(*cdict) : %u", (U32)sizeof(*cdict)); + DEBUGLOG(5, "ZSTD_sizeof_CCtx : %u", (U32)ZSTD_sizeof_CCtx(cdict->refContext)); + return ZSTD_sizeof_CCtx(cdict->refContext) + (cdict->dictBuffer ? cdict->dictContentSize : 0) + sizeof(*cdict); +} + +static size_t ZSTD_initCDict_internal( + ZSTD_CDict* cdict, + const void* dictBuffer, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictMode_e dictMode, + ZSTD_compressionParameters cParams) +{ + DEBUGLOG(5, "ZSTD_initCDict_internal, mode %u", (U32)dictMode); + if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) { + cdict->dictBuffer = NULL; + cdict->dictContent = dictBuffer; + } else { + void* const internalBuffer = ZSTD_malloc(dictSize, cdict->refContext->customMem); + cdict->dictBuffer = internalBuffer; + cdict->dictContent = internalBuffer; + if (!internalBuffer) return ERROR(memory_allocation); + memcpy(internalBuffer, dictBuffer, dictSize); + } + cdict->dictContentSize = dictSize; + + { ZSTD_CCtx_params cctxParams = cdict->refContext->requestedParams; + cctxParams.cParams = cParams; + CHECK_F( ZSTD_compressBegin_internal(cdict->refContext, + cdict->dictContent, dictSize, dictMode, + NULL, + cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, + ZSTDb_not_buffered) ); + } + + return 0; +} + +ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictMode_e dictMode, + ZSTD_compressionParameters cParams, ZSTD_customMem customMem) +{ + DEBUGLOG(5, "ZSTD_createCDict_advanced, mode %u", (U32)dictMode); + if (!customMem.customAlloc ^ !customMem.customFree) return NULL; + + { ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem); + ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem); + + if (!cdict || !cctx) { + ZSTD_free(cdict, customMem); + ZSTD_freeCCtx(cctx); + return NULL; + } + cdict->refContext = cctx; + if (ZSTD_isError( ZSTD_initCDict_internal(cdict, + dictBuffer, dictSize, + dictLoadMethod, dictMode, + cParams) )) { + ZSTD_freeCDict(cdict); + return NULL; + } + + return cdict; + } +} + +ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); + return ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_dlm_byCopy, ZSTD_dm_auto, + cParams, ZSTD_defaultCMem); +} + +ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); + return ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_dlm_byRef, ZSTD_dm_auto, + cParams, ZSTD_defaultCMem); +} + +size_t ZSTD_freeCDict(ZSTD_CDict* cdict) +{ + if (cdict==NULL) return 0; /* support free on NULL */ + { ZSTD_customMem const cMem = cdict->refContext->customMem; + ZSTD_freeCCtx(cdict->refContext); + ZSTD_free(cdict->dictBuffer, cMem); + ZSTD_free(cdict, cMem); + return 0; + } +} + +/*! ZSTD_initStaticCDict_advanced() : + * Generate a digested dictionary in provided memory area. + * workspace: The memory area to emplace the dictionary into. + * Provided pointer must 8-bytes aligned. + * It must outlive dictionary usage. + * workspaceSize: Use ZSTD_estimateCDictSize() + * to determine how large workspace must be. + * cParams : use ZSTD_getCParams() to transform a compression level + * into its relevants cParams. + * @return : pointer to ZSTD_CDict*, or NULL if error (size too small) + * Note : there is no corresponding "free" function. + * Since workspace was allocated externally, it must be freed externally. + */ +ZSTD_CDict* ZSTD_initStaticCDict(void* workspace, size_t workspaceSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictMode_e dictMode, + ZSTD_compressionParameters cParams) +{ + size_t const cctxSize = ZSTD_estimateCCtxSize_usingCParams(cParams); + size_t const neededSize = sizeof(ZSTD_CDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize) + + cctxSize; + ZSTD_CDict* const cdict = (ZSTD_CDict*) workspace; + void* ptr; + DEBUGLOG(5, "(size_t)workspace & 7 : %u", (U32)(size_t)workspace & 7); + if ((size_t)workspace & 7) return NULL; /* 8-aligned */ + DEBUGLOG(5, "(workspaceSize < neededSize) : (%u < %u) => %u", + (U32)workspaceSize, (U32)neededSize, (U32)(workspaceSize < neededSize)); + if (workspaceSize < neededSize) return NULL; + + if (dictLoadMethod == ZSTD_dlm_byCopy) { + memcpy(cdict+1, dict, dictSize); + dict = cdict+1; + ptr = (char*)workspace + sizeof(ZSTD_CDict) + dictSize; + } else { + ptr = cdict+1; + } + cdict->refContext = ZSTD_initStaticCCtx(ptr, cctxSize); + + if (ZSTD_isError( ZSTD_initCDict_internal(cdict, + dict, dictSize, + ZSTD_dlm_byRef, dictMode, + cParams) )) + return NULL; + + return cdict; +} + +ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict) { + return cdict->refContext->appliedParams.cParams; +} + +/* ZSTD_compressBegin_usingCDict_advanced() : + * cdict must be != NULL */ +size_t ZSTD_compressBegin_usingCDict_advanced( + ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, + ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize) +{ + if (cdict==NULL) return ERROR(dictionary_wrong); + { ZSTD_CCtx_params params = cctx->requestedParams; + params.cParams = ZSTD_getCParamsFromCDict(cdict); + params.fParams = fParams; + DEBUGLOG(5, "ZSTD_compressBegin_usingCDict_advanced"); + return ZSTD_compressBegin_internal(cctx, + NULL, 0, ZSTD_dm_auto, + cdict, + params, pledgedSrcSize, + ZSTDb_not_buffered); + } +} + +/* ZSTD_compressBegin_usingCDict() : + * pledgedSrcSize=0 means "unknown" + * if pledgedSrcSize>0, it will enable contentSizeFlag */ +size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) +{ + ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; + DEBUGLOG(5, "ZSTD_compressBegin_usingCDict : dictIDFlag == %u", !fParams.noDictIDFlag); + return ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, 0); +} + +size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, ZSTD_frameParameters fParams) +{ + CHECK_F (ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, srcSize)); /* will check if cdict != NULL */ + return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); +} + +/*! ZSTD_compress_usingCDict() : + * Compression using a digested Dictionary. + * Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. + * Note that compression parameters are decided at CDict creation time + * while frame parameters are hardcoded */ +size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict) +{ + ZSTD_frameParameters const fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; + return ZSTD_compress_usingCDict_advanced(cctx, dst, dstCapacity, src, srcSize, cdict, fParams); +} + + + +/* ****************************************************************** +* Streaming +********************************************************************/ + +ZSTD_CStream* ZSTD_createCStream(void) +{ + return ZSTD_createCStream_advanced(ZSTD_defaultCMem); +} + +ZSTD_CStream* ZSTD_initStaticCStream(void *workspace, size_t workspaceSize) +{ + return ZSTD_initStaticCCtx(workspace, workspaceSize); +} + +ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem) +{ /* CStream and CCtx are now same object */ + return ZSTD_createCCtx_advanced(customMem); +} + +size_t ZSTD_freeCStream(ZSTD_CStream* zcs) +{ + return ZSTD_freeCCtx(zcs); /* same object */ +} + + + +/*====== Initialization ======*/ + +size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX; } + +size_t ZSTD_CStreamOutSize(void) +{ + return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; +} + +static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, ZSTD_dictMode_e dictMode, + const ZSTD_CDict* cdict, + const ZSTD_CCtx_params params, unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_resetCStream_internal"); + /* params are supposed to be fully validated at this point */ + assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + assert(!((dict) && (cdict))); /* either dict or cdict, not both */ + + CHECK_F( ZSTD_compressBegin_internal(zcs, + dict, dictSize, dictMode, + cdict, + params, pledgedSrcSize, + ZSTDb_buffered) ); + + zcs->inToCompress = 0; + zcs->inBuffPos = 0; + zcs->inBuffTarget = zcs->blockSize; + zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; + zcs->streamStage = zcss_load; + zcs->frameEnded = 0; + return 0; /* ready to go */ +} + +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) +{ + ZSTD_CCtx_params params = zcs->requestedParams; + params.fParams.contentSizeFlag = (pledgedSrcSize > 0); + params.cParams = ZSTD_getCParamsFromCCtxParams(params, pledgedSrcSize, 0); + DEBUGLOG(4, "ZSTD_resetCStream"); + return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, zcs->cdict, params, pledgedSrcSize); +} + +/*! ZSTD_initCStream_internal() : + * Note : not static, but hidden (not exposed). Used by zstdmt_compress.c + * Assumption 1 : params are valid + * Assumption 2 : either dict, or cdict, is defined, not both */ +size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_initCStream_internal"); + assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + assert(!((dict) && (cdict))); /* either dict or cdict, not both */ + + if (dict && dictSize >= 8) { + DEBUGLOG(5, "loading dictionary of size %u", (U32)dictSize); + if (zcs->staticSize) { /* static CCtx : never uses malloc */ + /* incompatible with internal cdict creation */ + return ERROR(memory_allocation); + } + ZSTD_freeCDict(zcs->cdictLocal); + zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_dlm_byCopy, ZSTD_dm_auto, + params.cParams, zcs->customMem); + zcs->cdict = zcs->cdictLocal; + if (zcs->cdictLocal == NULL) return ERROR(memory_allocation); + } else { + if (cdict) { + params.cParams = ZSTD_getCParamsFromCDict(cdict); /* cParams are enforced from cdict */ + } + ZSTD_freeCDict(zcs->cdictLocal); + zcs->cdictLocal = NULL; + zcs->cdict = cdict; + } + + params.compressionLevel = ZSTD_CLEVEL_CUSTOM; + zcs->requestedParams = params; + + return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, zcs->cdict, params, pledgedSrcSize); +} + +/* ZSTD_initCStream_usingCDict_advanced() : + * same as ZSTD_initCStream_usingCDict(), with control over frame parameters */ +size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams, + unsigned long long pledgedSrcSize) +{ /* cannot handle NULL cdict (does not know what to do) */ + if (!cdict) return ERROR(dictionary_wrong); + { ZSTD_CCtx_params params = zcs->requestedParams; + params.cParams = ZSTD_getCParamsFromCDict(cdict); + params.fParams = fParams; + return ZSTD_initCStream_internal(zcs, + NULL, 0, cdict, + params, pledgedSrcSize); + } +} + +/* note : cdict must outlive compression session */ +size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict) +{ + ZSTD_frameParameters const fParams = { 0 /* contentSize */, 0 /* checksum */, 0 /* hideDictID */ }; + return ZSTD_initCStream_usingCDict_advanced(zcs, cdict, fParams, 0); /* note : will check that cdict != NULL */ +} + +size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pledgedSrcSize) +{ + ZSTD_CCtx_params const cctxParams = + ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params); + CHECK_F( ZSTD_checkCParams(params.cParams) ); + return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, cctxParams, pledgedSrcSize); +} + +size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); + ZSTD_CCtx_params const cctxParams = + ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params); + return ZSTD_initCStream_internal(zcs, dict, dictSize, NULL, cctxParams, 0); +} + +size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize) +{ + ZSTD_CCtx_params cctxParams; + ZSTD_parameters const params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0); + cctxParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params); + cctxParams.fParams.contentSizeFlag = (pledgedSrcSize>0); + return ZSTD_initCStream_internal(zcs, NULL, 0, NULL, cctxParams, pledgedSrcSize); +} + +size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) +{ + return ZSTD_initCStream_srcSize(zcs, compressionLevel, 0); +} + +/*====== Compression ======*/ + +MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t const length = MIN(dstCapacity, srcSize); + if (length) memcpy(dst, src, length); + return length; +} + +/** ZSTD_compressStream_generic(): + * internal function for all *compressStream*() variants and *compress_generic() + * non-static, because can be called from zstdmt.c + * @return : hint size for next input */ +size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective const flushMode) +{ + const char* const istart = (const char*)input->src; + const char* const iend = istart + input->size; + const char* ip = istart + input->pos; + char* const ostart = (char*)output->dst; + char* const oend = ostart + output->size; + char* op = ostart + output->pos; + U32 someMoreWork = 1; + + /* check expectations */ + DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%u", (U32)flushMode); + assert(zcs->inBuff != NULL); + assert(zcs->inBuffSize>0); + assert(zcs->outBuff!= NULL); + assert(zcs->outBuffSize>0); + assert(output->pos <= output->size); + assert(input->pos <= input->size); + + while (someMoreWork) { + switch(zcs->streamStage) + { + case zcss_init: + /* call ZSTD_initCStream() first ! */ + return ERROR(init_missing); + + case zcss_load: + if ( (flushMode == ZSTD_e_end) + && ((size_t)(oend-op) >= ZSTD_compressBound(iend-ip)) /* enough dstCapacity */ + && (zcs->inBuffPos == 0) ) { + /* shortcut to compression pass directly into output buffer */ + size_t const cSize = ZSTD_compressEnd(zcs, + op, oend-op, ip, iend-ip); + DEBUGLOG(4, "ZSTD_compressEnd : %u", (U32)cSize); + if (ZSTD_isError(cSize)) return cSize; + ip = iend; + op += cSize; + zcs->frameEnded = 1; + ZSTD_startNewCompression(zcs); + someMoreWork = 0; break; + } + /* complete loading into inBuffer */ + { size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos; + size_t const loaded = ZSTD_limitCopy( + zcs->inBuff + zcs->inBuffPos, toLoad, + ip, iend-ip); + zcs->inBuffPos += loaded; + ip += loaded; + if ( (flushMode == ZSTD_e_continue) + && (zcs->inBuffPos < zcs->inBuffTarget) ) { + /* not enough input to fill full block : stop here */ + someMoreWork = 0; break; + } + if ( (flushMode == ZSTD_e_flush) + && (zcs->inBuffPos == zcs->inToCompress) ) { + /* empty */ + someMoreWork = 0; break; + } + } + /* compress current block (note : this stage cannot be stopped in the middle) */ + DEBUGLOG(5, "stream compression stage (flushMode==%u)", flushMode); + { void* cDst; + size_t cSize; + size_t const iSize = zcs->inBuffPos - zcs->inToCompress; + size_t oSize = oend-op; + unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip==iend); + if (oSize >= ZSTD_compressBound(iSize)) + cDst = op; /* compress into output buffer, to skip flush stage */ + else + cDst = zcs->outBuff, oSize = zcs->outBuffSize; + cSize = lastBlock ? + ZSTD_compressEnd(zcs, cDst, oSize, + zcs->inBuff + zcs->inToCompress, iSize) : + ZSTD_compressContinue(zcs, cDst, oSize, + zcs->inBuff + zcs->inToCompress, iSize); + if (ZSTD_isError(cSize)) return cSize; + zcs->frameEnded = lastBlock; + /* prepare next block */ + zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize; + if (zcs->inBuffTarget > zcs->inBuffSize) + zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; + DEBUGLOG(5, "inBuffTarget:%u / inBuffSize:%u", + (U32)zcs->inBuffTarget, (U32)zcs->inBuffSize); + if (!lastBlock) + assert(zcs->inBuffTarget <= zcs->inBuffSize); + zcs->inToCompress = zcs->inBuffPos; + if (cDst == op) { /* no need to flush */ + op += cSize; + if (zcs->frameEnded) { + DEBUGLOG(5, "Frame completed directly in outBuffer"); + someMoreWork = 0; + ZSTD_startNewCompression(zcs); + } + break; + } + zcs->outBuffContentSize = cSize; + zcs->outBuffFlushedSize = 0; + zcs->streamStage = zcss_flush; /* pass-through to flush stage */ + } + /* fall-through */ + case zcss_flush: + DEBUGLOG(5, "flush stage"); + { size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; + size_t const flushed = ZSTD_limitCopy(op, oend-op, + zcs->outBuff + zcs->outBuffFlushedSize, toFlush); + DEBUGLOG(5, "toFlush: %u into %u ==> flushed: %u", + (U32)toFlush, (U32)(oend-op), (U32)flushed); + op += flushed; + zcs->outBuffFlushedSize += flushed; + if (toFlush!=flushed) { + /* flush not fully completed, presumably because dst is too small */ + assert(op==oend); + someMoreWork = 0; + break; + } + zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; + if (zcs->frameEnded) { + DEBUGLOG(5, "Frame completed on flush"); + someMoreWork = 0; + ZSTD_startNewCompression(zcs); + break; + } + zcs->streamStage = zcss_load; + break; + } + + default: /* impossible */ + assert(0); + } + } + + input->pos = ip - istart; + output->pos = op - ostart; + if (zcs->frameEnded) return 0; + { size_t hintInSize = zcs->inBuffTarget - zcs->inBuffPos; + if (hintInSize==0) hintInSize = zcs->blockSize; + return hintInSize; + } +} + +size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input) +{ + /* check conditions */ + if (output->pos > output->size) return ERROR(GENERIC); + if (input->pos > input->size) return ERROR(GENERIC); + + return ZSTD_compressStream_generic(zcs, output, input, ZSTD_e_continue); +} + + +size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp) +{ + DEBUGLOG(5, "ZSTD_compress_generic"); + /* check conditions */ + if (output->pos > output->size) return ERROR(GENERIC); + if (input->pos > input->size) return ERROR(GENERIC); + assert(cctx!=NULL); + + /* transparent initialization stage */ + if (cctx->streamStage == zcss_init) { + ZSTD_prefixDict const prefixDict = cctx->prefixDict; + ZSTD_CCtx_params params = cctx->requestedParams; + params.cParams = ZSTD_getCParamsFromCCtxParams( + cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, 0 /*dictSize*/); + memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */ + assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */ + DEBUGLOG(4, "ZSTD_compress_generic : transparent init stage"); + +#ifdef ZSTD_MULTITHREAD + if (params.nbThreads > 1) { + if (cctx->mtctx == NULL || cctx->appliedParams.nbThreads != params.nbThreads) { + ZSTDMT_freeCCtx(cctx->mtctx); + cctx->mtctx = ZSTDMT_createCCtx_advanced(params.nbThreads, cctx->customMem); + if (cctx->mtctx == NULL) return ERROR(memory_allocation); + } + DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbThreads=%u", params.nbThreads); + CHECK_F( ZSTDMT_initCStream_internal( + cctx->mtctx, + prefixDict.dict, prefixDict.dictSize, ZSTD_dm_rawContent, + cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) ); + cctx->streamStage = zcss_load; + cctx->appliedParams.nbThreads = params.nbThreads; + } else +#endif + { + CHECK_F( ZSTD_resetCStream_internal( + cctx, prefixDict.dict, prefixDict.dictSize, + prefixDict.dictMode, cctx->cdict, params, + cctx->pledgedSrcSizePlusOne-1) ); + } } + + /* compression stage */ +#ifdef ZSTD_MULTITHREAD + if (cctx->appliedParams.nbThreads > 1) { + size_t const flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp); + DEBUGLOG(5, "ZSTDMT_compressStream_generic result : %u", (U32)flushMin); + if ( ZSTD_isError(flushMin) + || (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */ + ZSTD_startNewCompression(cctx); + } + return flushMin; + } +#endif + CHECK_F( ZSTD_compressStream_generic(cctx, output, input, endOp) ); + DEBUGLOG(5, "completed ZSTD_compress_generic"); + return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */ +} + +size_t ZSTD_compress_generic_simpleArgs ( + ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos, + ZSTD_EndDirective endOp) +{ + ZSTD_outBuffer output = { dst, dstCapacity, *dstPos }; + ZSTD_inBuffer input = { src, srcSize, *srcPos }; + /* ZSTD_compress_generic() will check validity of dstPos and srcPos */ + size_t const cErr = ZSTD_compress_generic(cctx, &output, &input, endOp); + *dstPos = output.pos; + *srcPos = input.pos; + return cErr; +} + + +/*====== Finalize ======*/ + +/*! ZSTD_flushStream() : +* @return : amount of data remaining to flush */ +size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) +{ + ZSTD_inBuffer input = { NULL, 0, 0 }; + if (output->pos > output->size) return ERROR(GENERIC); + CHECK_F( ZSTD_compressStream_generic(zcs, output, &input, ZSTD_e_flush) ); + return zcs->outBuffContentSize - zcs->outBuffFlushedSize; /* remaining to flush */ +} + + +size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) +{ + ZSTD_inBuffer input = { NULL, 0, 0 }; + if (output->pos > output->size) return ERROR(GENERIC); + CHECK_F( ZSTD_compressStream_generic(zcs, output, &input, ZSTD_e_end) ); + { size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE; + size_t const checksumSize = zcs->frameEnded ? 0 : zcs->appliedParams.fParams.checksumFlag * 4; + size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize + lastBlockSize + checksumSize; + DEBUGLOG(5, "ZSTD_endStream : remaining to flush : %u", + (unsigned)toFlush); + return toFlush; + } +} + + +/*-===== Pre-defined compression levels =====-*/ + +#define ZSTD_MAX_CLEVEL 22 +int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } + +static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = { +{ /* "default" - guarantees a monotonically increasing memory budget */ + /* W, C, H, S, L, TL, strat */ + { 18, 12, 12, 1, 7, 16, ZSTD_fast }, /* level 0 - never used */ + { 19, 13, 14, 1, 7, 16, ZSTD_fast }, /* level 1 */ + { 19, 15, 16, 1, 6, 16, ZSTD_fast }, /* level 2 */ + { 20, 16, 17, 1, 5, 16, ZSTD_dfast }, /* level 3 */ + { 20, 17, 18, 1, 5, 16, ZSTD_dfast }, /* level 4 */ + { 20, 17, 18, 2, 5, 16, ZSTD_greedy }, /* level 5 */ + { 21, 17, 19, 2, 5, 16, ZSTD_lazy }, /* level 6 */ + { 21, 18, 19, 3, 5, 16, ZSTD_lazy }, /* level 7 */ + { 21, 18, 20, 3, 5, 16, ZSTD_lazy2 }, /* level 8 */ + { 21, 19, 20, 3, 5, 16, ZSTD_lazy2 }, /* level 9 */ + { 21, 19, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 10 */ + { 22, 20, 22, 4, 5, 16, ZSTD_lazy2 }, /* level 11 */ + { 22, 20, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 12 */ + { 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 13 */ + { 22, 21, 22, 6, 5, 16, ZSTD_lazy2 }, /* level 14 */ + { 22, 21, 22, 5, 5, 16, ZSTD_btlazy2 }, /* level 15 */ + { 23, 22, 22, 5, 5, 16, ZSTD_btlazy2 }, /* level 16 */ + { 23, 22, 22, 4, 5, 24, ZSTD_btopt }, /* level 17 */ + { 23, 22, 22, 5, 4, 32, ZSTD_btopt }, /* level 18 */ + { 23, 23, 22, 6, 3, 48, ZSTD_btopt }, /* level 19 */ + { 25, 25, 23, 7, 3, 64, ZSTD_btultra }, /* level 20 */ + { 26, 26, 24, 7, 3,256, ZSTD_btultra }, /* level 21 */ + { 27, 27, 25, 9, 3,512, ZSTD_btultra }, /* level 22 */ +}, +{ /* for srcSize <= 256 KB */ + /* W, C, H, S, L, T, strat */ + { 0, 0, 0, 0, 0, 0, ZSTD_fast }, /* level 0 - not used */ + { 18, 13, 14, 1, 6, 8, ZSTD_fast }, /* level 1 */ + { 18, 14, 13, 1, 5, 8, ZSTD_dfast }, /* level 2 */ + { 18, 16, 15, 1, 5, 8, ZSTD_dfast }, /* level 3 */ + { 18, 15, 17, 1, 5, 8, ZSTD_greedy }, /* level 4.*/ + { 18, 16, 17, 4, 5, 8, ZSTD_greedy }, /* level 5.*/ + { 18, 16, 17, 3, 5, 8, ZSTD_lazy }, /* level 6.*/ + { 18, 17, 17, 4, 4, 8, ZSTD_lazy }, /* level 7 */ + { 18, 17, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ + { 18, 17, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ + { 18, 17, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ + { 18, 18, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 11.*/ + { 18, 18, 17, 7, 4, 8, ZSTD_lazy2 }, /* level 12.*/ + { 18, 19, 17, 6, 4, 8, ZSTD_btlazy2 }, /* level 13 */ + { 18, 18, 18, 4, 4, 16, ZSTD_btopt }, /* level 14.*/ + { 18, 18, 18, 4, 3, 16, ZSTD_btopt }, /* level 15.*/ + { 18, 19, 18, 6, 3, 32, ZSTD_btopt }, /* level 16.*/ + { 18, 19, 18, 8, 3, 64, ZSTD_btopt }, /* level 17.*/ + { 18, 19, 18, 9, 3,128, ZSTD_btopt }, /* level 18.*/ + { 18, 19, 18, 10, 3,256, ZSTD_btopt }, /* level 19.*/ + { 18, 19, 18, 11, 3,512, ZSTD_btultra }, /* level 20.*/ + { 18, 19, 18, 12, 3,512, ZSTD_btultra }, /* level 21.*/ + { 18, 19, 18, 13, 3,512, ZSTD_btultra }, /* level 22.*/ +}, +{ /* for srcSize <= 128 KB */ + /* W, C, H, S, L, T, strat */ + { 17, 12, 12, 1, 7, 8, ZSTD_fast }, /* level 0 - not used */ + { 17, 12, 13, 1, 6, 8, ZSTD_fast }, /* level 1 */ + { 17, 13, 16, 1, 5, 8, ZSTD_fast }, /* level 2 */ + { 17, 16, 16, 2, 5, 8, ZSTD_dfast }, /* level 3 */ + { 17, 13, 15, 3, 4, 8, ZSTD_greedy }, /* level 4 */ + { 17, 15, 17, 4, 4, 8, ZSTD_greedy }, /* level 5 */ + { 17, 16, 17, 3, 4, 8, ZSTD_lazy }, /* level 6 */ + { 17, 15, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 7 */ + { 17, 17, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ + { 17, 17, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ + { 17, 17, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ + { 17, 17, 17, 7, 4, 8, ZSTD_lazy2 }, /* level 11 */ + { 17, 17, 17, 8, 4, 8, ZSTD_lazy2 }, /* level 12 */ + { 17, 18, 17, 6, 4, 8, ZSTD_btlazy2 }, /* level 13.*/ + { 17, 17, 17, 7, 3, 8, ZSTD_btopt }, /* level 14.*/ + { 17, 17, 17, 7, 3, 16, ZSTD_btopt }, /* level 15.*/ + { 17, 18, 17, 7, 3, 32, ZSTD_btopt }, /* level 16.*/ + { 17, 18, 17, 7, 3, 64, ZSTD_btopt }, /* level 17.*/ + { 17, 18, 17, 7, 3,256, ZSTD_btopt }, /* level 18.*/ + { 17, 18, 17, 8, 3,256, ZSTD_btopt }, /* level 19.*/ + { 17, 18, 17, 9, 3,256, ZSTD_btultra }, /* level 20.*/ + { 17, 18, 17, 10, 3,256, ZSTD_btultra }, /* level 21.*/ + { 17, 18, 17, 11, 3,512, ZSTD_btultra }, /* level 22.*/ +}, +{ /* for srcSize <= 16 KB */ + /* W, C, H, S, L, T, strat */ + { 14, 12, 12, 1, 7, 6, ZSTD_fast }, /* level 0 - not used */ + { 14, 14, 14, 1, 6, 6, ZSTD_fast }, /* level 1 */ + { 14, 14, 14, 1, 4, 6, ZSTD_fast }, /* level 2 */ + { 14, 14, 14, 1, 4, 6, ZSTD_dfast }, /* level 3.*/ + { 14, 14, 14, 4, 4, 6, ZSTD_greedy }, /* level 4.*/ + { 14, 14, 14, 3, 4, 6, ZSTD_lazy }, /* level 5.*/ + { 14, 14, 14, 4, 4, 6, ZSTD_lazy2 }, /* level 6 */ + { 14, 14, 14, 5, 4, 6, ZSTD_lazy2 }, /* level 7 */ + { 14, 14, 14, 6, 4, 6, ZSTD_lazy2 }, /* level 8.*/ + { 14, 15, 14, 6, 4, 6, ZSTD_btlazy2 }, /* level 9.*/ + { 14, 15, 14, 3, 3, 6, ZSTD_btopt }, /* level 10.*/ + { 14, 15, 14, 6, 3, 8, ZSTD_btopt }, /* level 11.*/ + { 14, 15, 14, 6, 3, 16, ZSTD_btopt }, /* level 12.*/ + { 14, 15, 14, 6, 3, 24, ZSTD_btopt }, /* level 13.*/ + { 14, 15, 15, 6, 3, 48, ZSTD_btopt }, /* level 14.*/ + { 14, 15, 15, 6, 3, 64, ZSTD_btopt }, /* level 15.*/ + { 14, 15, 15, 6, 3, 96, ZSTD_btopt }, /* level 16.*/ + { 14, 15, 15, 6, 3,128, ZSTD_btopt }, /* level 17.*/ + { 14, 15, 15, 6, 3,256, ZSTD_btopt }, /* level 18.*/ + { 14, 15, 15, 7, 3,256, ZSTD_btopt }, /* level 19.*/ + { 14, 15, 15, 8, 3,256, ZSTD_btultra }, /* level 20.*/ + { 14, 15, 15, 9, 3,256, ZSTD_btultra }, /* level 21.*/ + { 14, 15, 15, 10, 3,256, ZSTD_btultra }, /* level 22.*/ +}, +}; + +#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1) +/* This function just controls + * the monotonic memory budget increase of ZSTD_defaultCParameters[0]. + * Run once, on first ZSTD_getCParams() usage, if ZSTD_DEBUG is enabled + */ +MEM_STATIC void ZSTD_check_compressionLevel_monotonicIncrease_memoryBudget(void) +{ + int level; + for (level=1; level=1) + static int g_monotonicTest = 1; + if (g_monotonicTest) { + ZSTD_check_compressionLevel_monotonicIncrease_memoryBudget(); + g_monotonicTest=0; + } +#endif + + if (compressionLevel <= 0) compressionLevel = ZSTD_CLEVEL_DEFAULT; /* 0 == default; no negative compressionLevel yet */ + if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL; + { ZSTD_compressionParameters const cp = ZSTD_defaultCParameters[tableID][compressionLevel]; + return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize); } + +} + +/*! ZSTD_getParams() : +* same as ZSTD_getCParams(), but @return a `ZSTD_parameters` object (instead of `ZSTD_compressionParameters`). +* All fields of `ZSTD_frameParameters` are set to default (0) */ +ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) { + ZSTD_parameters params; + ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSizeHint, dictSize); + memset(¶ms, 0, sizeof(params)); + params.cParams = cParams; + return params; +} diff --git a/src/borg/algorithms/zstd/lib/compress/zstd_compress.h b/src/borg/algorithms/zstd/lib/compress/zstd_compress.h new file mode 100644 index 0000000000..94606edc93 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/compress/zstd_compress.h @@ -0,0 +1,307 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +#ifndef ZSTD_COMPRESS_H +#define ZSTD_COMPRESS_H + +/*-************************************* +* Dependencies +***************************************/ +#include "zstd_internal.h" +#ifdef ZSTD_MULTITHREAD +# include "zstdmt_compress.h" +#endif + +#if defined (__cplusplus) +extern "C" { +#endif + +/*-************************************* +* Constants +***************************************/ +static const U32 g_searchStrength = 8; +#define HASH_READ_SIZE 8 + + +/*-************************************* +* Context memory management +***************************************/ +typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e; +typedef enum { zcss_init=0, zcss_load, zcss_flush } ZSTD_cStreamStage; + +typedef struct ZSTD_prefixDict_s { + const void* dict; + size_t dictSize; + ZSTD_dictMode_e dictMode; +} ZSTD_prefixDict; + +struct ZSTD_CCtx_s { + const BYTE* nextSrc; /* next block here to continue on current prefix */ + const BYTE* base; /* All regular indexes relative to this position */ + const BYTE* dictBase; /* extDict indexes relative to this position */ + U32 dictLimit; /* below that point, need extDict */ + U32 lowLimit; /* below that point, no more data */ + U32 nextToUpdate; /* index from which to continue dictionary update */ + U32 nextToUpdate3; /* index from which to continue dictionary update */ + U32 hashLog3; /* dispatch table : larger == faster, more memory */ + U32 loadedDictEnd; /* index of end of dictionary */ + ZSTD_compressionStage_e stage; + U32 dictID; + ZSTD_CCtx_params requestedParams; + ZSTD_CCtx_params appliedParams; + void* workSpace; + size_t workSpaceSize; + size_t blockSize; + U64 pledgedSrcSizePlusOne; /* this way, 0 (default) == unknown */ + U64 consumedSrcSize; + XXH64_state_t xxhState; + ZSTD_customMem customMem; + size_t staticSize; + + seqStore_t seqStore; /* sequences storage ptrs */ + optState_t optState; + ldmState_t ldmState; /* long distance matching state */ + U32* hashTable; + U32* hashTable3; + U32* chainTable; + ZSTD_entropyCTables_t* entropy; + + /* streaming */ + char* inBuff; + size_t inBuffSize; + size_t inToCompress; + size_t inBuffPos; + size_t inBuffTarget; + char* outBuff; + size_t outBuffSize; + size_t outBuffContentSize; + size_t outBuffFlushedSize; + ZSTD_cStreamStage streamStage; + U32 frameEnded; + + /* Dictionary */ + ZSTD_CDict* cdictLocal; + const ZSTD_CDict* cdict; + ZSTD_prefixDict prefixDict; /* single-usage dictionary */ + + /* Multi-threading */ +#ifdef ZSTD_MULTITHREAD + ZSTDMT_CCtx* mtctx; +#endif +}; + + +static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 16, 17, 17, 18, 18, 19, 19, + 20, 20, 20, 20, 21, 21, 21, 21, + 22, 22, 22, 22, 22, 22, 22, 22, + 23, 23, 23, 23, 23, 23, 23, 23, + 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24 }; + +static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, + 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; + +/*! ZSTD_storeSeq() : + Store a sequence (literal length, literals, offset code and match length code) into seqStore_t. + `offsetCode` : distance to match, or 0 == repCode. + `matchCode` : matchLength - MINMATCH +*/ +MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t matchCode) +{ +#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG >= 6) + static const BYTE* g_start = NULL; + U32 const pos = (U32)((const BYTE*)literals - g_start); + if (g_start==NULL) g_start = (const BYTE*)literals; + if ((pos > 0) && (pos < 1000000000)) + DEBUGLOG(6, "Cpos %6u :%5u literals & match %3u bytes at distance %6u", + pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode); +#endif + /* copy Literals */ + assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + 128 KB); + ZSTD_wildcopy(seqStorePtr->lit, literals, litLength); + seqStorePtr->lit += litLength; + + /* literal Length */ + if (litLength>0xFFFF) { + seqStorePtr->longLengthID = 1; + seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + } + seqStorePtr->sequences[0].litLength = (U16)litLength; + + /* match offset */ + seqStorePtr->sequences[0].offset = offsetCode + 1; + + /* match Length */ + if (matchCode>0xFFFF) { + seqStorePtr->longLengthID = 2; + seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + } + seqStorePtr->sequences[0].matchLength = (U16)matchCode; + + seqStorePtr->sequences++; +} + + +/*-************************************* +* Match length counter +***************************************/ +static unsigned ZSTD_NbCommonBytes (register size_t val) +{ + if (MEM_isLittleEndian()) { + if (MEM_64bits()) { +# if defined(_MSC_VER) && defined(_WIN64) + unsigned long r = 0; + _BitScanForward64( &r, (U64)val ); + return (unsigned)(r>>3); +# elif defined(__GNUC__) && (__GNUC__ >= 4) + return (__builtin_ctzll((U64)val) >> 3); +# else + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, + 0, 3, 1, 3, 1, 4, 2, 7, + 0, 2, 3, 6, 1, 5, 3, 5, + 1, 3, 4, 4, 2, 5, 6, 7, + 7, 0, 1, 2, 3, 3, 4, 6, + 2, 6, 5, 5, 3, 4, 5, 6, + 7, 1, 2, 4, 6, 4, 4, 5, + 7, 2, 6, 5, 7, 6, 7, 7 }; + return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; +# endif + } else { /* 32 bits */ +# if defined(_MSC_VER) + unsigned long r=0; + _BitScanForward( &r, (U32)val ); + return (unsigned)(r>>3); +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_ctz((U32)val) >> 3); +# else + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, + 3, 2, 2, 1, 3, 2, 0, 1, + 3, 3, 1, 2, 2, 2, 2, 0, + 3, 1, 2, 0, 1, 0, 1, 1 }; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; +# endif + } + } else { /* Big Endian CPU */ + if (MEM_64bits()) { +# if defined(_MSC_VER) && defined(_WIN64) + unsigned long r = 0; + _BitScanReverse64( &r, val ); + return (unsigned)(r>>3); +# elif defined(__GNUC__) && (__GNUC__ >= 4) + return (__builtin_clzll(val) >> 3); +# else + unsigned r; + const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */ + if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; +# endif + } else { /* 32 bits */ +# if defined(_MSC_VER) + unsigned long r = 0; + _BitScanReverse( &r, (unsigned long)val ); + return (unsigned)(r>>3); +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_clz((U32)val) >> 3); +# else + unsigned r; + if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } + r += (!val); + return r; +# endif + } } +} + + +MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* const pInLimit) +{ + const BYTE* const pStart = pIn; + const BYTE* const pInLoopLimit = pInLimit - (sizeof(size_t)-1); + + while (pIn < pInLoopLimit) { + size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn); + if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; } + pIn += ZSTD_NbCommonBytes(diff); + return (size_t)(pIn - pStart); + } + if (MEM_64bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; } + if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; } + if ((pIn> (32-h) ; } +MEM_STATIC size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h); } /* only in zstd_opt.h */ + +static const U32 prime4bytes = 2654435761U; +static U32 ZSTD_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32-h) ; } +static size_t ZSTD_hash4Ptr(const void* ptr, U32 h) { return ZSTD_hash4(MEM_read32(ptr), h); } + +static const U64 prime5bytes = 889523592379ULL; +static size_t ZSTD_hash5(U64 u, U32 h) { return (size_t)(((u << (64-40)) * prime5bytes) >> (64-h)) ; } +static size_t ZSTD_hash5Ptr(const void* p, U32 h) { return ZSTD_hash5(MEM_readLE64(p), h); } + +static const U64 prime6bytes = 227718039650203ULL; +static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; } +static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); } + +static const U64 prime7bytes = 58295818150454627ULL; +static size_t ZSTD_hash7(U64 u, U32 h) { return (size_t)(((u << (64-56)) * prime7bytes) >> (64-h)) ; } +static size_t ZSTD_hash7Ptr(const void* p, U32 h) { return ZSTD_hash7(MEM_readLE64(p), h); } + +static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL; +static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; } +static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); } + +MEM_STATIC size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls) +{ + switch(mls) + { + default: + case 4: return ZSTD_hash4Ptr(p, hBits); + case 5: return ZSTD_hash5Ptr(p, hBits); + case 6: return ZSTD_hash6Ptr(p, hBits); + case 7: return ZSTD_hash7Ptr(p, hBits); + case 8: return ZSTD_hash8Ptr(p, hBits); + } +} + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_COMPRESS_H */ diff --git a/src/borg/algorithms/zstd/lib/compress/zstd_double_fast.c b/src/borg/algorithms/zstd/lib/compress/zstd_double_fast.c new file mode 100644 index 0000000000..876a36042c --- /dev/null +++ b/src/borg/algorithms/zstd/lib/compress/zstd_double_fast.c @@ -0,0 +1,308 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_double_fast.h" + + +void ZSTD_fillDoubleHashTable(ZSTD_CCtx* cctx, const void* end, const U32 mls) +{ + U32* const hashLarge = cctx->hashTable; + U32 const hBitsL = cctx->appliedParams.cParams.hashLog; + U32* const hashSmall = cctx->chainTable; + U32 const hBitsS = cctx->appliedParams.cParams.chainLog; + const BYTE* const base = cctx->base; + const BYTE* ip = base + cctx->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; + const size_t fastHashFillStep = 3; + + while(ip <= iend) { + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip - base); + hashLarge[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip - base); + ip += fastHashFillStep; + } +} + + +FORCE_INLINE_TEMPLATE +size_t ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx, + const void* src, size_t srcSize, + const U32 mls) +{ + U32* const hashLong = cctx->hashTable; + const U32 hBitsL = cctx->appliedParams.cParams.hashLog; + U32* const hashSmall = cctx->chainTable; + const U32 hBitsS = cctx->appliedParams.cParams.chainLog; + seqStore_t* seqStorePtr = &(cctx->seqStore); + const BYTE* const base = cctx->base; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const U32 lowestIndex = cctx->dictLimit; + const BYTE* const lowest = base + lowestIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - HASH_READ_SIZE; + U32 offset_1=seqStorePtr->rep[0], offset_2=seqStorePtr->rep[1]; + U32 offsetSaved = 0; + + /* init */ + ip += (ip==lowest); + { U32 const maxRep = (U32)(ip-lowest); + if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0; + if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0; + } + + /* Main Search Loop */ + while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ + size_t mLength; + size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8); + size_t const h = ZSTD_hashPtr(ip, hBitsS, mls); + U32 const current = (U32)(ip-base); + U32 const matchIndexL = hashLong[h2]; + U32 const matchIndexS = hashSmall[h]; + const BYTE* matchLong = base + matchIndexL; + const BYTE* match = base + matchIndexS; + hashLong[h2] = hashSmall[h] = current; /* update hash tables */ + + assert(offset_1 <= current); /* supposed guaranteed by construction */ + if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { + /* favor repcode */ + mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; + ip++; + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH); + } else { + U32 offset; + if ( (matchIndexL > lowestIndex) && (MEM_read64(matchLong) == MEM_read64(ip)) ) { + mLength = ZSTD_count(ip+8, matchLong+8, iend) + 8; + offset = (U32)(ip-matchLong); + while (((ip>anchor) & (matchLong>lowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ + } else if ( (matchIndexS > lowestIndex) && (MEM_read32(match) == MEM_read32(ip)) ) { + size_t const hl3 = ZSTD_hashPtr(ip+1, hBitsL, 8); + U32 const matchIndexL3 = hashLong[hl3]; + const BYTE* matchL3 = base + matchIndexL3; + hashLong[hl3] = current + 1; + if ( (matchIndexL3 > lowestIndex) && (MEM_read64(matchL3) == MEM_read64(ip+1)) ) { + mLength = ZSTD_count(ip+9, matchL3+8, iend) + 8; + ip++; + offset = (U32)(ip-matchL3); + while (((ip>anchor) & (matchL3>lowest)) && (ip[-1] == matchL3[-1])) { ip--; matchL3--; mLength++; } /* catch up */ + } else { + mLength = ZSTD_count(ip+4, match+4, iend) + 4; + offset = (U32)(ip-match); + while (((ip>anchor) & (match>lowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + } + } else { + ip += ((ip-anchor) >> g_searchStrength) + 1; + continue; + } + + offset_2 = offset_1; + offset_1 = offset; + + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + } + + /* match found */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Fill Table */ + hashLong[ZSTD_hashPtr(base+current+2, hBitsL, 8)] = + hashSmall[ZSTD_hashPtr(base+current+2, hBitsS, mls)] = current+2; /* here because current+2 could be > iend-8 */ + hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = + hashSmall[ZSTD_hashPtr(ip-2, hBitsS, mls)] = (U32)(ip-2-base); + + /* check immediate repcode */ + while ( (ip <= ilimit) + && ( (offset_2>0) + & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) { + /* store sequence */ + size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; + { U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */ + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base); + hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base); + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength-MINMATCH); + ip += rLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } } } + + /* save reps for next block */ + seqStorePtr->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved; + seqStorePtr->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved; + + /* Return the last literals size */ + return iend - anchor; +} + + +size_t ZSTD_compressBlock_doubleFast(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + const U32 mls = ctx->appliedParams.cParams.searchLength; + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 4); + case 5 : + return ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 5); + case 6 : + return ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 6); + case 7 : + return ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 7); + } +} + + +static size_t ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx, + const void* src, size_t srcSize, + const U32 mls) +{ + U32* const hashLong = ctx->hashTable; + U32 const hBitsL = ctx->appliedParams.cParams.hashLog; + U32* const hashSmall = ctx->chainTable; + U32 const hBitsS = ctx->appliedParams.cParams.chainLog; + seqStore_t* seqStorePtr = &(ctx->seqStore); + const BYTE* const base = ctx->base; + const BYTE* const dictBase = ctx->dictBase; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const U32 lowestIndex = ctx->lowLimit; + const BYTE* const dictStart = dictBase + lowestIndex; + const U32 dictLimit = ctx->dictLimit; + const BYTE* const lowPrefixPtr = base + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + U32 offset_1=seqStorePtr->rep[0], offset_2=seqStorePtr->rep[1]; + + /* Search Loop */ + while (ip < ilimit) { /* < instead of <=, because (ip+1) */ + const size_t hSmall = ZSTD_hashPtr(ip, hBitsS, mls); + const U32 matchIndex = hashSmall[hSmall]; + const BYTE* matchBase = matchIndex < dictLimit ? dictBase : base; + const BYTE* match = matchBase + matchIndex; + + const size_t hLong = ZSTD_hashPtr(ip, hBitsL, 8); + const U32 matchLongIndex = hashLong[hLong]; + const BYTE* matchLongBase = matchLongIndex < dictLimit ? dictBase : base; + const BYTE* matchLong = matchLongBase + matchLongIndex; + + const U32 current = (U32)(ip-base); + const U32 repIndex = current + 1 - offset_1; /* offset_1 expected <= current +1 */ + const BYTE* repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* repMatch = repBase + repIndex; + size_t mLength; + hashSmall[hSmall] = hashLong[hLong] = current; /* update hash table */ + + if ( (((U32)((dictLimit-1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex)) + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* repMatchEnd = repIndex < dictLimit ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, lowPrefixPtr) + 4; + ip++; + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH); + } else { + if ((matchLongIndex > lowestIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) { + const BYTE* matchEnd = matchLongIndex < dictLimit ? dictEnd : iend; + const BYTE* lowMatchPtr = matchLongIndex < dictLimit ? dictStart : lowPrefixPtr; + U32 offset; + mLength = ZSTD_count_2segments(ip+8, matchLong+8, iend, matchEnd, lowPrefixPtr) + 8; + offset = current - matchLongIndex; + while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + + } else if ((matchIndex > lowestIndex) && (MEM_read32(match) == MEM_read32(ip))) { + size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); + U32 const matchIndex3 = hashLong[h3]; + const BYTE* const match3Base = matchIndex3 < dictLimit ? dictBase : base; + const BYTE* match3 = match3Base + matchIndex3; + U32 offset; + hashLong[h3] = current + 1; + if ( (matchIndex3 > lowestIndex) && (MEM_read64(match3) == MEM_read64(ip+1)) ) { + const BYTE* matchEnd = matchIndex3 < dictLimit ? dictEnd : iend; + const BYTE* lowMatchPtr = matchIndex3 < dictLimit ? dictStart : lowPrefixPtr; + mLength = ZSTD_count_2segments(ip+9, match3+8, iend, matchEnd, lowPrefixPtr) + 8; + ip++; + offset = current+1 - matchIndex3; + while (((ip>anchor) & (match3>lowMatchPtr)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */ + } else { + const BYTE* matchEnd = matchIndex < dictLimit ? dictEnd : iend; + const BYTE* lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr; + mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, lowPrefixPtr) + 4; + offset = current - matchIndex; + while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + } + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + + } else { + ip += ((ip-anchor) >> g_searchStrength) + 1; + continue; + } } + + /* found a match : store it */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Fill Table */ + hashSmall[ZSTD_hashPtr(base+current+2, hBitsS, mls)] = current+2; + hashLong[ZSTD_hashPtr(base+current+2, hBitsL, 8)] = current+2; + hashSmall[ZSTD_hashPtr(ip-2, hBitsS, mls)] = (U32)(ip-2-base); + hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); + /* check immediate repcode */ + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* repMatch2 = repIndex2 < dictLimit ? dictBase + repIndex2 : base + repIndex2; + if ( (((U32)((dictLimit-1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */ + && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, lowPrefixPtr) + 4; + U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2-MINMATCH); + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; + hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; + ip += repLength2; + anchor = ip; + continue; + } + break; + } } } + + /* save reps for next block */ + seqStorePtr->repToConfirm[0] = offset_1; seqStorePtr->repToConfirm[1] = offset_2; + + /* Return the last literals size */ + return iend - anchor; +} + + +size_t ZSTD_compressBlock_doubleFast_extDict(ZSTD_CCtx* ctx, + const void* src, size_t srcSize) +{ + U32 const mls = ctx->appliedParams.cParams.searchLength; + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 4); + case 5 : + return ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 5); + case 6 : + return ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 6); + case 7 : + return ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 7); + } +} diff --git a/src/borg/algorithms/zstd/lib/compress/zstd_double_fast.h b/src/borg/algorithms/zstd/lib/compress/zstd_double_fast.h new file mode 100644 index 0000000000..3dba6c7108 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/compress/zstd_double_fast.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_DOUBLE_FAST_H +#define ZSTD_DOUBLE_FAST_H + +#include "zstd_compress.h" + +#if defined (__cplusplus) +extern "C" { +#endif + +void ZSTD_fillDoubleHashTable(ZSTD_CCtx* cctx, const void* end, const U32 mls); +size_t ZSTD_compressBlock_doubleFast(ZSTD_CCtx* ctx, const void* src, size_t srcSize); +size_t ZSTD_compressBlock_doubleFast_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize); + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_DOUBLE_FAST_H */ diff --git a/src/borg/algorithms/zstd/lib/compress/zstd_fast.c b/src/borg/algorithms/zstd/lib/compress/zstd_fast.c new file mode 100644 index 0000000000..2e057017b9 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/compress/zstd_fast.c @@ -0,0 +1,242 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_fast.h" + + +void ZSTD_fillHashTable (ZSTD_CCtx* zc, const void* end, const U32 mls) +{ + U32* const hashTable = zc->hashTable; + U32 const hBits = zc->appliedParams.cParams.hashLog; + const BYTE* const base = zc->base; + const BYTE* ip = base + zc->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; + const size_t fastHashFillStep = 3; + + while(ip <= iend) { + hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip - base); + ip += fastHashFillStep; + } +} + + +FORCE_INLINE_TEMPLATE +size_t ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx, + const void* src, size_t srcSize, + const U32 mls) +{ + U32* const hashTable = cctx->hashTable; + U32 const hBits = cctx->appliedParams.cParams.hashLog; + seqStore_t* seqStorePtr = &(cctx->seqStore); + const BYTE* const base = cctx->base; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const U32 lowestIndex = cctx->dictLimit; + const BYTE* const lowest = base + lowestIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - HASH_READ_SIZE; + U32 offset_1=seqStorePtr->rep[0], offset_2=seqStorePtr->rep[1]; + U32 offsetSaved = 0; + + /* init */ + ip += (ip==lowest); + { U32 const maxRep = (U32)(ip-lowest); + if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0; + if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0; + } + + /* Main Search Loop */ + while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ + size_t mLength; + size_t const h = ZSTD_hashPtr(ip, hBits, mls); + U32 const current = (U32)(ip-base); + U32 const matchIndex = hashTable[h]; + const BYTE* match = base + matchIndex; + hashTable[h] = current; /* update hash table */ + + if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { + mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; + ip++; + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH); + } else { + U32 offset; + if ( (matchIndex <= lowestIndex) || (MEM_read32(match) != MEM_read32(ip)) ) { + ip += ((ip-anchor) >> g_searchStrength) + 1; + continue; + } + mLength = ZSTD_count(ip+4, match+4, iend) + 4; + offset = (U32)(ip-match); + while (((ip>anchor) & (match>lowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + offset_2 = offset_1; + offset_1 = offset; + + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + } + + /* match found */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Fill Table */ + hashTable[ZSTD_hashPtr(base+current+2, hBits, mls)] = current+2; /* here because current+2 could be > iend-8 */ + hashTable[ZSTD_hashPtr(ip-2, hBits, mls)] = (U32)(ip-2-base); + /* check immediate repcode */ + while ( (ip <= ilimit) + && ( (offset_2>0) + & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) { + /* store sequence */ + size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; + { U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */ + hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip-base); + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength-MINMATCH); + ip += rLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } } } + + /* save reps for next block */ + seqStorePtr->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved; + seqStorePtr->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved; + + /* Return the last literals size */ + return iend - anchor; +} + + +size_t ZSTD_compressBlock_fast(ZSTD_CCtx* ctx, + const void* src, size_t srcSize) +{ + const U32 mls = ctx->appliedParams.cParams.searchLength; + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 4); + case 5 : + return ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 5); + case 6 : + return ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 6); + case 7 : + return ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 7); + } +} + + +static size_t ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx, + const void* src, size_t srcSize, + const U32 mls) +{ + U32* hashTable = ctx->hashTable; + const U32 hBits = ctx->appliedParams.cParams.hashLog; + seqStore_t* seqStorePtr = &(ctx->seqStore); + const BYTE* const base = ctx->base; + const BYTE* const dictBase = ctx->dictBase; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const U32 lowestIndex = ctx->lowLimit; + const BYTE* const dictStart = dictBase + lowestIndex; + const U32 dictLimit = ctx->dictLimit; + const BYTE* const lowPrefixPtr = base + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + U32 offset_1=seqStorePtr->rep[0], offset_2=seqStorePtr->rep[1]; + + /* Search Loop */ + while (ip < ilimit) { /* < instead of <=, because (ip+1) */ + const size_t h = ZSTD_hashPtr(ip, hBits, mls); + const U32 matchIndex = hashTable[h]; + const BYTE* matchBase = matchIndex < dictLimit ? dictBase : base; + const BYTE* match = matchBase + matchIndex; + const U32 current = (U32)(ip-base); + const U32 repIndex = current + 1 - offset_1; /* offset_1 expected <= current +1 */ + const BYTE* repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* repMatch = repBase + repIndex; + size_t mLength; + hashTable[h] = current; /* update hash table */ + + if ( (((U32)((dictLimit-1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex)) + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* repMatchEnd = repIndex < dictLimit ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, lowPrefixPtr) + 4; + ip++; + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH); + } else { + if ( (matchIndex < lowestIndex) || + (MEM_read32(match) != MEM_read32(ip)) ) { + ip += ((ip-anchor) >> g_searchStrength) + 1; + continue; + } + { const BYTE* matchEnd = matchIndex < dictLimit ? dictEnd : iend; + const BYTE* lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr; + U32 offset; + mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, lowPrefixPtr) + 4; + while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + offset = current - matchIndex; + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + } } + + /* found a match : store it */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Fill Table */ + hashTable[ZSTD_hashPtr(base+current+2, hBits, mls)] = current+2; + hashTable[ZSTD_hashPtr(ip-2, hBits, mls)] = (U32)(ip-2-base); + /* check immediate repcode */ + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* repMatch2 = repIndex2 < dictLimit ? dictBase + repIndex2 : base + repIndex2; + if ( (((U32)((dictLimit-1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */ + && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, lowPrefixPtr) + 4; + U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2-MINMATCH); + hashTable[ZSTD_hashPtr(ip, hBits, mls)] = current2; + ip += repLength2; + anchor = ip; + continue; + } + break; + } } } + + /* save reps for next block */ + seqStorePtr->repToConfirm[0] = offset_1; seqStorePtr->repToConfirm[1] = offset_2; + + /* Return the last literals size */ + return iend - anchor; +} + + +size_t ZSTD_compressBlock_fast_extDict(ZSTD_CCtx* ctx, + const void* src, size_t srcSize) +{ + U32 const mls = ctx->appliedParams.cParams.searchLength; + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 4); + case 5 : + return ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 5); + case 6 : + return ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 6); + case 7 : + return ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 7); + } +} diff --git a/src/borg/algorithms/zstd/lib/compress/zstd_fast.h b/src/borg/algorithms/zstd/lib/compress/zstd_fast.h new file mode 100644 index 0000000000..4205141a9a --- /dev/null +++ b/src/borg/algorithms/zstd/lib/compress/zstd_fast.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_FAST_H +#define ZSTD_FAST_H + +#include "zstd_compress.h" + +#if defined (__cplusplus) +extern "C" { +#endif + +void ZSTD_fillHashTable(ZSTD_CCtx* zc, const void* end, const U32 mls); +size_t ZSTD_compressBlock_fast(ZSTD_CCtx* ctx, + const void* src, size_t srcSize); +size_t ZSTD_compressBlock_fast_extDict(ZSTD_CCtx* ctx, + const void* src, size_t srcSize); + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_FAST_H */ diff --git a/src/borg/algorithms/zstd/lib/compress/zstd_lazy.c b/src/borg/algorithms/zstd/lib/compress/zstd_lazy.c new file mode 100644 index 0000000000..2a7f6a0fe2 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/compress/zstd_lazy.c @@ -0,0 +1,749 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_lazy.h" + + +/*-************************************* +* Binary Tree search +***************************************/ +/** ZSTD_insertBt1() : add one or multiple positions to tree. +* ip : assumed <= iend-8 . +* @return : nb of positions added */ +static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, const BYTE* const iend, U32 nbCompares, + U32 extDict) +{ + U32* const hashTable = zc->hashTable; + U32 const hashLog = zc->appliedParams.cParams.hashLog; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32* const bt = zc->chainTable; + U32 const btLog = zc->appliedParams.cParams.chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + U32 matchIndex = hashTable[h]; + size_t commonLengthSmaller=0, commonLengthLarger=0; + const BYTE* const base = zc->base; + const BYTE* const dictBase = zc->dictBase; + const U32 dictLimit = zc->dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* match; + const U32 current = (U32)(ip-base); + const U32 btLow = btMask >= current ? 0 : current - btMask; + U32* smallerPtr = bt + 2*(current&btMask); + U32* largerPtr = smallerPtr + 1; + U32 dummy32; /* to be nullified at the end */ + U32 const windowLow = zc->lowLimit; + U32 matchEndIdx = current+8; + size_t bestLength = 8; +#ifdef ZSTD_C_PREDICT + U32 predictedSmall = *(bt + 2*((current-1)&btMask) + 0); + U32 predictedLarge = *(bt + 2*((current-1)&btMask) + 1); + predictedSmall += (predictedSmall>0); + predictedLarge += (predictedLarge>0); +#endif /* ZSTD_C_PREDICT */ + + assert(ip <= iend-8); /* required for h calculation */ + hashTable[h] = current; /* Update Hash Table */ + + while (nbCompares-- && (matchIndex > windowLow)) { + U32* const nextPtr = bt + 2*(matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + +#ifdef ZSTD_C_PREDICT /* note : can create issues when hlog small <= 11 */ + const U32* predictPtr = bt + 2*((matchIndex-1) & btMask); /* written this way, as bt is a roll buffer */ + if (matchIndex == predictedSmall) { + /* no need to check length, result known */ + *smallerPtr = matchIndex; + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ + matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + predictedSmall = predictPtr[1] + (predictPtr[1]>0); + continue; + } + if (matchIndex == predictedLarge) { + *largerPtr = matchIndex; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + predictedLarge = predictPtr[0] + (predictPtr[0]>0); + continue; + } +#endif + if ((!extDict) || (matchIndex+matchLength >= dictLimit)) { + match = base + matchIndex; + if (match[matchLength] == ip[matchLength]) + matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iend) +1; + } else { + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ + } + + if (matchLength > bestLength) { + bestLength = matchLength; + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + } + + if (ip+matchLength == iend) /* equal : no way to know if inf or sup */ + break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt tree */ + + if (match[matchLength] < ip[matchLength]) { /* necessarily within buffer */ + /* match+1 is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop searching */ + smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ + matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop searching */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; + if (bestLength > 384) return MIN(192, (U32)(bestLength - 384)); /* speed optimization */ + if (matchEndIdx > current + 8) return matchEndIdx - (current + 8); + return 1; +} + + +static size_t ZSTD_insertBtAndFindBestMatch ( + ZSTD_CCtx* zc, + const BYTE* const ip, const BYTE* const iend, + size_t* offsetPtr, + U32 nbCompares, const U32 mls, + U32 extDict) +{ + U32* const hashTable = zc->hashTable; + U32 const hashLog = zc->appliedParams.cParams.hashLog; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32* const bt = zc->chainTable; + U32 const btLog = zc->appliedParams.cParams.chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + U32 matchIndex = hashTable[h]; + size_t commonLengthSmaller=0, commonLengthLarger=0; + const BYTE* const base = zc->base; + const BYTE* const dictBase = zc->dictBase; + const U32 dictLimit = zc->dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const U32 current = (U32)(ip-base); + const U32 btLow = btMask >= current ? 0 : current - btMask; + const U32 windowLow = zc->lowLimit; + U32* smallerPtr = bt + 2*(current&btMask); + U32* largerPtr = bt + 2*(current&btMask) + 1; + U32 matchEndIdx = current+8; + U32 dummy32; /* to be nullified at the end */ + size_t bestLength = 0; + + assert(ip <= iend-8); /* required for h calculation */ + hashTable[h] = current; /* Update Hash Table */ + + while (nbCompares-- && (matchIndex > windowLow)) { + U32* const nextPtr = bt + 2*(matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const BYTE* match; + + if ((!extDict) || (matchIndex+matchLength >= dictLimit)) { + match = base + matchIndex; + if (match[matchLength] == ip[matchLength]) + matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iend) +1; + } else { + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ + } + + if (matchLength > bestLength) { + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(current-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) + bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + current - matchIndex; + if (ip+matchLength == iend) /* equal : no way to know if inf or sup */ + break; /* drop, to guarantee consistency (miss a little bit of compression) */ + } + + if (match[matchLength] < ip[matchLength]) { + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ + matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; + + zc->nextToUpdate = (matchEndIdx > current + 8) ? matchEndIdx - 8 : current+1; + return bestLength; +} + + +void ZSTD_updateTree(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls) +{ + const BYTE* const base = zc->base; + const U32 target = (U32)(ip - base); + U32 idx = zc->nextToUpdate; + + while(idx < target) + idx += ZSTD_insertBt1(zc, base+idx, mls, iend, nbCompares, 0); +} + +/** ZSTD_BtFindBestMatch() : Tree updater, providing best match */ +static size_t ZSTD_BtFindBestMatch ( + ZSTD_CCtx* zc, + const BYTE* const ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 maxNbAttempts, const U32 mls) +{ + if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ + ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls); + return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 0); +} + + +static size_t ZSTD_BtFindBestMatch_selectMLS ( + ZSTD_CCtx* zc, /* Index table will be updated */ + const BYTE* ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 maxNbAttempts, const U32 matchLengthSearch) +{ + switch(matchLengthSearch) + { + default : /* includes case 3 */ + case 4 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4); + case 5 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5); + case 7 : + case 6 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6); + } +} + + +void ZSTD_updateTree_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls) +{ + const BYTE* const base = zc->base; + const U32 target = (U32)(ip - base); + U32 idx = zc->nextToUpdate; + + while (idx < target) idx += ZSTD_insertBt1(zc, base+idx, mls, iend, nbCompares, 1); +} + + +/** Tree updater, providing best match */ +static size_t ZSTD_BtFindBestMatch_extDict ( + ZSTD_CCtx* zc, + const BYTE* const ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 maxNbAttempts, const U32 mls) +{ + if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ + ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls); + return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 1); +} + + +static size_t ZSTD_BtFindBestMatch_selectMLS_extDict ( + ZSTD_CCtx* zc, /* Index table will be updated */ + const BYTE* ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 maxNbAttempts, const U32 matchLengthSearch) +{ + switch(matchLengthSearch) + { + default : /* includes case 3 */ + case 4 : return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4); + case 5 : return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5); + case 7 : + case 6 : return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6); + } +} + + + +/* ********************************* +* Hash Chain +***********************************/ +#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & mask] + +/* Update chains up to ip (excluded) + Assumption : always within prefix (i.e. not within extDict) */ +U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls) +{ + U32* const hashTable = zc->hashTable; + const U32 hashLog = zc->appliedParams.cParams.hashLog; + U32* const chainTable = zc->chainTable; + const U32 chainMask = (1 << zc->appliedParams.cParams.chainLog) - 1; + const BYTE* const base = zc->base; + const U32 target = (U32)(ip - base); + U32 idx = zc->nextToUpdate; + + while(idx < target) { /* catch up */ + size_t const h = ZSTD_hashPtr(base+idx, hashLog, mls); + NEXT_IN_CHAIN(idx, chainMask) = hashTable[h]; + hashTable[h] = idx; + idx++; + } + + zc->nextToUpdate = target; + return hashTable[ZSTD_hashPtr(ip, hashLog, mls)]; +} + + +/* inlining is important to hardwire a hot branch (template emulation) */ +FORCE_INLINE_TEMPLATE +size_t ZSTD_HcFindBestMatch_generic ( + ZSTD_CCtx* zc, /* Index table will be updated */ + const BYTE* const ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 maxNbAttempts, const U32 mls, const U32 extDict) +{ + U32* const chainTable = zc->chainTable; + const U32 chainSize = (1 << zc->appliedParams.cParams.chainLog); + const U32 chainMask = chainSize-1; + const BYTE* const base = zc->base; + const BYTE* const dictBase = zc->dictBase; + const U32 dictLimit = zc->dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const U32 lowLimit = zc->lowLimit; + const U32 current = (U32)(ip-base); + const U32 minChain = current > chainSize ? current - chainSize : 0; + int nbAttempts=maxNbAttempts; + size_t ml=4-1; + + /* HC4 match finder */ + U32 matchIndex = ZSTD_insertAndFindFirstIndex (zc, ip, mls); + + for ( ; (matchIndex>lowLimit) & (nbAttempts>0) ; nbAttempts--) { + const BYTE* match; + size_t currentMl=0; + if ((!extDict) || matchIndex >= dictLimit) { + match = base + matchIndex; + if (match[ml] == ip[ml]) /* potentially better */ + currentMl = ZSTD_count(ip, match, iLimit); + } else { + match = dictBase + matchIndex; + if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dictEnd, prefixStart) + 4; + } + + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = current - matchIndex + ZSTD_REP_MOVE; + if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ + } + + if (matchIndex <= minChain) break; + matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask); + } + + return ml; +} + + +FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_selectMLS ( + ZSTD_CCtx* zc, + const BYTE* ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 maxNbAttempts, const U32 matchLengthSearch) +{ + switch(matchLengthSearch) + { + default : /* includes case 3 */ + case 4 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 0); + case 5 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 0); + case 7 : + case 6 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 0); + } +} + + +FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS ( + ZSTD_CCtx* zc, + const BYTE* ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 maxNbAttempts, const U32 matchLengthSearch) +{ + switch(matchLengthSearch) + { + default : /* includes case 3 */ + case 4 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 1); + case 5 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 1); + case 7 : + case 6 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 1); + } +} + + +/* ******************************* +* Common parser - lazy strategy +*********************************/ +FORCE_INLINE_TEMPLATE +size_t ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, + const void* src, size_t srcSize, + const U32 searchMethod, const U32 depth) +{ + seqStore_t* seqStorePtr = &(ctx->seqStore); + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ctx->base + ctx->dictLimit; + + U32 const maxSearches = 1 << ctx->appliedParams.cParams.searchLog; + U32 const mls = ctx->appliedParams.cParams.searchLength; + + typedef size_t (*searchMax_f)(ZSTD_CCtx* zc, const BYTE* ip, const BYTE* iLimit, + size_t* offsetPtr, + U32 maxNbAttempts, U32 matchLengthSearch); + searchMax_f const searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS; + U32 offset_1 = seqStorePtr->rep[0], offset_2 = seqStorePtr->rep[1], savedOffset=0; + + /* init */ + ip += (ip==base); + ctx->nextToUpdate3 = ctx->nextToUpdate; + { U32 const maxRep = (U32)(ip-base); + if (offset_2 > maxRep) savedOffset = offset_2, offset_2 = 0; + if (offset_1 > maxRep) savedOffset = offset_1, offset_1 = 0; + } + + /* Match Loop */ + while (ip < ilimit) { + size_t matchLength=0; + size_t offset=0; + const BYTE* start=ip+1; + + /* check repCode */ + if ((offset_1>0) & (MEM_read32(ip+1) == MEM_read32(ip+1 - offset_1))) { + /* repcode : we take it */ + matchLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; + if (depth==0) goto _storeSequence; + } + + /* first search (depth 0) */ + { size_t offsetFound = 99999999; + size_t const ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls); + if (ml2 > matchLength) + matchLength = ml2, start = ip, offset=offsetFound; + } + + if (matchLength < 4) { + ip += ((ip-anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */ + continue; + } + + /* let's try to find a better solution */ + if (depth>=1) + while (ip0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { + size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4; + int const gain2 = (int)(mlRep * 3); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offset = 0, start = ip; + } + { size_t offset2=99999999; + size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offset = offset2, start = ip; + continue; /* search a better one */ + } } + + /* let's find an even better one */ + if ((depth==2) && (ip0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { + size_t const ml2 = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4; + int const gain2 = (int)(ml2 * 4); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1); + if ((ml2 >= 4) && (gain2 > gain1)) + matchLength = ml2, offset = 0, start = ip; + } + { size_t offset2=99999999; + size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offset = offset2, start = ip; + continue; + } } } + break; /* nothing found : store previous solution */ + } + + /* NOTE: + * start[-offset+ZSTD_REP_MOVE-1] is undefined behavior. + * (-offset+ZSTD_REP_MOVE-1) is unsigned, and is added to start, which + * overflows the pointer, which is undefined behavior. + */ + /* catch up */ + if (offset) { + while ( (start > anchor) + && (start > base+offset-ZSTD_REP_MOVE) + && (start[-1] == (start-offset+ZSTD_REP_MOVE)[-1]) ) /* only search for offset within prefix */ + { start--; matchLength++; } + offset_2 = offset_1; offset_1 = (U32)(offset - ZSTD_REP_MOVE); + } + /* store sequence */ +_storeSequence: + { size_t const litLength = start - anchor; + ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength-MINMATCH); + anchor = ip = start + matchLength; + } + + /* check immediate repcode */ + while ( (ip <= ilimit) + && ((offset_2>0) + & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) { + /* store sequence */ + matchLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; + offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap repcodes */ + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength-MINMATCH); + ip += matchLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } } + + /* Save reps for next block */ + seqStorePtr->repToConfirm[0] = offset_1 ? offset_1 : savedOffset; + seqStorePtr->repToConfirm[1] = offset_2 ? offset_2 : savedOffset; + + /* Return the last literals size */ + return iend - anchor; +} + + +size_t ZSTD_compressBlock_btlazy2(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 1, 2); +} + +size_t ZSTD_compressBlock_lazy2(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 2); +} + +size_t ZSTD_compressBlock_lazy(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 1); +} + +size_t ZSTD_compressBlock_greedy(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 0); +} + + +FORCE_INLINE_TEMPLATE +size_t ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, + const void* src, size_t srcSize, + const U32 searchMethod, const U32 depth) +{ + seqStore_t* seqStorePtr = &(ctx->seqStore); + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ctx->base; + const U32 dictLimit = ctx->dictLimit; + const U32 lowestIndex = ctx->lowLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictBase = ctx->dictBase; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const dictStart = dictBase + ctx->lowLimit; + + const U32 maxSearches = 1 << ctx->appliedParams.cParams.searchLog; + const U32 mls = ctx->appliedParams.cParams.searchLength; + + typedef size_t (*searchMax_f)(ZSTD_CCtx* zc, const BYTE* ip, const BYTE* iLimit, + size_t* offsetPtr, + U32 maxNbAttempts, U32 matchLengthSearch); + searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS_extDict : ZSTD_HcFindBestMatch_extDict_selectMLS; + + U32 offset_1 = seqStorePtr->rep[0], offset_2 = seqStorePtr->rep[1]; + + /* init */ + ctx->nextToUpdate3 = ctx->nextToUpdate; + ip += (ip == prefixStart); + + /* Match Loop */ + while (ip < ilimit) { + size_t matchLength=0; + size_t offset=0; + const BYTE* start=ip+1; + U32 current = (U32)(ip-base); + + /* check repCode */ + { const U32 repIndex = (U32)(current+1 - offset_1); + const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ + if (MEM_read32(ip+1) == MEM_read32(repMatch)) { + /* repcode detected we should take it */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repEnd, prefixStart) + 4; + if (depth==0) goto _storeSequence; + } } + + /* first search (depth 0) */ + { size_t offsetFound = 99999999; + size_t const ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls); + if (ml2 > matchLength) + matchLength = ml2, start = ip, offset=offsetFound; + } + + if (matchLength < 4) { + ip += ((ip-anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */ + continue; + } + + /* let's try to find a better solution */ + if (depth>=1) + while (ip= 3) & (repIndex > lowestIndex)) /* intentional overflow */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; + int const gain2 = (int)(repLength * 3); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1); + if ((repLength >= 4) && (gain2 > gain1)) + matchLength = repLength, offset = 0, start = ip; + } } + + /* search match, depth 1 */ + { size_t offset2=99999999; + size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offset = offset2, start = ip; + continue; /* search a better one */ + } } + + /* let's find an even better one */ + if ((depth==2) && (ip= 3) & (repIndex > lowestIndex)) /* intentional overflow */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; + int const gain2 = (int)(repLength * 4); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1); + if ((repLength >= 4) && (gain2 > gain1)) + matchLength = repLength, offset = 0, start = ip; + } } + + /* search match, depth 2 */ + { size_t offset2=99999999; + size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offset = offset2, start = ip; + continue; + } } } + break; /* nothing found : store previous solution */ + } + + /* catch up */ + if (offset) { + U32 const matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE)); + const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex; + const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart; + while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */ + offset_2 = offset_1; offset_1 = (U32)(offset - ZSTD_REP_MOVE); + } + + /* store sequence */ +_storeSequence: + { size_t const litLength = start - anchor; + ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength-MINMATCH); + anchor = ip = start + matchLength; + } + + /* check immediate repcode */ + while (ip <= ilimit) { + const U32 repIndex = (U32)((ip-base) - offset_2); + const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected we should take it */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; + offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap offset history */ + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength-MINMATCH); + ip += matchLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } + break; + } } + + /* Save reps for next block */ + seqStorePtr->repToConfirm[0] = offset_1; seqStorePtr->repToConfirm[1] = offset_2; + + /* Return the last literals size */ + return iend - anchor; +} + + +size_t ZSTD_compressBlock_greedy_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 0); +} + +size_t ZSTD_compressBlock_lazy_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 1); +} + +size_t ZSTD_compressBlock_lazy2_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 2); +} + +size_t ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 1, 2); +} diff --git a/src/borg/algorithms/zstd/lib/compress/zstd_lazy.h b/src/borg/algorithms/zstd/lib/compress/zstd_lazy.h new file mode 100644 index 0000000000..a9c4daed25 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/compress/zstd_lazy.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_LAZY_H +#define ZSTD_LAZY_H + +#include "zstd_compress.h" + +#if defined (__cplusplus) +extern "C" { +#endif + +U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls); +void ZSTD_updateTree(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls); +void ZSTD_updateTree_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls); + +size_t ZSTD_compressBlock_btlazy2(ZSTD_CCtx* ctx, const void* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2(ZSTD_CCtx* ctx, const void* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy(ZSTD_CCtx* ctx, const void* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy(ZSTD_CCtx* ctx, const void* src, size_t srcSize); + +size_t ZSTD_compressBlock_greedy_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize); +size_t ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize); + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_LAZY_H */ diff --git a/src/borg/algorithms/zstd/lib/compress/zstd_ldm.c b/src/borg/algorithms/zstd/lib/compress/zstd_ldm.c new file mode 100644 index 0000000000..be50872cf7 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/compress/zstd_ldm.c @@ -0,0 +1,707 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + */ + +#include "zstd_ldm.h" + +#include "zstd_fast.h" /* ZSTD_fillHashTable() */ +#include "zstd_double_fast.h" /* ZSTD_fillDoubleHashTable() */ + +#define LDM_BUCKET_SIZE_LOG 3 +#define LDM_MIN_MATCH_LENGTH 64 +#define LDM_HASH_RLOG 7 +#define LDM_HASH_CHAR_OFFSET 10 + +size_t ZSTD_ldm_initializeParameters(ldmParams_t* params, U32 enableLdm) +{ + ZSTD_STATIC_ASSERT(LDM_BUCKET_SIZE_LOG <= ZSTD_LDM_BUCKETSIZELOG_MAX); + params->enableLdm = enableLdm>0; + params->hashLog = 0; + params->bucketSizeLog = LDM_BUCKET_SIZE_LOG; + params->minMatchLength = LDM_MIN_MATCH_LENGTH; + params->hashEveryLog = ZSTD_LDM_HASHEVERYLOG_NOTSET; + return 0; +} + +void ZSTD_ldm_adjustParameters(ldmParams_t* params, U32 windowLog) +{ + if (params->hashLog == 0) { + params->hashLog = MAX(ZSTD_HASHLOG_MIN, windowLog - LDM_HASH_RLOG); + assert(params->hashLog <= ZSTD_HASHLOG_MAX); + } + if (params->hashEveryLog == ZSTD_LDM_HASHEVERYLOG_NOTSET) { + params->hashEveryLog = + windowLog < params->hashLog ? 0 : windowLog - params->hashLog; + } + params->bucketSizeLog = MIN(params->bucketSizeLog, params->hashLog); +} + +size_t ZSTD_ldm_getTableSize(U32 hashLog, U32 bucketSizeLog) { + size_t const ldmHSize = ((size_t)1) << hashLog; + size_t const ldmBucketSizeLog = MIN(bucketSizeLog, hashLog); + size_t const ldmBucketSize = + ((size_t)1) << (hashLog - ldmBucketSizeLog); + return ldmBucketSize + (ldmHSize * (sizeof(ldmEntry_t))); +} + +/** ZSTD_ldm_getSmallHash() : + * numBits should be <= 32 + * If numBits==0, returns 0. + * @return : the most significant numBits of value. */ +static U32 ZSTD_ldm_getSmallHash(U64 value, U32 numBits) +{ + assert(numBits <= 32); + return numBits == 0 ? 0 : (U32)(value >> (64 - numBits)); +} + +/** ZSTD_ldm_getChecksum() : + * numBitsToDiscard should be <= 32 + * @return : the next most significant 32 bits after numBitsToDiscard */ +static U32 ZSTD_ldm_getChecksum(U64 hash, U32 numBitsToDiscard) +{ + assert(numBitsToDiscard <= 32); + return (hash >> (64 - 32 - numBitsToDiscard)) & 0xFFFFFFFF; +} + +/** ZSTD_ldm_getTag() ; + * Given the hash, returns the most significant numTagBits bits + * after (32 + hbits) bits. + * + * If there are not enough bits remaining, return the last + * numTagBits bits. */ +static U32 ZSTD_ldm_getTag(U64 hash, U32 hbits, U32 numTagBits) +{ + assert(numTagBits < 32 && hbits <= 32); + if (32 - hbits < numTagBits) { + return hash & (((U32)1 << numTagBits) - 1); + } else { + return (hash >> (32 - hbits - numTagBits)) & (((U32)1 << numTagBits) - 1); + } +} + +/** ZSTD_ldm_getBucket() : + * Returns a pointer to the start of the bucket associated with hash. */ +static ldmEntry_t* ZSTD_ldm_getBucket( + ldmState_t* ldmState, size_t hash, ldmParams_t const ldmParams) +{ + return ldmState->hashTable + (hash << ldmParams.bucketSizeLog); +} + +/** ZSTD_ldm_insertEntry() : + * Insert the entry with corresponding hash into the hash table */ +static void ZSTD_ldm_insertEntry(ldmState_t* ldmState, + size_t const hash, const ldmEntry_t entry, + ldmParams_t const ldmParams) +{ + BYTE* const bucketOffsets = ldmState->bucketOffsets; + *(ZSTD_ldm_getBucket(ldmState, hash, ldmParams) + bucketOffsets[hash]) = entry; + bucketOffsets[hash]++; + bucketOffsets[hash] &= ((U32)1 << ldmParams.bucketSizeLog) - 1; +} + +/** ZSTD_ldm_makeEntryAndInsertByTag() : + * + * Gets the small hash, checksum, and tag from the rollingHash. + * + * If the tag matches (1 << ldmParams.hashEveryLog)-1, then + * creates an ldmEntry from the offset, and inserts it into the hash table. + * + * hBits is the length of the small hash, which is the most significant hBits + * of rollingHash. The checksum is the next 32 most significant bits, followed + * by ldmParams.hashEveryLog bits that make up the tag. */ +static void ZSTD_ldm_makeEntryAndInsertByTag(ldmState_t* ldmState, + U64 const rollingHash, + U32 const hBits, + U32 const offset, + ldmParams_t const ldmParams) +{ + U32 const tag = ZSTD_ldm_getTag(rollingHash, hBits, ldmParams.hashEveryLog); + U32 const tagMask = ((U32)1 << ldmParams.hashEveryLog) - 1; + if (tag == tagMask) { + U32 const hash = ZSTD_ldm_getSmallHash(rollingHash, hBits); + U32 const checksum = ZSTD_ldm_getChecksum(rollingHash, hBits); + ldmEntry_t entry; + entry.offset = offset; + entry.checksum = checksum; + ZSTD_ldm_insertEntry(ldmState, hash, entry, ldmParams); + } +} + +/** ZSTD_ldm_getRollingHash() : + * Get a 64-bit hash using the first len bytes from buf. + * + * Giving bytes s = s_1, s_2, ... s_k, the hash is defined to be + * H(s) = s_1*(a^(k-1)) + s_2*(a^(k-2)) + ... + s_k*(a^0) + * + * where the constant a is defined to be prime8bytes. + * + * The implementation adds an offset to each byte, so + * H(s) = (s_1 + HASH_CHAR_OFFSET)*(a^(k-1)) + ... */ +static U64 ZSTD_ldm_getRollingHash(const BYTE* buf, U32 len) +{ + U64 ret = 0; + U32 i; + for (i = 0; i < len; i++) { + ret *= prime8bytes; + ret += buf[i] + LDM_HASH_CHAR_OFFSET; + } + return ret; +} + +/** ZSTD_ldm_ipow() : + * Return base^exp. */ +static U64 ZSTD_ldm_ipow(U64 base, U64 exp) +{ + U64 ret = 1; + while (exp) { + if (exp & 1) { ret *= base; } + exp >>= 1; + base *= base; + } + return ret; +} + +U64 ZSTD_ldm_getHashPower(U32 minMatchLength) { + assert(minMatchLength >= ZSTD_LDM_MINMATCH_MIN); + return ZSTD_ldm_ipow(prime8bytes, minMatchLength - 1); +} + +/** ZSTD_ldm_updateHash() : + * Updates hash by removing toRemove and adding toAdd. */ +static U64 ZSTD_ldm_updateHash(U64 hash, BYTE toRemove, BYTE toAdd, U64 hashPower) +{ + hash -= ((toRemove + LDM_HASH_CHAR_OFFSET) * hashPower); + hash *= prime8bytes; + hash += toAdd + LDM_HASH_CHAR_OFFSET; + return hash; +} + +/** ZSTD_ldm_countBackwardsMatch() : + * Returns the number of bytes that match backwards before pIn and pMatch. + * + * We count only bytes where pMatch >= pBase and pIn >= pAnchor. */ +static size_t ZSTD_ldm_countBackwardsMatch( + const BYTE* pIn, const BYTE* pAnchor, + const BYTE* pMatch, const BYTE* pBase) +{ + size_t matchLength = 0; + while (pIn > pAnchor && pMatch > pBase && pIn[-1] == pMatch[-1]) { + pIn--; + pMatch--; + matchLength++; + } + return matchLength; +} + +/** ZSTD_ldm_fillFastTables() : + * + * Fills the relevant tables for the ZSTD_fast and ZSTD_dfast strategies. + * This is similar to ZSTD_loadDictionaryContent. + * + * The tables for the other strategies are filled within their + * block compressors. */ +static size_t ZSTD_ldm_fillFastTables(ZSTD_CCtx* zc, const void* end) +{ + const BYTE* const iend = (const BYTE*)end; + const U32 mls = zc->appliedParams.cParams.searchLength; + + switch(zc->appliedParams.cParams.strategy) + { + case ZSTD_fast: + ZSTD_fillHashTable(zc, iend, mls); + zc->nextToUpdate = (U32)(iend - zc->base); + break; + + case ZSTD_dfast: + ZSTD_fillDoubleHashTable(zc, iend, mls); + zc->nextToUpdate = (U32)(iend - zc->base); + break; + + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + case ZSTD_btlazy2: + case ZSTD_btopt: + case ZSTD_btultra: + break; + default: + assert(0); /* not possible : not a valid strategy id */ + } + + return 0; +} + +/** ZSTD_ldm_fillLdmHashTable() : + * + * Fills hashTable from (lastHashed + 1) to iend (non-inclusive). + * lastHash is the rolling hash that corresponds to lastHashed. + * + * Returns the rolling hash corresponding to position iend-1. */ +static U64 ZSTD_ldm_fillLdmHashTable(ldmState_t* state, + U64 lastHash, const BYTE* lastHashed, + const BYTE* iend, const BYTE* base, + U32 hBits, ldmParams_t const ldmParams) +{ + U64 rollingHash = lastHash; + const BYTE* cur = lastHashed + 1; + + while (cur < iend) { + rollingHash = ZSTD_ldm_updateHash(rollingHash, cur[-1], + cur[ldmParams.minMatchLength-1], + state->hashPower); + ZSTD_ldm_makeEntryAndInsertByTag(state, + rollingHash, hBits, + (U32)(cur - base), ldmParams); + ++cur; + } + return rollingHash; +} + + +/** ZSTD_ldm_limitTableUpdate() : + * + * Sets cctx->nextToUpdate to a position corresponding closer to anchor + * if it is far way + * (after a long match, only update tables a limited amount). */ +static void ZSTD_ldm_limitTableUpdate(ZSTD_CCtx* cctx, const BYTE* anchor) +{ + U32 const current = (U32)(anchor - cctx->base); + if (current > cctx->nextToUpdate + 1024) { + cctx->nextToUpdate = + current - MIN(512, current - cctx->nextToUpdate - 1024); + } +} + +typedef size_t (*ZSTD_blockCompressor) (ZSTD_CCtx* ctx, const void* src, size_t srcSize); +/* defined in zstd_compress.c */ +ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict); + +FORCE_INLINE_TEMPLATE +size_t ZSTD_compressBlock_ldm_generic(ZSTD_CCtx* cctx, + const void* src, size_t srcSize) +{ + ldmState_t* const ldmState = &(cctx->ldmState); + const ldmParams_t ldmParams = cctx->appliedParams.ldmParams; + const U64 hashPower = ldmState->hashPower; + const U32 hBits = ldmParams.hashLog - ldmParams.bucketSizeLog; + const U32 ldmBucketSize = ((U32)1 << ldmParams.bucketSizeLog); + const U32 ldmTagMask = ((U32)1 << ldmParams.hashEveryLog) - 1; + seqStore_t* const seqStorePtr = &(cctx->seqStore); + const BYTE* const base = cctx->base; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const U32 lowestIndex = cctx->dictLimit; + const BYTE* const lowest = base + lowestIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - MAX(ldmParams.minMatchLength, HASH_READ_SIZE); + + const ZSTD_blockCompressor blockCompressor = + ZSTD_selectBlockCompressor(cctx->appliedParams.cParams.strategy, 0); + U32* const repToConfirm = seqStorePtr->repToConfirm; + U32 savedRep[ZSTD_REP_NUM]; + U64 rollingHash = 0; + const BYTE* lastHashed = NULL; + size_t i, lastLiterals; + + /* Save seqStorePtr->rep and copy repToConfirm */ + for (i = 0; i < ZSTD_REP_NUM; i++) + savedRep[i] = repToConfirm[i] = seqStorePtr->rep[i]; + + /* Main Search Loop */ + while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ + size_t mLength; + U32 const current = (U32)(ip - base); + size_t forwardMatchLength = 0, backwardMatchLength = 0; + ldmEntry_t* bestEntry = NULL; + if (ip != istart) { + rollingHash = ZSTD_ldm_updateHash(rollingHash, lastHashed[0], + lastHashed[ldmParams.minMatchLength], + hashPower); + } else { + rollingHash = ZSTD_ldm_getRollingHash(ip, ldmParams.minMatchLength); + } + lastHashed = ip; + + /* Do not insert and do not look for a match */ + if (ZSTD_ldm_getTag(rollingHash, hBits, ldmParams.hashEveryLog) != + ldmTagMask) { + ip++; + continue; + } + + /* Get the best entry and compute the match lengths */ + { + ldmEntry_t* const bucket = + ZSTD_ldm_getBucket(ldmState, + ZSTD_ldm_getSmallHash(rollingHash, hBits), + ldmParams); + ldmEntry_t* cur; + size_t bestMatchLength = 0; + U32 const checksum = ZSTD_ldm_getChecksum(rollingHash, hBits); + + for (cur = bucket; cur < bucket + ldmBucketSize; ++cur) { + const BYTE* const pMatch = cur->offset + base; + size_t curForwardMatchLength, curBackwardMatchLength, + curTotalMatchLength; + if (cur->checksum != checksum || cur->offset <= lowestIndex) { + continue; + } + + curForwardMatchLength = ZSTD_count(ip, pMatch, iend); + if (curForwardMatchLength < ldmParams.minMatchLength) { + continue; + } + curBackwardMatchLength = ZSTD_ldm_countBackwardsMatch( + ip, anchor, pMatch, lowest); + curTotalMatchLength = curForwardMatchLength + + curBackwardMatchLength; + + if (curTotalMatchLength > bestMatchLength) { + bestMatchLength = curTotalMatchLength; + forwardMatchLength = curForwardMatchLength; + backwardMatchLength = curBackwardMatchLength; + bestEntry = cur; + } + } + } + + /* No match found -- continue searching */ + if (bestEntry == NULL) { + ZSTD_ldm_makeEntryAndInsertByTag(ldmState, rollingHash, + hBits, current, + ldmParams); + ip++; + continue; + } + + /* Match found */ + mLength = forwardMatchLength + backwardMatchLength; + ip -= backwardMatchLength; + + /* Call the block compressor on the remaining literals */ + { + U32 const matchIndex = bestEntry->offset; + const BYTE* const match = base + matchIndex - backwardMatchLength; + U32 const offset = (U32)(ip - match); + + /* Overwrite rep codes */ + for (i = 0; i < ZSTD_REP_NUM; i++) + seqStorePtr->rep[i] = repToConfirm[i]; + + /* Fill tables for block compressor */ + ZSTD_ldm_limitTableUpdate(cctx, anchor); + ZSTD_ldm_fillFastTables(cctx, anchor); + + /* Call block compressor and get remaining literals */ + lastLiterals = blockCompressor(cctx, anchor, ip - anchor); + cctx->nextToUpdate = (U32)(ip - base); + + /* Update repToConfirm with the new offset */ + for (i = ZSTD_REP_NUM - 1; i > 0; i--) + repToConfirm[i] = repToConfirm[i-1]; + repToConfirm[0] = offset; + + /* Store the sequence with the leftover literals */ + ZSTD_storeSeq(seqStorePtr, lastLiterals, ip - lastLiterals, + offset + ZSTD_REP_MOVE, mLength - MINMATCH); + } + + /* Insert the current entry into the hash table */ + ZSTD_ldm_makeEntryAndInsertByTag(ldmState, rollingHash, hBits, + (U32)(lastHashed - base), + ldmParams); + + assert(ip + backwardMatchLength == lastHashed); + + /* Fill the hash table from lastHashed+1 to ip+mLength*/ + /* Heuristic: don't need to fill the entire table at end of block */ + if (ip + mLength < ilimit) { + rollingHash = ZSTD_ldm_fillLdmHashTable( + ldmState, rollingHash, lastHashed, + ip + mLength, base, hBits, ldmParams); + lastHashed = ip + mLength - 1; + } + ip += mLength; + anchor = ip; + /* Check immediate repcode */ + while ( (ip < ilimit) + && ( (repToConfirm[1] > 0) && (repToConfirm[1] <= (U32)(ip-lowest)) + && (MEM_read32(ip) == MEM_read32(ip - repToConfirm[1])) )) { + + size_t const rLength = ZSTD_count(ip+4, ip+4-repToConfirm[1], + iend) + 4; + /* Swap repToConfirm[1] <=> repToConfirm[0] */ + { + U32 const tmpOff = repToConfirm[1]; + repToConfirm[1] = repToConfirm[0]; + repToConfirm[0] = tmpOff; + } + + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength-MINMATCH); + + /* Fill the hash table from lastHashed+1 to ip+rLength*/ + if (ip + rLength < ilimit) { + rollingHash = ZSTD_ldm_fillLdmHashTable( + ldmState, rollingHash, lastHashed, + ip + rLength, base, hBits, ldmParams); + lastHashed = ip + rLength - 1; + } + ip += rLength; + anchor = ip; + } + } + + /* Overwrite rep */ + for (i = 0; i < ZSTD_REP_NUM; i++) + seqStorePtr->rep[i] = repToConfirm[i]; + + ZSTD_ldm_limitTableUpdate(cctx, anchor); + ZSTD_ldm_fillFastTables(cctx, anchor); + + lastLiterals = blockCompressor(cctx, anchor, iend - anchor); + cctx->nextToUpdate = (U32)(iend - base); + + /* Restore seqStorePtr->rep */ + for (i = 0; i < ZSTD_REP_NUM; i++) + seqStorePtr->rep[i] = savedRep[i]; + + /* Return the last literals size */ + return lastLiterals; +} + +size_t ZSTD_compressBlock_ldm(ZSTD_CCtx* ctx, + const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_ldm_generic(ctx, src, srcSize); +} + +static size_t ZSTD_compressBlock_ldm_extDict_generic( + ZSTD_CCtx* ctx, + const void* src, size_t srcSize) +{ + ldmState_t* const ldmState = &(ctx->ldmState); + const ldmParams_t ldmParams = ctx->appliedParams.ldmParams; + const U64 hashPower = ldmState->hashPower; + const U32 hBits = ldmParams.hashLog - ldmParams.bucketSizeLog; + const U32 ldmBucketSize = ((U32)1 << ldmParams.bucketSizeLog); + const U32 ldmTagMask = ((U32)1 << ldmParams.hashEveryLog) - 1; + seqStore_t* const seqStorePtr = &(ctx->seqStore); + const BYTE* const base = ctx->base; + const BYTE* const dictBase = ctx->dictBase; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const U32 lowestIndex = ctx->lowLimit; + const BYTE* const dictStart = dictBase + lowestIndex; + const U32 dictLimit = ctx->dictLimit; + const BYTE* const lowPrefixPtr = base + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - MAX(ldmParams.minMatchLength, HASH_READ_SIZE); + + const ZSTD_blockCompressor blockCompressor = + ZSTD_selectBlockCompressor(ctx->appliedParams.cParams.strategy, 1); + U32* const repToConfirm = seqStorePtr->repToConfirm; + U32 savedRep[ZSTD_REP_NUM]; + U64 rollingHash = 0; + const BYTE* lastHashed = NULL; + size_t i, lastLiterals; + + /* Save seqStorePtr->rep and copy repToConfirm */ + for (i = 0; i < ZSTD_REP_NUM; i++) { + savedRep[i] = repToConfirm[i] = seqStorePtr->rep[i]; + } + + /* Search Loop */ + while (ip < ilimit) { /* < instead of <=, because (ip+1) */ + size_t mLength; + const U32 current = (U32)(ip-base); + size_t forwardMatchLength = 0, backwardMatchLength = 0; + ldmEntry_t* bestEntry = NULL; + if (ip != istart) { + rollingHash = ZSTD_ldm_updateHash(rollingHash, lastHashed[0], + lastHashed[ldmParams.minMatchLength], + hashPower); + } else { + rollingHash = ZSTD_ldm_getRollingHash(ip, ldmParams.minMatchLength); + } + lastHashed = ip; + + if (ZSTD_ldm_getTag(rollingHash, hBits, ldmParams.hashEveryLog) != + ldmTagMask) { + /* Don't insert and don't look for a match */ + ip++; + continue; + } + + /* Get the best entry and compute the match lengths */ + { + ldmEntry_t* const bucket = + ZSTD_ldm_getBucket(ldmState, + ZSTD_ldm_getSmallHash(rollingHash, hBits), + ldmParams); + ldmEntry_t* cur; + size_t bestMatchLength = 0; + U32 const checksum = ZSTD_ldm_getChecksum(rollingHash, hBits); + + for (cur = bucket; cur < bucket + ldmBucketSize; ++cur) { + const BYTE* const curMatchBase = + cur->offset < dictLimit ? dictBase : base; + const BYTE* const pMatch = curMatchBase + cur->offset; + const BYTE* const matchEnd = + cur->offset < dictLimit ? dictEnd : iend; + const BYTE* const lowMatchPtr = + cur->offset < dictLimit ? dictStart : lowPrefixPtr; + size_t curForwardMatchLength, curBackwardMatchLength, + curTotalMatchLength; + + if (cur->checksum != checksum || cur->offset <= lowestIndex) { + continue; + } + + curForwardMatchLength = ZSTD_count_2segments( + ip, pMatch, iend, + matchEnd, lowPrefixPtr); + if (curForwardMatchLength < ldmParams.minMatchLength) { + continue; + } + curBackwardMatchLength = ZSTD_ldm_countBackwardsMatch( + ip, anchor, pMatch, lowMatchPtr); + curTotalMatchLength = curForwardMatchLength + + curBackwardMatchLength; + + if (curTotalMatchLength > bestMatchLength) { + bestMatchLength = curTotalMatchLength; + forwardMatchLength = curForwardMatchLength; + backwardMatchLength = curBackwardMatchLength; + bestEntry = cur; + } + } + } + + /* No match found -- continue searching */ + if (bestEntry == NULL) { + ZSTD_ldm_makeEntryAndInsertByTag(ldmState, rollingHash, hBits, + (U32)(lastHashed - base), + ldmParams); + ip++; + continue; + } + + /* Match found */ + mLength = forwardMatchLength + backwardMatchLength; + ip -= backwardMatchLength; + + /* Call the block compressor on the remaining literals */ + { + /* ip = current - backwardMatchLength + * The match is at (bestEntry->offset - backwardMatchLength) */ + U32 const matchIndex = bestEntry->offset; + U32 const offset = current - matchIndex; + + /* Overwrite rep codes */ + for (i = 0; i < ZSTD_REP_NUM; i++) + seqStorePtr->rep[i] = repToConfirm[i]; + + /* Fill the hash table for the block compressor */ + ZSTD_ldm_limitTableUpdate(ctx, anchor); + ZSTD_ldm_fillFastTables(ctx, anchor); + + /* Call block compressor and get remaining literals */ + lastLiterals = blockCompressor(ctx, anchor, ip - anchor); + ctx->nextToUpdate = (U32)(ip - base); + + /* Update repToConfirm with the new offset */ + for (i = ZSTD_REP_NUM - 1; i > 0; i--) + repToConfirm[i] = repToConfirm[i-1]; + repToConfirm[0] = offset; + + /* Store the sequence with the leftover literals */ + ZSTD_storeSeq(seqStorePtr, lastLiterals, ip - lastLiterals, + offset + ZSTD_REP_MOVE, mLength - MINMATCH); + } + + /* Insert the current entry into the hash table */ + ZSTD_ldm_makeEntryAndInsertByTag(ldmState, rollingHash, hBits, + (U32)(lastHashed - base), + ldmParams); + + /* Fill the hash table from lastHashed+1 to ip+mLength */ + assert(ip + backwardMatchLength == lastHashed); + if (ip + mLength < ilimit) { + rollingHash = ZSTD_ldm_fillLdmHashTable( + ldmState, rollingHash, lastHashed, + ip + mLength, base, hBits, + ldmParams); + lastHashed = ip + mLength - 1; + } + ip += mLength; + anchor = ip; + + /* check immediate repcode */ + while (ip < ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex2 = current2 - repToConfirm[1]; + const BYTE* repMatch2 = repIndex2 < dictLimit ? + dictBase + repIndex2 : base + repIndex2; + if ( (((U32)((dictLimit-1) - repIndex2) >= 3) & + (repIndex2 > lowestIndex)) /* intentional overflow */ + && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex2 < dictLimit ? + dictEnd : iend; + size_t const repLength2 = + ZSTD_count_2segments(ip+4, repMatch2+4, iend, + repEnd2, lowPrefixPtr) + 4; + + U32 tmpOffset = repToConfirm[1]; + repToConfirm[1] = repToConfirm[0]; + repToConfirm[0] = tmpOffset; + + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2-MINMATCH); + + /* Fill the hash table from lastHashed+1 to ip+repLength2*/ + if (ip + repLength2 < ilimit) { + rollingHash = ZSTD_ldm_fillLdmHashTable( + ldmState, rollingHash, lastHashed, + ip + repLength2, base, hBits, + ldmParams); + lastHashed = ip + repLength2 - 1; + } + ip += repLength2; + anchor = ip; + continue; + } + break; + } + } + + /* Overwrite rep */ + for (i = 0; i < ZSTD_REP_NUM; i++) + seqStorePtr->rep[i] = repToConfirm[i]; + + ZSTD_ldm_limitTableUpdate(ctx, anchor); + ZSTD_ldm_fillFastTables(ctx, anchor); + + /* Call the block compressor one last time on the last literals */ + lastLiterals = blockCompressor(ctx, anchor, iend - anchor); + ctx->nextToUpdate = (U32)(iend - base); + + /* Restore seqStorePtr->rep */ + for (i = 0; i < ZSTD_REP_NUM; i++) + seqStorePtr->rep[i] = savedRep[i]; + + /* Return the last literals size */ + return lastLiterals; +} + +size_t ZSTD_compressBlock_ldm_extDict(ZSTD_CCtx* ctx, + const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_ldm_extDict_generic(ctx, src, srcSize); +} diff --git a/src/borg/algorithms/zstd/lib/compress/zstd_ldm.h b/src/borg/algorithms/zstd/lib/compress/zstd_ldm.h new file mode 100644 index 0000000000..d6d3d42c33 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/compress/zstd_ldm.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + */ + +#ifndef ZSTD_LDM_H +#define ZSTD_LDM_H + +#include "zstd_compress.h" + +#if defined (__cplusplus) +extern "C" { +#endif + +/*-************************************* +* Long distance matching +***************************************/ + +#define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_DEFAULTMAX +#define ZSTD_LDM_HASHEVERYLOG_NOTSET 9999 + +/** ZSTD_compressBlock_ldm_generic() : + * + * This is a block compressor intended for long distance matching. + * + * The function searches for matches of length at least + * ldmParams.minMatchLength using a hash table in cctx->ldmState. + * Matches can be at a distance of up to cParams.windowLog. + * + * Upon finding a match, the unmatched literals are compressed using a + * ZSTD_blockCompressor (depending on the strategy in the compression + * parameters), which stores the matched sequences. The "long distance" + * match is then stored with the remaining literals from the + * ZSTD_blockCompressor. */ +size_t ZSTD_compressBlock_ldm(ZSTD_CCtx* cctx, const void* src, size_t srcSize); +size_t ZSTD_compressBlock_ldm_extDict(ZSTD_CCtx* ctx, + const void* src, size_t srcSize); + +/** ZSTD_ldm_initializeParameters() : + * Initialize the long distance matching parameters to their default values. */ +size_t ZSTD_ldm_initializeParameters(ldmParams_t* params, U32 enableLdm); + +/** ZSTD_ldm_getTableSize() : + * Estimate the space needed for long distance matching tables. */ +size_t ZSTD_ldm_getTableSize(U32 hashLog, U32 bucketSizeLog); + +/** ZSTD_ldm_getTableSize() : + * Return prime8bytes^(minMatchLength-1) */ +U64 ZSTD_ldm_getHashPower(U32 minMatchLength); + +/** ZSTD_ldm_adjustParameters() : + * If the params->hashEveryLog is not set, set it to its default value based on + * windowLog and params->hashLog. + * + * Ensures that params->bucketSizeLog is <= params->hashLog (setting it to + * params->hashLog if it is not). */ +void ZSTD_ldm_adjustParameters(ldmParams_t* params, U32 windowLog); + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_FAST_H */ diff --git a/src/borg/algorithms/zstd/lib/compress/zstd_opt.c b/src/borg/algorithms/zstd/lib/compress/zstd_opt.c new file mode 100644 index 0000000000..c47ce23ad5 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/compress/zstd_opt.c @@ -0,0 +1,957 @@ +/* + * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_opt.h" +#include "zstd_lazy.h" + + +#define ZSTD_LITFREQ_ADD 2 +#define ZSTD_FREQ_DIV 4 +#define ZSTD_MAX_PRICE (1<<30) + +/*-************************************* +* Price functions for optimal parser +***************************************/ +static void ZSTD_setLog2Prices(optState_t* optPtr) +{ + optPtr->log2matchLengthSum = ZSTD_highbit32(optPtr->matchLengthSum+1); + optPtr->log2litLengthSum = ZSTD_highbit32(optPtr->litLengthSum+1); + optPtr->log2litSum = ZSTD_highbit32(optPtr->litSum+1); + optPtr->log2offCodeSum = ZSTD_highbit32(optPtr->offCodeSum+1); + optPtr->factor = 1 + ((optPtr->litSum>>5) / optPtr->litLengthSum) + ((optPtr->litSum<<1) / (optPtr->litSum + optPtr->matchSum)); +} + + +static void ZSTD_rescaleFreqs(optState_t* optPtr, const BYTE* src, size_t srcSize) +{ + unsigned u; + + optPtr->cachedLiterals = NULL; + optPtr->cachedPrice = optPtr->cachedLitLength = 0; + optPtr->staticPrices = 0; + + if (optPtr->litLengthSum == 0) { + if (srcSize <= 1024) optPtr->staticPrices = 1; + + assert(optPtr->litFreq!=NULL); + for (u=0; u<=MaxLit; u++) + optPtr->litFreq[u] = 0; + for (u=0; ulitFreq[src[u]]++; + + optPtr->litSum = 0; + optPtr->litLengthSum = MaxLL+1; + optPtr->matchLengthSum = MaxML+1; + optPtr->offCodeSum = (MaxOff+1); + optPtr->matchSum = (ZSTD_LITFREQ_ADD<litFreq[u] = 1 + (optPtr->litFreq[u]>>ZSTD_FREQ_DIV); + optPtr->litSum += optPtr->litFreq[u]; + } + for (u=0; u<=MaxLL; u++) + optPtr->litLengthFreq[u] = 1; + for (u=0; u<=MaxML; u++) + optPtr->matchLengthFreq[u] = 1; + for (u=0; u<=MaxOff; u++) + optPtr->offCodeFreq[u] = 1; + } else { + optPtr->matchLengthSum = 0; + optPtr->litLengthSum = 0; + optPtr->offCodeSum = 0; + optPtr->matchSum = 0; + optPtr->litSum = 0; + + for (u=0; u<=MaxLit; u++) { + optPtr->litFreq[u] = 1 + (optPtr->litFreq[u]>>(ZSTD_FREQ_DIV+1)); + optPtr->litSum += optPtr->litFreq[u]; + } + for (u=0; u<=MaxLL; u++) { + optPtr->litLengthFreq[u] = 1 + (optPtr->litLengthFreq[u]>>(ZSTD_FREQ_DIV+1)); + optPtr->litLengthSum += optPtr->litLengthFreq[u]; + } + for (u=0; u<=MaxML; u++) { + optPtr->matchLengthFreq[u] = 1 + (optPtr->matchLengthFreq[u]>>ZSTD_FREQ_DIV); + optPtr->matchLengthSum += optPtr->matchLengthFreq[u]; + optPtr->matchSum += optPtr->matchLengthFreq[u] * (u + 3); + } + optPtr->matchSum *= ZSTD_LITFREQ_ADD; + for (u=0; u<=MaxOff; u++) { + optPtr->offCodeFreq[u] = 1 + (optPtr->offCodeFreq[u]>>ZSTD_FREQ_DIV); + optPtr->offCodeSum += optPtr->offCodeFreq[u]; + } + } + + ZSTD_setLog2Prices(optPtr); +} + + +static U32 ZSTD_getLiteralPrice(optState_t* optPtr, U32 litLength, const BYTE* literals) +{ + U32 price, u; + + if (optPtr->staticPrices) + return ZSTD_highbit32((U32)litLength+1) + (litLength*6); + + if (litLength == 0) + return optPtr->log2litLengthSum - ZSTD_highbit32(optPtr->litLengthFreq[0]+1); + + /* literals */ + if (optPtr->cachedLiterals == literals) { + U32 const additional = litLength - optPtr->cachedLitLength; + const BYTE* literals2 = optPtr->cachedLiterals + optPtr->cachedLitLength; + price = optPtr->cachedPrice + additional * optPtr->log2litSum; + for (u=0; u < additional; u++) + price -= ZSTD_highbit32(optPtr->litFreq[literals2[u]]+1); + optPtr->cachedPrice = price; + optPtr->cachedLitLength = litLength; + } else { + price = litLength * optPtr->log2litSum; + for (u=0; u < litLength; u++) + price -= ZSTD_highbit32(optPtr->litFreq[literals[u]]+1); + + if (litLength >= 12) { + optPtr->cachedLiterals = literals; + optPtr->cachedPrice = price; + optPtr->cachedLitLength = litLength; + } + } + + /* literal Length */ + { const BYTE LL_deltaCode = 19; + const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; + price += LL_bits[llCode] + optPtr->log2litLengthSum - ZSTD_highbit32(optPtr->litLengthFreq[llCode]+1); + } + + return price; +} + + +FORCE_INLINE_TEMPLATE U32 ZSTD_getPrice(optState_t* optPtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength, const int ultra) +{ + /* offset */ + U32 price; + BYTE const offCode = (BYTE)ZSTD_highbit32(offset+1); + + if (optPtr->staticPrices) + return ZSTD_getLiteralPrice(optPtr, litLength, literals) + ZSTD_highbit32((U32)matchLength+1) + 16 + offCode; + + price = offCode + optPtr->log2offCodeSum - ZSTD_highbit32(optPtr->offCodeFreq[offCode]+1); + if (!ultra && offCode >= 20) price += (offCode-19)*2; + + /* match Length */ + { const BYTE ML_deltaCode = 36; + const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength]; + price += ML_bits[mlCode] + optPtr->log2matchLengthSum - ZSTD_highbit32(optPtr->matchLengthFreq[mlCode]+1); + } + + return price + ZSTD_getLiteralPrice(optPtr, litLength, literals) + optPtr->factor; +} + + +static void ZSTD_updatePrice(optState_t* optPtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength) +{ + U32 u; + + /* literals */ + optPtr->litSum += litLength*ZSTD_LITFREQ_ADD; + for (u=0; u < litLength; u++) + optPtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD; + + /* literal Length */ + { const BYTE LL_deltaCode = 19; + const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; + optPtr->litLengthFreq[llCode]++; + optPtr->litLengthSum++; + } + + /* match offset */ + { BYTE const offCode = (BYTE)ZSTD_highbit32(offset+1); + optPtr->offCodeSum++; + optPtr->offCodeFreq[offCode]++; + } + + /* match Length */ + { const BYTE ML_deltaCode = 36; + const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength]; + optPtr->matchLengthFreq[mlCode]++; + optPtr->matchLengthSum++; + } + + ZSTD_setLog2Prices(optPtr); +} + + +#define SET_PRICE(pos, mlen_, offset_, litlen_, price_) \ + { \ + while (last_pos < pos) { opt[last_pos+1].price = ZSTD_MAX_PRICE; last_pos++; } \ + opt[pos].mlen = mlen_; \ + opt[pos].off = offset_; \ + opt[pos].litlen = litlen_; \ + opt[pos].price = price_; \ + } + + +/* function safe only for comparisons */ +static U32 ZSTD_readMINMATCH(const void* memPtr, U32 length) +{ + switch (length) + { + default : + case 4 : return MEM_read32(memPtr); + case 3 : if (MEM_isLittleEndian()) + return MEM_read32(memPtr)<<8; + else + return MEM_read32(memPtr)>>8; + } +} + + +/* Update hashTable3 up to ip (excluded) + Assumption : always within prefix (i.e. not within extDict) */ +static +U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_CCtx* zc, const BYTE* ip) +{ + U32* const hashTable3 = zc->hashTable3; + U32 const hashLog3 = zc->hashLog3; + const BYTE* const base = zc->base; + U32 idx = zc->nextToUpdate3; + const U32 target = zc->nextToUpdate3 = (U32)(ip - base); + const size_t hash3 = ZSTD_hash3Ptr(ip, hashLog3); + + while(idx < target) { + hashTable3[ZSTD_hash3Ptr(base+idx, hashLog3)] = idx; + idx++; + } + + return hashTable3[hash3]; +} + + +/*-************************************* +* Binary Tree search +***************************************/ +static U32 ZSTD_insertBtAndGetAllMatches ( + ZSTD_CCtx* zc, + const BYTE* const ip, const BYTE* const iLimit, + U32 nbCompares, const U32 mls, + U32 extDict, ZSTD_match_t* matches, const U32 minMatchLen) +{ + const BYTE* const base = zc->base; + const U32 current = (U32)(ip-base); + const U32 hashLog = zc->appliedParams.cParams.hashLog; + const size_t h = ZSTD_hashPtr(ip, hashLog, mls); + U32* const hashTable = zc->hashTable; + U32 matchIndex = hashTable[h]; + U32* const bt = zc->chainTable; + const U32 btLog = zc->appliedParams.cParams.chainLog - 1; + const U32 btMask= (1U << btLog) - 1; + size_t commonLengthSmaller=0, commonLengthLarger=0; + const BYTE* const dictBase = zc->dictBase; + const U32 dictLimit = zc->dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const U32 btLow = btMask >= current ? 0 : current - btMask; + const U32 windowLow = zc->lowLimit; + U32* smallerPtr = bt + 2*(current&btMask); + U32* largerPtr = bt + 2*(current&btMask) + 1; + U32 matchEndIdx = current+8; + U32 dummy32; /* to be nullified at the end */ + U32 mnum = 0; + + const U32 minMatch = (mls == 3) ? 3 : 4; + size_t bestLength = minMatchLen-1; + + if (minMatch == 3) { /* HC3 match finder */ + U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3 (zc, ip); + if (matchIndex3>windowLow && (current - matchIndex3 < (1<<18))) { + const BYTE* match; + size_t currentMl=0; + if ((!extDict) || matchIndex3 >= dictLimit) { + match = base + matchIndex3; + if (match[bestLength] == ip[bestLength]) currentMl = ZSTD_count(ip, match, iLimit); + } else { + match = dictBase + matchIndex3; + if (ZSTD_readMINMATCH(match, MINMATCH) == ZSTD_readMINMATCH(ip, MINMATCH)) /* assumption : matchIndex3 <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+MINMATCH, match+MINMATCH, iLimit, dictEnd, prefixStart) + MINMATCH; + } + + /* save best solution */ + if (currentMl > bestLength) { + bestLength = currentMl; + matches[mnum].off = ZSTD_REP_MOVE_OPT + current - matchIndex3; + matches[mnum].len = (U32)currentMl; + mnum++; + if (currentMl > ZSTD_OPT_NUM) goto update; + if (ip+currentMl == iLimit) goto update; /* best possible, and avoid read overflow*/ + } + } + } + + hashTable[h] = current; /* Update Hash Table */ + + while (nbCompares-- && (matchIndex > windowLow)) { + U32* nextPtr = bt + 2*(matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const BYTE* match; + + if ((!extDict) || (matchIndex+matchLength >= dictLimit)) { + match = base + matchIndex; + if (match[matchLength] == ip[matchLength]) { + matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iLimit) +1; + } + } else { + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ + } + + if (matchLength > bestLength) { + if (matchLength > matchEndIdx - matchIndex) matchEndIdx = matchIndex + (U32)matchLength; + bestLength = matchLength; + matches[mnum].off = ZSTD_REP_MOVE_OPT + current - matchIndex; + matches[mnum].len = (U32)matchLength; + mnum++; + if (matchLength > ZSTD_OPT_NUM) break; + if (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */ + break; /* drop, to guarantee consistency (miss a little bit of compression) */ + } + + if (match[matchLength] < ip[matchLength]) { + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ + matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; + +update: + zc->nextToUpdate = (matchEndIdx > current + 8) ? matchEndIdx - 8 : current+1; + return mnum; +} + + +/** Tree updater, providing best match */ +static U32 ZSTD_BtGetAllMatches ( + ZSTD_CCtx* zc, + const BYTE* const ip, const BYTE* const iLimit, + const U32 maxNbAttempts, const U32 mls, ZSTD_match_t* matches, const U32 minMatchLen) +{ + if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ + ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls); + return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 0, matches, minMatchLen); +} + + +static U32 ZSTD_BtGetAllMatches_selectMLS ( + ZSTD_CCtx* zc, /* Index table will be updated */ + const BYTE* ip, const BYTE* const iHighLimit, + const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, const U32 minMatchLen) +{ + switch(matchLengthSearch) + { + case 3 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen); + default : + case 4 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen); + case 5 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen); + case 7 : + case 6 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen); + } +} + +/** Tree updater, providing best match */ +static U32 ZSTD_BtGetAllMatches_extDict ( + ZSTD_CCtx* zc, + const BYTE* const ip, const BYTE* const iLimit, + const U32 maxNbAttempts, const U32 mls, ZSTD_match_t* matches, const U32 minMatchLen) +{ + if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ + ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls); + return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 1, matches, minMatchLen); +} + + +static U32 ZSTD_BtGetAllMatches_selectMLS_extDict ( + ZSTD_CCtx* zc, /* Index table will be updated */ + const BYTE* ip, const BYTE* const iHighLimit, + const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, const U32 minMatchLen) +{ + switch(matchLengthSearch) + { + case 3 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen); + default : + case 4 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen); + case 5 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen); + case 7 : + case 6 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen); + } +} + + +/*-******************************* +* Optimal parser +*********************************/ +FORCE_INLINE_TEMPLATE +size_t ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, + const void* src, size_t srcSize, const int ultra) +{ + seqStore_t* seqStorePtr = &(ctx->seqStore); + optState_t* optStatePtr = &(ctx->optState); + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ctx->base; + const BYTE* const prefixStart = base + ctx->dictLimit; + + const U32 maxSearches = 1U << ctx->appliedParams.cParams.searchLog; + const U32 sufficient_len = ctx->appliedParams.cParams.targetLength; + const U32 mls = ctx->appliedParams.cParams.searchLength; + const U32 minMatch = (ctx->appliedParams.cParams.searchLength == 3) ? 3 : 4; + + ZSTD_optimal_t* opt = optStatePtr->priceTable; + ZSTD_match_t* matches = optStatePtr->matchTable; + const BYTE* inr; + U32 offset, rep[ZSTD_REP_NUM]; + + /* init */ + ctx->nextToUpdate3 = ctx->nextToUpdate; + ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize); + ip += (ip==prefixStart); + { U32 i; for (i=0; irep[i]; } + + /* Match Loop */ + while (ip < ilimit) { + U32 cur, match_num, last_pos, litlen, price; + U32 u, mlen, best_mlen, best_off, litLength; + memset(opt, 0, sizeof(ZSTD_optimal_t)); + last_pos = 0; + litlen = (U32)(ip - anchor); + + /* check repCode */ + { U32 i, last_i = ZSTD_REP_CHECK + (ip==anchor); + for (i=(ip == anchor); i 0) && (repCur < (S32)(ip-prefixStart)) + && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repCur, minMatch))) { + mlen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repCur, iend) + minMatch; + if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { + best_mlen = mlen; best_off = i; cur = 0; last_pos = 1; + goto _storeSequence; + } + best_off = i - (ip == anchor); + do { + price = ZSTD_getPrice(optStatePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra); + if (mlen > last_pos || price < opt[mlen].price) + SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */ + mlen--; + } while (mlen >= minMatch); + } } } + + match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, ip, iend, maxSearches, mls, matches, minMatch); + + if (!last_pos && !match_num) { ip++; continue; } + + if (match_num && (matches[match_num-1].len > sufficient_len || matches[match_num-1].len >= ZSTD_OPT_NUM)) { + best_mlen = matches[match_num-1].len; + best_off = matches[match_num-1].off; + cur = 0; + last_pos = 1; + goto _storeSequence; + } + + /* set prices using matches at position = 0 */ + best_mlen = (last_pos) ? last_pos : minMatch; + for (u = 0; u < match_num; u++) { + mlen = (u>0) ? matches[u-1].len+1 : best_mlen; + best_mlen = matches[u].len; + while (mlen <= best_mlen) { + price = ZSTD_getPrice(optStatePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra); + if (mlen > last_pos || price < opt[mlen].price) + SET_PRICE(mlen, mlen, matches[u].off, litlen, price); /* note : macro modifies last_pos */ + mlen++; + } } + + if (last_pos < minMatch) { ip++; continue; } + + /* initialize opt[0] */ + { U32 i ; for (i=0; i litlen) { + price = opt[cur - litlen].price + ZSTD_getLiteralPrice(optStatePtr, litlen, inr-litlen); + } else + price = ZSTD_getLiteralPrice(optStatePtr, litlen, anchor); + } else { + litlen = 1; + price = opt[cur - 1].price + ZSTD_getLiteralPrice(optStatePtr, litlen, inr-1); + } + + if (cur > last_pos || price <= opt[cur].price) + SET_PRICE(cur, 1, 0, litlen, price); + + if (cur == last_pos) break; + + if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */ + continue; + + mlen = opt[cur].mlen; + if (opt[cur].off > ZSTD_REP_MOVE_OPT) { + opt[cur].rep[2] = opt[cur-mlen].rep[1]; + opt[cur].rep[1] = opt[cur-mlen].rep[0]; + opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT; + } else { + opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2]; + opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1]; + /* If opt[cur].off == ZSTD_REP_MOVE_OPT, then mlen != 1. + * offset ZSTD_REP_MOVE_OPT is used for the special case + * litLength == 0, where offset 0 means something special. + * mlen == 1 means the previous byte was stored as a literal, + * so they are mutually exclusive. + */ + assert(!(opt[cur].off == ZSTD_REP_MOVE_OPT && mlen == 1)); + opt[cur].rep[0] = (opt[cur].off == ZSTD_REP_MOVE_OPT) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]); + } + + best_mlen = minMatch; + { U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1); + for (i=(opt[cur].mlen != 1); i 0) && (repCur < (S32)(inr-prefixStart)) + && (ZSTD_readMINMATCH(inr, minMatch) == ZSTD_readMINMATCH(inr - repCur, minMatch))) { + mlen = (U32)ZSTD_count(inr+minMatch, inr+minMatch - repCur, iend) + minMatch; + + if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { + best_mlen = mlen; best_off = i; last_pos = cur + 1; + goto _storeSequence; + } + + best_off = i - (opt[cur].mlen != 1); + if (mlen > best_mlen) best_mlen = mlen; + + do { + if (opt[cur].mlen == 1) { + litlen = opt[cur].litlen; + if (cur > litlen) { + price = opt[cur - litlen].price + ZSTD_getPrice(optStatePtr, litlen, inr-litlen, best_off, mlen - MINMATCH, ultra); + } else + price = ZSTD_getPrice(optStatePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra); + } else { + litlen = 0; + price = opt[cur].price + ZSTD_getPrice(optStatePtr, 0, NULL, best_off, mlen - MINMATCH, ultra); + } + + if (cur + mlen > last_pos || price <= opt[cur + mlen].price) + SET_PRICE(cur + mlen, mlen, i, litlen, price); + mlen--; + } while (mlen >= minMatch); + } } } + + match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, inr, iend, maxSearches, mls, matches, best_mlen); + + if (match_num > 0 && (matches[match_num-1].len > sufficient_len || cur + matches[match_num-1].len >= ZSTD_OPT_NUM)) { + best_mlen = matches[match_num-1].len; + best_off = matches[match_num-1].off; + last_pos = cur + 1; + goto _storeSequence; + } + + /* set prices using matches at position = cur */ + for (u = 0; u < match_num; u++) { + mlen = (u>0) ? matches[u-1].len+1 : best_mlen; + best_mlen = matches[u].len; + + while (mlen <= best_mlen) { + if (opt[cur].mlen == 1) { + litlen = opt[cur].litlen; + if (cur > litlen) + price = opt[cur - litlen].price + ZSTD_getPrice(optStatePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH, ultra); + else + price = ZSTD_getPrice(optStatePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra); + } else { + litlen = 0; + price = opt[cur].price + ZSTD_getPrice(optStatePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH, ultra); + } + + if (cur + mlen > last_pos || (price < opt[cur + mlen].price)) + SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price); + + mlen++; + } } } + + best_mlen = opt[last_pos].mlen; + best_off = opt[last_pos].off; + cur = last_pos - best_mlen; + + /* store sequence */ +_storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ + opt[0].mlen = 1; + + while (1) { + mlen = opt[cur].mlen; + offset = opt[cur].off; + opt[cur].mlen = best_mlen; + opt[cur].off = best_off; + best_mlen = mlen; + best_off = offset; + if (mlen > cur) break; + cur -= mlen; + } + + for (u = 0; u <= last_pos;) { + u += opt[u].mlen; + } + + for (cur=0; cur < last_pos; ) { + mlen = opt[cur].mlen; + if (mlen == 1) { ip++; cur++; continue; } + offset = opt[cur].off; + cur += mlen; + litLength = (U32)(ip - anchor); + + if (offset > ZSTD_REP_MOVE_OPT) { + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = offset - ZSTD_REP_MOVE_OPT; + offset--; + } else { + if (offset != 0) { + best_off = (offset==ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : (rep[offset]); + if (offset != 1) rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = best_off; + } + if (litLength==0) offset--; + } + + ZSTD_updatePrice(optStatePtr, litLength, anchor, offset, mlen-MINMATCH); + ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); + anchor = ip = ip + mlen; + } } /* for (cur=0; cur < last_pos; ) */ + + /* Save reps for next block */ + { int i; for (i=0; irepToConfirm[i] = rep[i]; } + + /* Return the last literals size */ + return iend - anchor; +} + + +size_t ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 0); +} + +size_t ZSTD_compressBlock_btultra(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 1); +} + + +FORCE_INLINE_TEMPLATE +size_t ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, + const void* src, size_t srcSize, const int ultra) +{ + seqStore_t* seqStorePtr = &(ctx->seqStore); + optState_t* optStatePtr = &(ctx->optState); + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ctx->base; + const U32 lowestIndex = ctx->lowLimit; + const U32 dictLimit = ctx->dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictBase = ctx->dictBase; + const BYTE* const dictEnd = dictBase + dictLimit; + + const U32 maxSearches = 1U << ctx->appliedParams.cParams.searchLog; + const U32 sufficient_len = ctx->appliedParams.cParams.targetLength; + const U32 mls = ctx->appliedParams.cParams.searchLength; + const U32 minMatch = (ctx->appliedParams.cParams.searchLength == 3) ? 3 : 4; + + ZSTD_optimal_t* opt = optStatePtr->priceTable; + ZSTD_match_t* matches = optStatePtr->matchTable; + const BYTE* inr; + + /* init */ + U32 offset, rep[ZSTD_REP_NUM]; + { U32 i; for (i=0; irep[i]; } + + ctx->nextToUpdate3 = ctx->nextToUpdate; + ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize); + ip += (ip==prefixStart); + + /* Match Loop */ + while (ip < ilimit) { + U32 cur, match_num, last_pos, litlen, price; + U32 u, mlen, best_mlen, best_off, litLength; + U32 current = (U32)(ip-base); + memset(opt, 0, sizeof(ZSTD_optimal_t)); + last_pos = 0; + opt[0].litlen = (U32)(ip - anchor); + + /* check repCode */ + { U32 i, last_i = ZSTD_REP_CHECK + (ip==anchor); + for (i = (ip==anchor); i 0 && repCur <= (S32)current) + && (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */ + && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { + /* repcode detected we should take it */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + mlen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; + + if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { + best_mlen = mlen; best_off = i; cur = 0; last_pos = 1; + goto _storeSequence; + } + + best_off = i - (ip==anchor); + litlen = opt[0].litlen; + do { + price = ZSTD_getPrice(optStatePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra); + if (mlen > last_pos || price < opt[mlen].price) + SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */ + mlen--; + } while (mlen >= minMatch); + } } } + + match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, ip, iend, maxSearches, mls, matches, minMatch); /* first search (depth 0) */ + + if (!last_pos && !match_num) { ip++; continue; } + + { U32 i; for (i=0; i sufficient_len || matches[match_num-1].len >= ZSTD_OPT_NUM)) { + best_mlen = matches[match_num-1].len; + best_off = matches[match_num-1].off; + cur = 0; + last_pos = 1; + goto _storeSequence; + } + + best_mlen = (last_pos) ? last_pos : minMatch; + + /* set prices using matches at position = 0 */ + for (u = 0; u < match_num; u++) { + mlen = (u>0) ? matches[u-1].len+1 : best_mlen; + best_mlen = matches[u].len; + litlen = opt[0].litlen; + while (mlen <= best_mlen) { + price = ZSTD_getPrice(optStatePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra); + if (mlen > last_pos || price < opt[mlen].price) + SET_PRICE(mlen, mlen, matches[u].off, litlen, price); + mlen++; + } } + + if (last_pos < minMatch) { + ip++; continue; + } + + /* check further positions */ + for (cur = 1; cur <= last_pos; cur++) { + inr = ip + cur; + + if (opt[cur-1].mlen == 1) { + litlen = opt[cur-1].litlen + 1; + if (cur > litlen) { + price = opt[cur - litlen].price + ZSTD_getLiteralPrice(optStatePtr, litlen, inr-litlen); + } else + price = ZSTD_getLiteralPrice(optStatePtr, litlen, anchor); + } else { + litlen = 1; + price = opt[cur - 1].price + ZSTD_getLiteralPrice(optStatePtr, litlen, inr-1); + } + + if (cur > last_pos || price <= opt[cur].price) + SET_PRICE(cur, 1, 0, litlen, price); + + if (cur == last_pos) break; + + if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */ + continue; + + mlen = opt[cur].mlen; + if (opt[cur].off > ZSTD_REP_MOVE_OPT) { + opt[cur].rep[2] = opt[cur-mlen].rep[1]; + opt[cur].rep[1] = opt[cur-mlen].rep[0]; + opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT; + } else { + opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2]; + opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1]; + assert(!(opt[cur].off == ZSTD_REP_MOVE_OPT && mlen == 1)); + opt[cur].rep[0] = (opt[cur].off == ZSTD_REP_MOVE_OPT) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]); + } + + best_mlen = minMatch; + { U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1); + for (i = (mlen != 1); i 0 && repCur <= (S32)(current+cur)) + && (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */ + && (ZSTD_readMINMATCH(inr, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { + /* repcode detected */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + mlen = (U32)ZSTD_count_2segments(inr+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; + + if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { + best_mlen = mlen; best_off = i; last_pos = cur + 1; + goto _storeSequence; + } + + best_off = i - (opt[cur].mlen != 1); + if (mlen > best_mlen) best_mlen = mlen; + + do { + if (opt[cur].mlen == 1) { + litlen = opt[cur].litlen; + if (cur > litlen) { + price = opt[cur - litlen].price + ZSTD_getPrice(optStatePtr, litlen, inr-litlen, best_off, mlen - MINMATCH, ultra); + } else + price = ZSTD_getPrice(optStatePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra); + } else { + litlen = 0; + price = opt[cur].price + ZSTD_getPrice(optStatePtr, 0, NULL, best_off, mlen - MINMATCH, ultra); + } + + if (cur + mlen > last_pos || price <= opt[cur + mlen].price) + SET_PRICE(cur + mlen, mlen, i, litlen, price); + mlen--; + } while (mlen >= minMatch); + } } } + + match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches, minMatch); + + if (match_num > 0 && (matches[match_num-1].len > sufficient_len || cur + matches[match_num-1].len >= ZSTD_OPT_NUM)) { + best_mlen = matches[match_num-1].len; + best_off = matches[match_num-1].off; + last_pos = cur + 1; + goto _storeSequence; + } + + /* set prices using matches at position = cur */ + for (u = 0; u < match_num; u++) { + mlen = (u>0) ? matches[u-1].len+1 : best_mlen; + best_mlen = matches[u].len; + + while (mlen <= best_mlen) { + if (opt[cur].mlen == 1) { + litlen = opt[cur].litlen; + if (cur > litlen) + price = opt[cur - litlen].price + ZSTD_getPrice(optStatePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH, ultra); + else + price = ZSTD_getPrice(optStatePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra); + } else { + litlen = 0; + price = opt[cur].price + ZSTD_getPrice(optStatePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH, ultra); + } + + if (cur + mlen > last_pos || (price < opt[cur + mlen].price)) + SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price); + + mlen++; + } } } /* for (cur = 1; cur <= last_pos; cur++) */ + + best_mlen = opt[last_pos].mlen; + best_off = opt[last_pos].off; + cur = last_pos - best_mlen; + + /* store sequence */ +_storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ + opt[0].mlen = 1; + + while (1) { + mlen = opt[cur].mlen; + offset = opt[cur].off; + opt[cur].mlen = best_mlen; + opt[cur].off = best_off; + best_mlen = mlen; + best_off = offset; + if (mlen > cur) break; + cur -= mlen; + } + + for (u = 0; u <= last_pos; ) { + u += opt[u].mlen; + } + + for (cur=0; cur < last_pos; ) { + mlen = opt[cur].mlen; + if (mlen == 1) { ip++; cur++; continue; } + offset = opt[cur].off; + cur += mlen; + litLength = (U32)(ip - anchor); + + if (offset > ZSTD_REP_MOVE_OPT) { + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = offset - ZSTD_REP_MOVE_OPT; + offset--; + } else { + if (offset != 0) { + best_off = (offset==ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : (rep[offset]); + if (offset != 1) rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = best_off; + } + + if (litLength==0) offset--; + } + + ZSTD_updatePrice(optStatePtr, litLength, anchor, offset, mlen-MINMATCH); + ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); + anchor = ip = ip + mlen; + } } /* for (cur=0; cur < last_pos; ) */ + + /* Save reps for next block */ + { int i; for (i=0; irepToConfirm[i] = rep[i]; } + + /* Return the last literals size */ + return iend - anchor; +} + + +size_t ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 0); +} + +size_t ZSTD_compressBlock_btultra_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 1); +} diff --git a/src/borg/algorithms/zstd/lib/compress/zstd_opt.h b/src/borg/algorithms/zstd/lib/compress/zstd_opt.h new file mode 100644 index 0000000000..816a1fabbf --- /dev/null +++ b/src/borg/algorithms/zstd/lib/compress/zstd_opt.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_OPT_H +#define ZSTD_OPT_H + +#include "zstd_compress.h" + +#if defined (__cplusplus) +extern "C" { +#endif + +size_t ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t srcSize); +size_t ZSTD_compressBlock_btultra(ZSTD_CCtx* ctx, const void* src, size_t srcSize); + +size_t ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize); +size_t ZSTD_compressBlock_btultra_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize); + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_OPT_H */ diff --git a/src/borg/algorithms/zstd/lib/compress/zstdmt_compress.c b/src/borg/algorithms/zstd/lib/compress/zstdmt_compress.c new file mode 100644 index 0000000000..7831cd3bd8 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/compress/zstdmt_compress.c @@ -0,0 +1,1099 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* ====== Tuning parameters ====== */ +#define ZSTDMT_NBTHREADS_MAX 200 +#define ZSTDMT_OVERLAPLOG_DEFAULT 6 + + +/* ====== Compiler specifics ====== */ +#if defined(_MSC_VER) +# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ +#endif + + +/* ====== Dependencies ====== */ +#include /* memcpy, memset */ +#include "pool.h" /* threadpool */ +#include "threading.h" /* mutex */ +#include "zstd_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */ +#include "zstdmt_compress.h" + + +/* ====== Debug ====== */ +#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2) + +# include +# include +# include +# define DEBUGLOGRAW(l, ...) if (l<=ZSTD_DEBUG) { fprintf(stderr, __VA_ARGS__); } + +# define DEBUG_PRINTHEX(l,p,n) { \ + unsigned debug_u; \ + for (debug_u=0; debug_u<(n); debug_u++) \ + DEBUGLOGRAW(l, "%02X ", ((const unsigned char*)(p))[debug_u]); \ + DEBUGLOGRAW(l, " \n"); \ +} + +static unsigned long long GetCurrentClockTimeMicroseconds(void) +{ + static clock_t _ticksPerSecond = 0; + if (_ticksPerSecond <= 0) _ticksPerSecond = sysconf(_SC_CLK_TCK); + + { struct tms junk; clock_t newTicks = (clock_t) times(&junk); + return ((((unsigned long long)newTicks)*(1000000))/_ticksPerSecond); } +} + +#define MUTEX_WAIT_TIME_DLEVEL 6 +#define ZSTD_PTHREAD_MUTEX_LOCK(mutex) { \ + if (ZSTD_DEBUG >= MUTEX_WAIT_TIME_DLEVEL) { \ + unsigned long long const beforeTime = GetCurrentClockTimeMicroseconds(); \ + ZSTD_pthread_mutex_lock(mutex); \ + { unsigned long long const afterTime = GetCurrentClockTimeMicroseconds(); \ + unsigned long long const elapsedTime = (afterTime-beforeTime); \ + if (elapsedTime > 1000) { /* or whatever threshold you like; I'm using 1 millisecond here */ \ + DEBUGLOG(MUTEX_WAIT_TIME_DLEVEL, "Thread took %llu microseconds to acquire mutex %s \n", \ + elapsedTime, #mutex); \ + } } \ + } else { \ + ZSTD_pthread_mutex_lock(mutex); \ + } \ +} + +#else + +# define ZSTD_PTHREAD_MUTEX_LOCK(m) ZSTD_pthread_mutex_lock(m) +# define DEBUG_PRINTHEX(l,p,n) {} + +#endif + + +/* ===== Buffer Pool ===== */ +/* a single Buffer Pool can be invoked from multiple threads in parallel */ + +typedef struct buffer_s { + void* start; + size_t size; +} buffer_t; + +static const buffer_t g_nullBuffer = { NULL, 0 }; + +typedef struct ZSTDMT_bufferPool_s { + ZSTD_pthread_mutex_t poolMutex; + size_t bufferSize; + unsigned totalBuffers; + unsigned nbBuffers; + ZSTD_customMem cMem; + buffer_t bTable[1]; /* variable size */ +} ZSTDMT_bufferPool; + +static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned nbThreads, ZSTD_customMem cMem) +{ + unsigned const maxNbBuffers = 2*nbThreads + 3; + ZSTDMT_bufferPool* const bufPool = (ZSTDMT_bufferPool*)ZSTD_calloc( + sizeof(ZSTDMT_bufferPool) + (maxNbBuffers-1) * sizeof(buffer_t), cMem); + if (bufPool==NULL) return NULL; + if (ZSTD_pthread_mutex_init(&bufPool->poolMutex, NULL)) { + ZSTD_free(bufPool, cMem); + return NULL; + } + bufPool->bufferSize = 64 KB; + bufPool->totalBuffers = maxNbBuffers; + bufPool->nbBuffers = 0; + bufPool->cMem = cMem; + return bufPool; +} + +static void ZSTDMT_freeBufferPool(ZSTDMT_bufferPool* bufPool) +{ + unsigned u; + DEBUGLOG(3, "ZSTDMT_freeBufferPool (address:%08X)", (U32)(size_t)bufPool); + if (!bufPool) return; /* compatibility with free on NULL */ + for (u=0; utotalBuffers; u++) { + DEBUGLOG(4, "free buffer %2u (address:%08X)", u, (U32)(size_t)bufPool->bTable[u].start); + ZSTD_free(bufPool->bTable[u].start, bufPool->cMem); + } + ZSTD_pthread_mutex_destroy(&bufPool->poolMutex); + ZSTD_free(bufPool, bufPool->cMem); +} + +/* only works at initialization, not during compression */ +static size_t ZSTDMT_sizeof_bufferPool(ZSTDMT_bufferPool* bufPool) +{ + size_t const poolSize = sizeof(*bufPool) + + (bufPool->totalBuffers - 1) * sizeof(buffer_t); + unsigned u; + size_t totalBufferSize = 0; + ZSTD_pthread_mutex_lock(&bufPool->poolMutex); + for (u=0; utotalBuffers; u++) + totalBufferSize += bufPool->bTable[u].size; + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + + return poolSize + totalBufferSize; +} + +static void ZSTDMT_setBufferSize(ZSTDMT_bufferPool* bufPool, size_t bSize) +{ + bufPool->bufferSize = bSize; +} + +/** ZSTDMT_getBuffer() : + * assumption : bufPool must be valid */ +static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool) +{ + size_t const bSize = bufPool->bufferSize; + DEBUGLOG(5, "ZSTDMT_getBuffer"); + ZSTD_pthread_mutex_lock(&bufPool->poolMutex); + if (bufPool->nbBuffers) { /* try to use an existing buffer */ + buffer_t const buf = bufPool->bTable[--(bufPool->nbBuffers)]; + size_t const availBufferSize = buf.size; + bufPool->bTable[bufPool->nbBuffers] = g_nullBuffer; + if ((availBufferSize >= bSize) & (availBufferSize <= 10*bSize)) { + /* large enough, but not too much */ + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + return buf; + } + /* size conditions not respected : scratch this buffer, create new one */ + DEBUGLOG(5, "existing buffer does not meet size conditions => freeing"); + ZSTD_free(buf.start, bufPool->cMem); + } + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + /* create new buffer */ + DEBUGLOG(5, "create a new buffer"); + { buffer_t buffer; + void* const start = ZSTD_malloc(bSize, bufPool->cMem); + buffer.start = start; /* note : start can be NULL if malloc fails ! */ + buffer.size = (start==NULL) ? 0 : bSize; + return buffer; + } +} + +/* store buffer for later re-use, up to pool capacity */ +static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buf) +{ + if (buf.start == NULL) return; /* compatible with release on NULL */ + DEBUGLOG(5, "ZSTDMT_releaseBuffer"); + ZSTD_pthread_mutex_lock(&bufPool->poolMutex); + if (bufPool->nbBuffers < bufPool->totalBuffers) { + bufPool->bTable[bufPool->nbBuffers++] = buf; /* stored for later use */ + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + return; + } + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + /* Reached bufferPool capacity (should not happen) */ + DEBUGLOG(5, "buffer pool capacity reached => freeing "); + ZSTD_free(buf.start, bufPool->cMem); +} + +/* Sets parameters relevant to the compression job, initializing others to + * default values. Notably, nbThreads should probably be zero. */ +static ZSTD_CCtx_params ZSTDMT_makeJobCCtxParams(ZSTD_CCtx_params const params) +{ + ZSTD_CCtx_params jobParams; + memset(&jobParams, 0, sizeof(jobParams)); + + jobParams.cParams = params.cParams; + jobParams.fParams = params.fParams; + jobParams.compressionLevel = params.compressionLevel; + + jobParams.ldmParams = params.ldmParams; + return jobParams; +} + +/* ===== CCtx Pool ===== */ +/* a single CCtx Pool can be invoked from multiple threads in parallel */ + +typedef struct { + ZSTD_pthread_mutex_t poolMutex; + unsigned totalCCtx; + unsigned availCCtx; + ZSTD_customMem cMem; + ZSTD_CCtx* cctx[1]; /* variable size */ +} ZSTDMT_CCtxPool; + +/* note : all CCtx borrowed from the pool should be released back to the pool _before_ freeing the pool */ +static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool) +{ + unsigned u; + for (u=0; utotalCCtx; u++) + ZSTD_freeCCtx(pool->cctx[u]); /* note : compatible with free on NULL */ + ZSTD_pthread_mutex_destroy(&pool->poolMutex); + ZSTD_free(pool, pool->cMem); +} + +/* ZSTDMT_createCCtxPool() : + * implies nbThreads >= 1 , checked by caller ZSTDMT_createCCtx() */ +static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbThreads, + ZSTD_customMem cMem) +{ + ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) ZSTD_calloc( + sizeof(ZSTDMT_CCtxPool) + (nbThreads-1)*sizeof(ZSTD_CCtx*), cMem); + if (!cctxPool) return NULL; + if (ZSTD_pthread_mutex_init(&cctxPool->poolMutex, NULL)) { + ZSTD_free(cctxPool, cMem); + return NULL; + } + cctxPool->cMem = cMem; + cctxPool->totalCCtx = nbThreads; + cctxPool->availCCtx = 1; /* at least one cctx for single-thread mode */ + cctxPool->cctx[0] = ZSTD_createCCtx_advanced(cMem); + if (!cctxPool->cctx[0]) { ZSTDMT_freeCCtxPool(cctxPool); return NULL; } + DEBUGLOG(3, "cctxPool created, with %u threads", nbThreads); + return cctxPool; +} + +/* only works during initialization phase, not during compression */ +static size_t ZSTDMT_sizeof_CCtxPool(ZSTDMT_CCtxPool* cctxPool) +{ + ZSTD_pthread_mutex_lock(&cctxPool->poolMutex); + { unsigned const nbThreads = cctxPool->totalCCtx; + size_t const poolSize = sizeof(*cctxPool) + + (nbThreads-1)*sizeof(ZSTD_CCtx*); + unsigned u; + size_t totalCCtxSize = 0; + for (u=0; ucctx[u]); + } + ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex); + return poolSize + totalCCtxSize; + } +} + +static ZSTD_CCtx* ZSTDMT_getCCtx(ZSTDMT_CCtxPool* cctxPool) +{ + DEBUGLOG(5, "ZSTDMT_getCCtx"); + ZSTD_pthread_mutex_lock(&cctxPool->poolMutex); + if (cctxPool->availCCtx) { + cctxPool->availCCtx--; + { ZSTD_CCtx* const cctx = cctxPool->cctx[cctxPool->availCCtx]; + ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex); + return cctx; + } } + ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex); + DEBUGLOG(5, "create one more CCtx"); + return ZSTD_createCCtx_advanced(cctxPool->cMem); /* note : can be NULL, when creation fails ! */ +} + +static void ZSTDMT_releaseCCtx(ZSTDMT_CCtxPool* pool, ZSTD_CCtx* cctx) +{ + if (cctx==NULL) return; /* compatibility with release on NULL */ + ZSTD_pthread_mutex_lock(&pool->poolMutex); + if (pool->availCCtx < pool->totalCCtx) + pool->cctx[pool->availCCtx++] = cctx; + else { + /* pool overflow : should not happen, since totalCCtx==nbThreads */ + DEBUGLOG(5, "CCtx pool overflow : free cctx"); + ZSTD_freeCCtx(cctx); + } + ZSTD_pthread_mutex_unlock(&pool->poolMutex); +} + + +/* ===== Thread worker ===== */ + +typedef struct { + buffer_t src; + const void* srcStart; + size_t dictSize; + size_t srcSize; + buffer_t dstBuff; + size_t cSize; + size_t dstFlushed; + unsigned firstChunk; + unsigned lastChunk; + unsigned jobCompleted; + unsigned jobScanned; + ZSTD_pthread_mutex_t* jobCompleted_mutex; + ZSTD_pthread_cond_t* jobCompleted_cond; + ZSTD_CCtx_params params; + const ZSTD_CDict* cdict; + ZSTDMT_CCtxPool* cctxPool; + ZSTDMT_bufferPool* bufPool; + unsigned long long fullFrameSize; +} ZSTDMT_jobDescription; + +/* ZSTDMT_compressChunk() : POOL_function type */ +void ZSTDMT_compressChunk(void* jobDescription) +{ + ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)jobDescription; + ZSTD_CCtx* cctx = ZSTDMT_getCCtx(job->cctxPool); + const void* const src = (const char*)job->srcStart + job->dictSize; + buffer_t dstBuff = job->dstBuff; + DEBUGLOG(5, "job (first:%u) (last:%u) : dictSize %u, srcSize %u", + job->firstChunk, job->lastChunk, (U32)job->dictSize, (U32)job->srcSize); + + if (cctx==NULL) { + job->cSize = ERROR(memory_allocation); + goto _endJob; + } + + if (dstBuff.start == NULL) { + dstBuff = ZSTDMT_getBuffer(job->bufPool); + if (dstBuff.start==NULL) { + job->cSize = ERROR(memory_allocation); + goto _endJob; + } + job->dstBuff = dstBuff; + } + + if (job->cdict) { /* should only happen for first segment */ + size_t const initError = ZSTD_compressBegin_usingCDict_advanced(cctx, job->cdict, job->params.fParams, job->fullFrameSize); + DEBUGLOG(5, "using CDict"); + if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; } + } else { /* srcStart points at reloaded section */ + if (!job->firstChunk) job->params.fParams.contentSizeFlag = 0; /* ensure no srcSize control */ + { ZSTD_CCtx_params jobParams = job->params; + size_t const forceWindowError = + ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstChunk); + /* Force loading dictionary in "content-only" mode (no header analysis) */ + size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, job->srcStart, job->dictSize, ZSTD_dm_rawContent, jobParams, job->fullFrameSize); + if (ZSTD_isError(initError) || ZSTD_isError(forceWindowError)) { + job->cSize = initError; + goto _endJob; + } + } } + if (!job->firstChunk) { /* flush and overwrite frame header when it's not first segment */ + size_t const hSize = ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, 0); + if (ZSTD_isError(hSize)) { job->cSize = hSize; goto _endJob; } + ZSTD_invalidateRepCodes(cctx); + } + + DEBUGLOG(5, "Compressing : "); + DEBUG_PRINTHEX(4, job->srcStart, 12); + job->cSize = (job->lastChunk) ? + ZSTD_compressEnd (cctx, dstBuff.start, dstBuff.size, src, job->srcSize) : + ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.size, src, job->srcSize); + DEBUGLOG(5, "compressed %u bytes into %u bytes (first:%u) (last:%u)", + (unsigned)job->srcSize, (unsigned)job->cSize, job->firstChunk, job->lastChunk); + DEBUGLOG(5, "dstBuff.size : %u ; => %s", (U32)dstBuff.size, ZSTD_getErrorName(job->cSize)); + +_endJob: + ZSTDMT_releaseCCtx(job->cctxPool, cctx); + ZSTDMT_releaseBuffer(job->bufPool, job->src); + job->src = g_nullBuffer; job->srcStart = NULL; + ZSTD_PTHREAD_MUTEX_LOCK(job->jobCompleted_mutex); + job->jobCompleted = 1; + job->jobScanned = 0; + ZSTD_pthread_cond_signal(job->jobCompleted_cond); + ZSTD_pthread_mutex_unlock(job->jobCompleted_mutex); +} + + +/* ------------------------------------------ */ +/* ===== Multi-threaded compression ===== */ +/* ------------------------------------------ */ + +typedef struct { + buffer_t buffer; + size_t filled; +} inBuff_t; + +struct ZSTDMT_CCtx_s { + POOL_ctx* factory; + ZSTDMT_jobDescription* jobs; + ZSTDMT_bufferPool* bufPool; + ZSTDMT_CCtxPool* cctxPool; + ZSTD_pthread_mutex_t jobCompleted_mutex; + ZSTD_pthread_cond_t jobCompleted_cond; + size_t targetSectionSize; + size_t inBuffSize; + size_t dictSize; + size_t targetDictSize; + inBuff_t inBuff; + ZSTD_CCtx_params params; + XXH64_state_t xxhState; + unsigned jobIDMask; + unsigned doneJobID; + unsigned nextJobID; + unsigned frameEnded; + unsigned allJobsCompleted; + unsigned long long frameContentSize; + ZSTD_customMem cMem; + ZSTD_CDict* cdictLocal; + const ZSTD_CDict* cdict; +}; + +static ZSTDMT_jobDescription* ZSTDMT_allocJobsTable(U32* nbJobsPtr, ZSTD_customMem cMem) +{ + U32 const nbJobsLog2 = ZSTD_highbit32(*nbJobsPtr) + 1; + U32 const nbJobs = 1 << nbJobsLog2; + *nbJobsPtr = nbJobs; + return (ZSTDMT_jobDescription*) ZSTD_calloc( + nbJobs * sizeof(ZSTDMT_jobDescription), cMem); +} + +/* Internal only */ +size_t ZSTDMT_initializeCCtxParameters(ZSTD_CCtx_params* params, unsigned nbThreads) +{ + params->nbThreads = nbThreads; + params->overlapSizeLog = ZSTDMT_OVERLAPLOG_DEFAULT; + params->jobSize = 0; + return 0; +} + +ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads, ZSTD_customMem cMem) +{ + ZSTDMT_CCtx* mtctx; + U32 nbJobs = nbThreads + 2; + DEBUGLOG(3, "ZSTDMT_createCCtx_advanced"); + + if (nbThreads < 1) return NULL; + nbThreads = MIN(nbThreads , ZSTDMT_NBTHREADS_MAX); + if ((cMem.customAlloc!=NULL) ^ (cMem.customFree!=NULL)) + /* invalid custom allocator */ + return NULL; + + mtctx = (ZSTDMT_CCtx*) ZSTD_calloc(sizeof(ZSTDMT_CCtx), cMem); + if (!mtctx) return NULL; + ZSTDMT_initializeCCtxParameters(&mtctx->params, nbThreads); + mtctx->cMem = cMem; + mtctx->allJobsCompleted = 1; + mtctx->factory = POOL_create_advanced(nbThreads, 0, cMem); + mtctx->jobs = ZSTDMT_allocJobsTable(&nbJobs, cMem); + mtctx->jobIDMask = nbJobs - 1; + mtctx->bufPool = ZSTDMT_createBufferPool(nbThreads, cMem); + mtctx->cctxPool = ZSTDMT_createCCtxPool(nbThreads, cMem); + if (!mtctx->factory | !mtctx->jobs | !mtctx->bufPool | !mtctx->cctxPool) { + ZSTDMT_freeCCtx(mtctx); + return NULL; + } + if (ZSTD_pthread_mutex_init(&mtctx->jobCompleted_mutex, NULL)) { + ZSTDMT_freeCCtx(mtctx); + return NULL; + } + if (ZSTD_pthread_cond_init(&mtctx->jobCompleted_cond, NULL)) { + ZSTDMT_freeCCtx(mtctx); + return NULL; + } + DEBUGLOG(3, "mt_cctx created, for %u threads", nbThreads); + return mtctx; +} + +ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbThreads) +{ + return ZSTDMT_createCCtx_advanced(nbThreads, ZSTD_defaultCMem); +} + +/* ZSTDMT_releaseAllJobResources() : + * note : ensure all workers are killed first ! */ +static void ZSTDMT_releaseAllJobResources(ZSTDMT_CCtx* mtctx) +{ + unsigned jobID; + DEBUGLOG(3, "ZSTDMT_releaseAllJobResources"); + for (jobID=0; jobID <= mtctx->jobIDMask; jobID++) { + DEBUGLOG(4, "job%02u: release dst address %08X", jobID, (U32)(size_t)mtctx->jobs[jobID].dstBuff.start); + ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[jobID].dstBuff); + mtctx->jobs[jobID].dstBuff = g_nullBuffer; + DEBUGLOG(4, "job%02u: release src address %08X", jobID, (U32)(size_t)mtctx->jobs[jobID].src.start); + ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[jobID].src); + mtctx->jobs[jobID].src = g_nullBuffer; + } + memset(mtctx->jobs, 0, (mtctx->jobIDMask+1)*sizeof(ZSTDMT_jobDescription)); + DEBUGLOG(4, "input: release address %08X", (U32)(size_t)mtctx->inBuff.buffer.start); + ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->inBuff.buffer); + mtctx->inBuff.buffer = g_nullBuffer; + mtctx->allJobsCompleted = 1; +} + +static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* zcs) +{ + DEBUGLOG(4, "ZSTDMT_waitForAllJobsCompleted"); + while (zcs->doneJobID < zcs->nextJobID) { + unsigned const jobID = zcs->doneJobID & zcs->jobIDMask; + ZSTD_PTHREAD_MUTEX_LOCK(&zcs->jobCompleted_mutex); + while (zcs->jobs[jobID].jobCompleted==0) { + DEBUGLOG(5, "waiting for jobCompleted signal from chunk %u", zcs->doneJobID); /* we want to block when waiting for data to flush */ + ZSTD_pthread_cond_wait(&zcs->jobCompleted_cond, &zcs->jobCompleted_mutex); + } + ZSTD_pthread_mutex_unlock(&zcs->jobCompleted_mutex); + zcs->doneJobID++; + } +} + +size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx) +{ + if (mtctx==NULL) return 0; /* compatible with free on NULL */ + POOL_free(mtctx->factory); /* stop and free worker threads */ + ZSTDMT_releaseAllJobResources(mtctx); /* release job resources into pools first */ + ZSTD_free(mtctx->jobs, mtctx->cMem); + ZSTDMT_freeBufferPool(mtctx->bufPool); + ZSTDMT_freeCCtxPool(mtctx->cctxPool); + ZSTD_freeCDict(mtctx->cdictLocal); + ZSTD_pthread_mutex_destroy(&mtctx->jobCompleted_mutex); + ZSTD_pthread_cond_destroy(&mtctx->jobCompleted_cond); + ZSTD_free(mtctx, mtctx->cMem); + return 0; +} + +size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx) +{ + if (mtctx == NULL) return 0; /* supports sizeof NULL */ + return sizeof(*mtctx) + + POOL_sizeof(mtctx->factory) + + ZSTDMT_sizeof_bufferPool(mtctx->bufPool) + + (mtctx->jobIDMask+1) * sizeof(ZSTDMT_jobDescription) + + ZSTDMT_sizeof_CCtxPool(mtctx->cctxPool) + + ZSTD_sizeof_CDict(mtctx->cdictLocal); +} + +/* Internal only */ +size_t ZSTDMT_CCtxParam_setMTCtxParameter( + ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, unsigned value) { + switch(parameter) + { + case ZSTDMT_p_sectionSize : + params->jobSize = value; + return 0; + case ZSTDMT_p_overlapSectionLog : + DEBUGLOG(4, "ZSTDMT_p_overlapSectionLog : %u", value); + params->overlapSizeLog = (value >= 9) ? 9 : value; + return 0; + default : + return ERROR(parameter_unsupported); + } +} + +size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, unsigned value) +{ + switch(parameter) + { + case ZSTDMT_p_sectionSize : + return ZSTDMT_CCtxParam_setMTCtxParameter(&mtctx->params, parameter, value); + case ZSTDMT_p_overlapSectionLog : + return ZSTDMT_CCtxParam_setMTCtxParameter(&mtctx->params, parameter, value); + default : + return ERROR(parameter_unsupported); + } +} + +/* ------------------------------------------ */ +/* ===== Multi-threaded compression ===== */ +/* ------------------------------------------ */ + +static unsigned computeNbChunks(size_t srcSize, unsigned windowLog, unsigned nbThreads) { + size_t const chunkSizeTarget = (size_t)1 << (windowLog + 2); + size_t const chunkMaxSize = chunkSizeTarget << 2; + size_t const passSizeMax = chunkMaxSize * nbThreads; + unsigned const multiplier = (unsigned)(srcSize / passSizeMax) + 1; + unsigned const nbChunksLarge = multiplier * nbThreads; + unsigned const nbChunksMax = (unsigned)(srcSize / chunkSizeTarget) + 1; + unsigned const nbChunksSmall = MIN(nbChunksMax, nbThreads); + return (multiplier>1) ? nbChunksLarge : nbChunksSmall; +} + +static size_t ZSTDMT_compress_advanced_internal( + ZSTDMT_CCtx* mtctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, + ZSTD_CCtx_params const params) +{ + ZSTD_CCtx_params const jobParams = ZSTDMT_makeJobCCtxParams(params); + unsigned const overlapRLog = (params.overlapSizeLog>9) ? 0 : 9-params.overlapSizeLog; + size_t const overlapSize = (overlapRLog>=9) ? 0 : (size_t)1 << (params.cParams.windowLog - overlapRLog); + unsigned nbChunks = computeNbChunks(srcSize, params.cParams.windowLog, params.nbThreads); + size_t const proposedChunkSize = (srcSize + (nbChunks-1)) / nbChunks; + size_t const avgChunkSize = ((proposedChunkSize & 0x1FFFF) < 0x7FFF) ? proposedChunkSize + 0xFFFF : proposedChunkSize; /* avoid too small last block */ + const char* const srcStart = (const char*)src; + size_t remainingSrcSize = srcSize; + unsigned const compressWithinDst = (dstCapacity >= ZSTD_compressBound(srcSize)) ? nbChunks : (unsigned)(dstCapacity / ZSTD_compressBound(avgChunkSize)); /* presumes avgChunkSize >= 256 KB, which should be the case */ + size_t frameStartPos = 0, dstBufferPos = 0; + XXH64_state_t xxh64; + assert(jobParams.nbThreads == 0); + assert(mtctx->cctxPool->totalCCtx == params.nbThreads); + + DEBUGLOG(4, "nbChunks : %2u (chunkSize : %u bytes) ", nbChunks, (U32)avgChunkSize); + if (nbChunks==1) { /* fallback to single-thread mode */ + ZSTD_CCtx* const cctx = mtctx->cctxPool->cctx[0]; + if (cdict) return ZSTD_compress_usingCDict_advanced(cctx, dst, dstCapacity, src, srcSize, cdict, jobParams.fParams); + return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, NULL, 0, jobParams); + } + assert(avgChunkSize >= 256 KB); /* condition for ZSTD_compressBound(A) + ZSTD_compressBound(B) <= ZSTD_compressBound(A+B), which is required for compressWithinDst */ + ZSTDMT_setBufferSize(mtctx->bufPool, ZSTD_compressBound(avgChunkSize) ); + XXH64_reset(&xxh64, 0); + + if (nbChunks > mtctx->jobIDMask+1) { /* enlarge job table */ + U32 nbJobs = nbChunks; + ZSTD_free(mtctx->jobs, mtctx->cMem); + mtctx->jobIDMask = 0; + mtctx->jobs = ZSTDMT_allocJobsTable(&nbJobs, mtctx->cMem); + if (mtctx->jobs==NULL) return ERROR(memory_allocation); + mtctx->jobIDMask = nbJobs - 1; + } + + { unsigned u; + for (u=0; ujobs[u].src = g_nullBuffer; + mtctx->jobs[u].srcStart = srcStart + frameStartPos - dictSize; + mtctx->jobs[u].dictSize = dictSize; + mtctx->jobs[u].srcSize = chunkSize; + mtctx->jobs[u].cdict = mtctx->nextJobID==0 ? cdict : NULL; + mtctx->jobs[u].fullFrameSize = srcSize; + mtctx->jobs[u].params = jobParams; + /* do not calculate checksum within sections, but write it in header for first section */ + if (u!=0) mtctx->jobs[u].params.fParams.checksumFlag = 0; + mtctx->jobs[u].dstBuff = dstBuffer; + mtctx->jobs[u].cctxPool = mtctx->cctxPool; + mtctx->jobs[u].bufPool = mtctx->bufPool; + mtctx->jobs[u].firstChunk = (u==0); + mtctx->jobs[u].lastChunk = (u==nbChunks-1); + mtctx->jobs[u].jobCompleted = 0; + mtctx->jobs[u].jobCompleted_mutex = &mtctx->jobCompleted_mutex; + mtctx->jobs[u].jobCompleted_cond = &mtctx->jobCompleted_cond; + + if (params.fParams.checksumFlag) { + XXH64_update(&xxh64, srcStart + frameStartPos, chunkSize); + } + + DEBUGLOG(5, "posting job %u (%u bytes)", u, (U32)chunkSize); + DEBUG_PRINTHEX(6, mtctx->jobs[u].srcStart, 12); + POOL_add(mtctx->factory, ZSTDMT_compressChunk, &mtctx->jobs[u]); + + frameStartPos += chunkSize; + dstBufferPos += dstBufferCapacity; + remainingSrcSize -= chunkSize; + } } + + /* collect result */ + { size_t error = 0, dstPos = 0; + unsigned chunkID; + for (chunkID=0; chunkIDjobCompleted_mutex); + while (mtctx->jobs[chunkID].jobCompleted==0) { + DEBUGLOG(5, "waiting for jobCompleted signal from chunk %u", chunkID); + ZSTD_pthread_cond_wait(&mtctx->jobCompleted_cond, &mtctx->jobCompleted_mutex); + } + ZSTD_pthread_mutex_unlock(&mtctx->jobCompleted_mutex); + DEBUGLOG(5, "ready to write chunk %u ", chunkID); + + mtctx->jobs[chunkID].srcStart = NULL; + { size_t const cSize = mtctx->jobs[chunkID].cSize; + if (ZSTD_isError(cSize)) error = cSize; + if ((!error) && (dstPos + cSize > dstCapacity)) error = ERROR(dstSize_tooSmall); + if (chunkID) { /* note : chunk 0 is written directly at dst, which is correct position */ + if (!error) + memmove((char*)dst + dstPos, mtctx->jobs[chunkID].dstBuff.start, cSize); /* may overlap when chunk compressed within dst */ + if (chunkID >= compressWithinDst) { /* chunk compressed into its own buffer, which must be released */ + DEBUGLOG(5, "releasing buffer %u>=%u", chunkID, compressWithinDst); + ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[chunkID].dstBuff); + } } + mtctx->jobs[chunkID].dstBuff = g_nullBuffer; + dstPos += cSize ; + } + } /* for (chunkID=0; chunkID dstCapacity) { + error = ERROR(dstSize_tooSmall); + } else { + DEBUGLOG(4, "writing checksum : %08X \n", checksum); + MEM_writeLE32((char*)dst + dstPos, checksum); + dstPos += 4; + } } + + if (!error) DEBUGLOG(4, "compressed size : %u ", (U32)dstPos); + return error ? error : dstPos; + } +} + +size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, + ZSTD_parameters const params, + unsigned overlapLog) +{ + ZSTD_CCtx_params cctxParams = mtctx->params; + cctxParams.cParams = params.cParams; + cctxParams.fParams = params.fParams; + cctxParams.overlapSizeLog = overlapLog; + return ZSTDMT_compress_advanced_internal(mtctx, + dst, dstCapacity, + src, srcSize, + cdict, cctxParams); +} + + +size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel) +{ + U32 const overlapLog = (compressionLevel >= ZSTD_maxCLevel()) ? 9 : ZSTDMT_OVERLAPLOG_DEFAULT; + ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, 0); + params.fParams.contentSizeFlag = 1; + return ZSTDMT_compress_advanced(mtctx, dst, dstCapacity, src, srcSize, NULL, params, overlapLog); +} + + +/* ====================================== */ +/* ======= Streaming API ======= */ +/* ====================================== */ + +size_t ZSTDMT_initCStream_internal( + ZSTDMT_CCtx* zcs, + const void* dict, size_t dictSize, ZSTD_dictMode_e dictMode, + const ZSTD_CDict* cdict, ZSTD_CCtx_params params, + unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTDMT_initCStream_internal"); + /* params are supposed to be fully validated at this point */ + assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + assert(!((dict) && (cdict))); /* either dict or cdict, not both */ + assert(zcs->cctxPool->totalCCtx == params.nbThreads); + + if (params.nbThreads==1) { + ZSTD_CCtx_params const singleThreadParams = ZSTDMT_makeJobCCtxParams(params); + DEBUGLOG(4, "single thread mode"); + assert(singleThreadParams.nbThreads == 0); + return ZSTD_initCStream_internal(zcs->cctxPool->cctx[0], + dict, dictSize, cdict, + singleThreadParams, pledgedSrcSize); + } + + if (zcs->allJobsCompleted == 0) { /* previous compression not correctly finished */ + ZSTDMT_waitForAllJobsCompleted(zcs); + ZSTDMT_releaseAllJobResources(zcs); + zcs->allJobsCompleted = 1; + } + + zcs->params = params; + zcs->frameContentSize = pledgedSrcSize; + if (dict) { + DEBUGLOG(4,"cdictLocal: %08X", (U32)(size_t)zcs->cdictLocal); + ZSTD_freeCDict(zcs->cdictLocal); + zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_dlm_byCopy, dictMode, /* note : a loadPrefix becomes an internal CDict */ + params.cParams, zcs->cMem); + zcs->cdict = zcs->cdictLocal; + if (zcs->cdictLocal == NULL) return ERROR(memory_allocation); + } else { + DEBUGLOG(4,"cdictLocal: %08X", (U32)(size_t)zcs->cdictLocal); + ZSTD_freeCDict(zcs->cdictLocal); + zcs->cdictLocal = NULL; + zcs->cdict = cdict; + } + + zcs->targetDictSize = (params.overlapSizeLog==0) ? 0 : (size_t)1 << (params.cParams.windowLog - (9 - params.overlapSizeLog)); + DEBUGLOG(4, "overlapLog : %u ", params.overlapSizeLog); + DEBUGLOG(4, "overlap Size : %u KB", (U32)(zcs->targetDictSize>>10)); + zcs->targetSectionSize = params.jobSize ? params.jobSize : (size_t)1 << (params.cParams.windowLog + 2); + zcs->targetSectionSize = MAX(ZSTDMT_SECTION_SIZE_MIN, zcs->targetSectionSize); + zcs->targetSectionSize = MAX(zcs->targetDictSize, zcs->targetSectionSize); + DEBUGLOG(4, "Section Size : %u KB", (U32)(zcs->targetSectionSize>>10)); + zcs->inBuffSize = zcs->targetDictSize + zcs->targetSectionSize; + ZSTDMT_setBufferSize(zcs->bufPool, MAX(zcs->inBuffSize, ZSTD_compressBound(zcs->targetSectionSize)) ); + zcs->inBuff.buffer = g_nullBuffer; + zcs->dictSize = 0; + zcs->doneJobID = 0; + zcs->nextJobID = 0; + zcs->frameEnded = 0; + zcs->allJobsCompleted = 0; + if (params.fParams.checksumFlag) XXH64_reset(&zcs->xxhState, 0); + return 0; +} + +size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx, + const void* dict, size_t dictSize, + ZSTD_parameters params, + unsigned long long pledgedSrcSize) +{ + ZSTD_CCtx_params cctxParams = mtctx->params; + DEBUGLOG(5, "ZSTDMT_initCStream_advanced"); + cctxParams.cParams = params.cParams; + cctxParams.fParams = params.fParams; + return ZSTDMT_initCStream_internal(mtctx, dict, dictSize, ZSTD_dm_auto, NULL, + cctxParams, pledgedSrcSize); +} + +size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams, + unsigned long long pledgedSrcSize) +{ + ZSTD_CCtx_params cctxParams = mtctx->params; + cctxParams.cParams = ZSTD_getCParamsFromCDict(cdict); + cctxParams.fParams = fParams; + if (cdict==NULL) return ERROR(dictionary_wrong); /* method incompatible with NULL cdict */ + return ZSTDMT_initCStream_internal(mtctx, NULL, 0 /*dictSize*/, ZSTD_dm_auto, cdict, + cctxParams, pledgedSrcSize); +} + + +/* ZSTDMT_resetCStream() : + * pledgedSrcSize is optional and can be zero == unknown */ +size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* zcs, unsigned long long pledgedSrcSize) +{ + if (zcs->params.nbThreads==1) + return ZSTD_resetCStream(zcs->cctxPool->cctx[0], pledgedSrcSize); + return ZSTDMT_initCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, 0, zcs->params, + pledgedSrcSize); +} + +size_t ZSTDMT_initCStream(ZSTDMT_CCtx* zcs, int compressionLevel) { + ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, 0); + ZSTD_CCtx_params cctxParams = zcs->params; + cctxParams.cParams = params.cParams; + cctxParams.fParams = params.fParams; + return ZSTDMT_initCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, NULL, cctxParams, 0); +} + + +static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* zcs, size_t srcSize, unsigned endFrame) +{ + unsigned const jobID = zcs->nextJobID & zcs->jobIDMask; + + DEBUGLOG(4, "preparing job %u to compress %u bytes with %u preload ", + zcs->nextJobID, (U32)srcSize, (U32)zcs->dictSize); + zcs->jobs[jobID].src = zcs->inBuff.buffer; + zcs->jobs[jobID].srcStart = zcs->inBuff.buffer.start; + zcs->jobs[jobID].srcSize = srcSize; + zcs->jobs[jobID].dictSize = zcs->dictSize; + assert(zcs->inBuff.filled >= srcSize + zcs->dictSize); + zcs->jobs[jobID].params = zcs->params; + /* do not calculate checksum within sections, but write it in header for first section */ + if (zcs->nextJobID) zcs->jobs[jobID].params.fParams.checksumFlag = 0; + zcs->jobs[jobID].cdict = zcs->nextJobID==0 ? zcs->cdict : NULL; + zcs->jobs[jobID].fullFrameSize = zcs->frameContentSize; + zcs->jobs[jobID].dstBuff = g_nullBuffer; + zcs->jobs[jobID].cctxPool = zcs->cctxPool; + zcs->jobs[jobID].bufPool = zcs->bufPool; + zcs->jobs[jobID].firstChunk = (zcs->nextJobID==0); + zcs->jobs[jobID].lastChunk = endFrame; + zcs->jobs[jobID].jobCompleted = 0; + zcs->jobs[jobID].dstFlushed = 0; + zcs->jobs[jobID].jobCompleted_mutex = &zcs->jobCompleted_mutex; + zcs->jobs[jobID].jobCompleted_cond = &zcs->jobCompleted_cond; + + if (zcs->params.fParams.checksumFlag) + XXH64_update(&zcs->xxhState, (const char*)zcs->inBuff.buffer.start + zcs->dictSize, srcSize); + + /* get a new buffer for next input */ + if (!endFrame) { + size_t const newDictSize = MIN(srcSize + zcs->dictSize, zcs->targetDictSize); + zcs->inBuff.buffer = ZSTDMT_getBuffer(zcs->bufPool); + if (zcs->inBuff.buffer.start == NULL) { /* not enough memory to allocate next input buffer */ + zcs->jobs[jobID].jobCompleted = 1; + zcs->nextJobID++; + ZSTDMT_waitForAllJobsCompleted(zcs); + ZSTDMT_releaseAllJobResources(zcs); + return ERROR(memory_allocation); + } + zcs->inBuff.filled -= srcSize + zcs->dictSize - newDictSize; + memmove(zcs->inBuff.buffer.start, + (const char*)zcs->jobs[jobID].srcStart + zcs->dictSize + srcSize - newDictSize, + zcs->inBuff.filled); + zcs->dictSize = newDictSize; + } else { /* if (endFrame==1) */ + zcs->inBuff.buffer = g_nullBuffer; + zcs->inBuff.filled = 0; + zcs->dictSize = 0; + zcs->frameEnded = 1; + if (zcs->nextJobID == 0) { + /* single chunk exception : checksum is calculated directly within worker thread */ + zcs->params.fParams.checksumFlag = 0; + } } + + DEBUGLOG(4, "posting job %u : %u bytes (end:%u) (note : doneJob = %u=>%u)", + zcs->nextJobID, + (U32)zcs->jobs[jobID].srcSize, + zcs->jobs[jobID].lastChunk, + zcs->doneJobID, + zcs->doneJobID & zcs->jobIDMask); + POOL_add(zcs->factory, ZSTDMT_compressChunk, &zcs->jobs[jobID]); /* this call is blocking when thread worker pool is exhausted */ + zcs->nextJobID++; + return 0; +} + + +/* ZSTDMT_flushNextJob() : + * output : will be updated with amount of data flushed . + * blockToFlush : if >0, the function will block and wait if there is no data available to flush . + * @return : amount of data remaining within internal buffer, 1 if unknown but > 0, 0 if no more, or an error code */ +static size_t ZSTDMT_flushNextJob(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsigned blockToFlush) +{ + unsigned const wJobID = zcs->doneJobID & zcs->jobIDMask; + if (zcs->doneJobID == zcs->nextJobID) return 0; /* all flushed ! */ + ZSTD_PTHREAD_MUTEX_LOCK(&zcs->jobCompleted_mutex); + while (zcs->jobs[wJobID].jobCompleted==0) { + DEBUGLOG(5, "waiting for jobCompleted signal from job %u", zcs->doneJobID); + if (!blockToFlush) { ZSTD_pthread_mutex_unlock(&zcs->jobCompleted_mutex); return 0; } /* nothing ready to be flushed => skip */ + ZSTD_pthread_cond_wait(&zcs->jobCompleted_cond, &zcs->jobCompleted_mutex); /* block when nothing available to flush */ + } + ZSTD_pthread_mutex_unlock(&zcs->jobCompleted_mutex); + /* compression job completed : output can be flushed */ + { ZSTDMT_jobDescription job = zcs->jobs[wJobID]; + if (!job.jobScanned) { + if (ZSTD_isError(job.cSize)) { + DEBUGLOG(5, "compression error detected "); + ZSTDMT_waitForAllJobsCompleted(zcs); + ZSTDMT_releaseAllJobResources(zcs); + return job.cSize; + } + DEBUGLOG(5, "zcs->params.fParams.checksumFlag : %u ", zcs->params.fParams.checksumFlag); + if (zcs->params.fParams.checksumFlag) { + if (zcs->frameEnded && (zcs->doneJobID+1 == zcs->nextJobID)) { /* write checksum at end of last section */ + U32 const checksum = (U32)XXH64_digest(&zcs->xxhState); + DEBUGLOG(5, "writing checksum : %08X \n", checksum); + MEM_writeLE32((char*)job.dstBuff.start + job.cSize, checksum); + job.cSize += 4; + zcs->jobs[wJobID].cSize += 4; + } } + zcs->jobs[wJobID].jobScanned = 1; + } + { size_t const toWrite = MIN(job.cSize - job.dstFlushed, output->size - output->pos); + DEBUGLOG(5, "Flushing %u bytes from job %u ", (U32)toWrite, zcs->doneJobID); + memcpy((char*)output->dst + output->pos, (const char*)job.dstBuff.start + job.dstFlushed, toWrite); + output->pos += toWrite; + job.dstFlushed += toWrite; + } + if (job.dstFlushed == job.cSize) { /* output buffer fully flushed => move to next one */ + ZSTDMT_releaseBuffer(zcs->bufPool, job.dstBuff); + zcs->jobs[wJobID].dstBuff = g_nullBuffer; + zcs->jobs[wJobID].jobCompleted = 0; + zcs->doneJobID++; + } else { + zcs->jobs[wJobID].dstFlushed = job.dstFlushed; + } + /* return value : how many bytes left in buffer ; fake it to 1 if unknown but >0 */ + if (job.cSize > job.dstFlushed) return (job.cSize - job.dstFlushed); + if (zcs->doneJobID < zcs->nextJobID) return 1; /* still some buffer to flush */ + zcs->allJobsCompleted = zcs->frameEnded; /* frame completed and entirely flushed */ + return 0; /* everything flushed */ +} } + + +/** ZSTDMT_compressStream_generic() : + * internal use only - exposed to be invoked from zstd_compress.c + * assumption : output and input are valid (pos <= size) + * @return : minimum amount of data remaining to flush, 0 if none */ +size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp) +{ + size_t const newJobThreshold = mtctx->dictSize + mtctx->targetSectionSize; + unsigned forwardInputProgress = 0; + assert(output->pos <= output->size); + assert(input->pos <= input->size); + if ((mtctx->frameEnded) && (endOp==ZSTD_e_continue)) { + /* current frame being ended. Only flush/end are allowed */ + return ERROR(stage_wrong); + } + if (mtctx->params.nbThreads==1) { /* delegate to single-thread (synchronous) */ + return ZSTD_compressStream_generic(mtctx->cctxPool->cctx[0], output, input, endOp); + } + + /* single-pass shortcut (note : synchronous-mode) */ + if ( (mtctx->nextJobID == 0) /* just started */ + && (mtctx->inBuff.filled == 0) /* nothing buffered */ + && (endOp == ZSTD_e_end) /* end order */ + && (output->size - output->pos >= ZSTD_compressBound(input->size - input->pos)) ) { /* enough room */ + size_t const cSize = ZSTDMT_compress_advanced_internal(mtctx, + (char*)output->dst + output->pos, output->size - output->pos, + (const char*)input->src + input->pos, input->size - input->pos, + mtctx->cdict, mtctx->params); + if (ZSTD_isError(cSize)) return cSize; + input->pos = input->size; + output->pos += cSize; + ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->inBuff.buffer); /* was allocated in initStream */ + mtctx->allJobsCompleted = 1; + mtctx->frameEnded = 1; + return 0; + } + + /* fill input buffer */ + if (input->size > input->pos) { /* support NULL input */ + if (mtctx->inBuff.buffer.start == NULL) { + mtctx->inBuff.buffer = ZSTDMT_getBuffer(mtctx->bufPool); /* note : may fail, in which case, no forward input progress */ + mtctx->inBuff.filled = 0; + } + if (mtctx->inBuff.buffer.start) { + size_t const toLoad = MIN(input->size - input->pos, mtctx->inBuffSize - mtctx->inBuff.filled); + DEBUGLOG(5, "inBuff:%08X; inBuffSize=%u; ToCopy=%u", (U32)(size_t)mtctx->inBuff.buffer.start, (U32)mtctx->inBuffSize, (U32)toLoad); + memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, (const char*)input->src + input->pos, toLoad); + input->pos += toLoad; + mtctx->inBuff.filled += toLoad; + forwardInputProgress = toLoad>0; + } } + + if ( (mtctx->inBuff.filled >= newJobThreshold) /* filled enough : let's compress */ + && (mtctx->nextJobID <= mtctx->doneJobID + mtctx->jobIDMask) ) { /* avoid overwriting job round buffer */ + CHECK_F( ZSTDMT_createCompressionJob(mtctx, mtctx->targetSectionSize, 0 /* endFrame */) ); + } + + /* check for potential compressed data ready to be flushed */ + CHECK_F( ZSTDMT_flushNextJob(mtctx, output, !forwardInputProgress /* blockToFlush */) ); /* block if there was no forward input progress */ + + if (input->pos < input->size) /* input not consumed : do not flush yet */ + endOp = ZSTD_e_continue; + + switch(endOp) + { + case ZSTD_e_flush: + return ZSTDMT_flushStream(mtctx, output); + case ZSTD_e_end: + return ZSTDMT_endStream(mtctx, output); + case ZSTD_e_continue: + return 1; + default: + return ERROR(GENERIC); /* invalid endDirective */ + } +} + + +size_t ZSTDMT_compressStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input) +{ + CHECK_F( ZSTDMT_compressStream_generic(zcs, output, input, ZSTD_e_continue) ); + + /* recommended next input size : fill current input buffer */ + return zcs->inBuffSize - zcs->inBuff.filled; /* note : could be zero when input buffer is fully filled and no more availability to create new job */ +} + + +static size_t ZSTDMT_flushStream_internal(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output, unsigned endFrame) +{ + size_t const srcSize = zcs->inBuff.filled - zcs->dictSize; + + if ( ((srcSize > 0) || (endFrame && !zcs->frameEnded)) + && (zcs->nextJobID <= zcs->doneJobID + zcs->jobIDMask) ) { + CHECK_F( ZSTDMT_createCompressionJob(zcs, srcSize, endFrame) ); + } + + /* check if there is any data available to flush */ + return ZSTDMT_flushNextJob(zcs, output, 1 /* blockToFlush */); +} + + +size_t ZSTDMT_flushStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output) +{ + DEBUGLOG(5, "ZSTDMT_flushStream"); + if (zcs->params.nbThreads==1) + return ZSTD_flushStream(zcs->cctxPool->cctx[0], output); + return ZSTDMT_flushStream_internal(zcs, output, 0 /* endFrame */); +} + +size_t ZSTDMT_endStream(ZSTDMT_CCtx* zcs, ZSTD_outBuffer* output) +{ + DEBUGLOG(4, "ZSTDMT_endStream"); + if (zcs->params.nbThreads==1) + return ZSTD_endStream(zcs->cctxPool->cctx[0], output); + return ZSTDMT_flushStream_internal(zcs, output, 1 /* endFrame */); +} diff --git a/src/borg/algorithms/zstd/lib/compress/zstdmt_compress.h b/src/borg/algorithms/zstd/lib/compress/zstdmt_compress.h new file mode 100644 index 0000000000..8c59c684f1 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/compress/zstdmt_compress.h @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + #ifndef ZSTDMT_COMPRESS_H + #define ZSTDMT_COMPRESS_H + + #if defined (__cplusplus) + extern "C" { + #endif + + +/* Note : This is an internal API. + * Some methods are still exposed (ZSTDLIB_API), + * because it used to be the only way to invoke MT compression. + * Now, it's recommended to use ZSTD_compress_generic() instead. + * These methods will stop being exposed in a future version */ + +/* === Dependencies === */ +#include /* size_t */ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */ +#include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */ + + +/* === Memory management === */ +typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx; +ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbThreads); +ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbThreads, + ZSTD_customMem cMem); +ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx); + +ZSTDLIB_API size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx); + + +/* === Simple buffer-to-butter one-pass function === */ + +ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel); + + + +/* === Streaming functions === */ + +ZSTDLIB_API size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel); +ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */ + +ZSTDLIB_API size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input); + +ZSTDLIB_API size_t ZSTDMT_flushStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output); /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */ +ZSTDLIB_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output); /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */ + + +/* === Advanced functions and parameters === */ + +#ifndef ZSTDMT_SECTION_SIZE_MIN +# define ZSTDMT_SECTION_SIZE_MIN (1U << 20) /* 1 MB - Minimum size of each compression job */ +#endif + +ZSTDLIB_API size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, + ZSTD_parameters const params, + unsigned overlapLog); + +ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx, + const void* dict, size_t dictSize, /* dict can be released after init, a local copy is preserved within zcs */ + ZSTD_parameters params, + unsigned long long pledgedSrcSize); /* pledgedSrcSize is optional and can be zero == unknown */ + +ZSTDLIB_API size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fparams, + unsigned long long pledgedSrcSize); /* note : zero means empty */ + +/* ZSTDMT_parameter : + * List of parameters that can be set using ZSTDMT_setMTCtxParameter() */ +typedef enum { + ZSTDMT_p_sectionSize, /* size of input "section". Each section is compressed in parallel. 0 means default, which is dynamically determined within compression functions */ + ZSTDMT_p_overlapSectionLog /* Log of overlapped section; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window */ +} ZSTDMT_parameter; + +/* ZSTDMT_setMTCtxParameter() : + * allow setting individual parameters, one at a time, among a list of enums defined in ZSTDMT_parameter. + * The function must be called typically after ZSTD_createCCtx(). + * Parameters not explicitly reset by ZSTDMT_init*() remain the same in consecutive compression sessions. + * @return : 0, or an error code (which can be tested using ZSTD_isError()) */ +ZSTDLIB_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, unsigned value); + + +/*! ZSTDMT_compressStream_generic() : + * Combines ZSTDMT_compressStream() with ZSTDMT_flushStream() or ZSTDMT_endStream() + * depending on flush directive. + * @return : minimum amount of data still to be flushed + * 0 if fully flushed + * or an error code */ +ZSTDLIB_API size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp); + + +/* === Private definitions; never ever use directly === */ + +size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, unsigned value); + +size_t ZSTDMT_initializeCCtxParameters(ZSTD_CCtx_params* params, unsigned nbThreads); + +/*! ZSTDMT_initCStream_internal() : + * Private use only. Init streaming operation. + * expects params to be valid. + * must receive dict, or cdict, or none, but not both. + * @return : 0, or an error code */ +size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs, + const void* dict, size_t dictSize, ZSTD_dictMode_e dictMode, + const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, unsigned long long pledgedSrcSize); + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTDMT_COMPRESS_H */ diff --git a/src/borg/algorithms/zstd/lib/decompress/huf_decompress.c b/src/borg/algorithms/zstd/lib/decompress/huf_decompress.c new file mode 100644 index 0000000000..79ded96bf6 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/decompress/huf_decompress.c @@ -0,0 +1,996 @@ +/* ****************************************************************** + Huffman decoder, part of New Generation Entropy library + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + +/* ************************************************************** +* Dependencies +****************************************************************/ +#include /* memcpy, memset */ +#include "bitstream.h" /* BIT_* */ +#include "compiler.h" +#include "fse.h" /* header compression */ +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" +#include "error_private.h" + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define HUF_isError ERR_isError +#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + + +/* ************************************************************** +* Byte alignment for workSpace management +****************************************************************/ +#define HUF_ALIGN(x, a) HUF_ALIGN_MASK((x), (a) - 1) +#define HUF_ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask)) + +/*-***************************/ +/* generic DTableDesc */ +/*-***************************/ + +typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc; + +static DTableDesc HUF_getDTableDesc(const HUF_DTable* table) +{ + DTableDesc dtd; + memcpy(&dtd, table, sizeof(dtd)); + return dtd; +} + + +/*-***************************/ +/* single-symbol decoding */ +/*-***************************/ + +typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX2; /* single-symbol decoding */ + +size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize) +{ + U32 tableLog = 0; + U32 nbSymbols = 0; + size_t iSize; + void* const dtPtr = DTable + 1; + HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; + + U32* rankVal; + BYTE* huffWeight; + size_t spaceUsed32 = 0; + + rankVal = (U32 *)workSpace + spaceUsed32; + spaceUsed32 += HUF_TABLELOG_ABSOLUTEMAX + 1; + huffWeight = (BYTE *)((U32 *)workSpace + spaceUsed32); + spaceUsed32 += HUF_ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2; + + if ((spaceUsed32 << 2) > wkspSize) + return ERROR(tableLog_tooLarge); + workSpace = (U32 *)workSpace + spaceUsed32; + wkspSize -= (spaceUsed32 << 2); + + HUF_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); + /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize); + if (HUF_isError(iSize)) return iSize; + + /* Table header */ + { DTableDesc dtd = HUF_getDTableDesc(DTable); + if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */ + dtd.tableType = 0; + dtd.tableLog = (BYTE)tableLog; + memcpy(DTable, &dtd, sizeof(dtd)); + } + + /* Calculate starting value for each rank */ + { U32 n, nextRankStart = 0; + for (n=1; n> 1; + U32 u; + HUF_DEltX2 D; + D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w); + for (u = rankVal[w]; u < rankVal[w] + length; u++) + dt[u] = D; + rankVal[w] += length; + } } + + return iSize; +} + +size_t HUF_readDTableX2(HUF_DTable* DTable, const void* src, size_t srcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_readDTableX2_wksp(DTable, src, srcSize, + workSpace, sizeof(workSpace)); +} + + +static BYTE HUF_decodeSymbolX2(BIT_DStream_t* Dstream, const HUF_DEltX2* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ + BYTE const c = dt[val].byte; + BIT_skipBits(Dstream, dt[val].nbBits); + return c; +} + +#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ + *ptr++ = HUF_decodeSymbolX2(DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) + +#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ + if (MEM_64bits()) \ + HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) + +HINT_INLINE size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX2* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 4 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-4)) { + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_1(p, bitDPtr); + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + } + + /* closer to the end */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd)) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + + /* no more data to retrieve from bitstream, hence no need to reload */ + while (p < pEnd) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + + return pEnd-pStart; +} + +static size_t HUF_decompress1X2_usingDTable_internal( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + BYTE* op = (BYTE*)dst; + BYTE* const oend = op + dstSize; + const void* dtPtr = DTable + 1; + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + BIT_DStream_t bitD; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); + if (HUF_isError(errorCode)) return errorCode; } + + HUF_decodeStreamX2(op, &bitD, oend, dt, dtLog); + + /* check */ + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + + return dstSize; +} + +size_t HUF_decompress1X2_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 0) return ERROR(GENERIC); + return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); +} + +size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize, workSpace, wkspSize); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress1X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx); +} + + +size_t HUF_decompress1X2_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_decompress1X2_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize, + workSpace, sizeof(workSpace)); +} + +size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); + return HUF_decompress1X2_DCtx (DTable, dst, dstSize, cSrc, cSrcSize); +} + + +static size_t HUF_decompress4X2_usingDTable_internal( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + /* Check */ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable + 1; + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + const size_t segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + U32 endSignal; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + { size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); + if (HUF_isError(errorCode)) return errorCode; } + + /* 16-32 symbols per loop (4-8 symbols per stream) */ + endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + for ( ; (endSignal==BIT_DStream_unfinished) && (op4<(oend-7)) ; ) { + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + } + + /* check corruption */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 supposed already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); + + /* check */ + endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endSignal) return ERROR(corruption_detected); + + /* decoded size */ + return dstSize; + } +} + + +size_t HUF_decompress4X2_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 0) return ERROR(GENERIC); + return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); +} + + +size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX2_wksp (dctx, cSrc, cSrcSize, + workSpace, wkspSize); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress4X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, dctx); +} + + +size_t HUF_decompress4X2_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, + workSpace, sizeof(workSpace)); +} +size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); + return HUF_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); +} + + +/* *************************/ +/* double-symbols decoding */ +/* *************************/ +typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX4; /* double-symbols decoding */ + +typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t; + +/* HUF_fillDTableX4Level2() : + * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */ +static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 consumed, + const U32* rankValOrigin, const int minWeight, + const sortedSymbol_t* sortedSymbols, const U32 sortedListSize, + U32 nbBitsBaseline, U16 baseSeq) +{ + HUF_DEltX4 DElt; + U32 rankVal[HUF_TABLELOG_MAX + 1]; + + /* get pre-calculated rankVal */ + memcpy(rankVal, rankValOrigin, sizeof(rankVal)); + + /* fill skipped values */ + if (minWeight>1) { + U32 i, skipSize = rankVal[minWeight]; + MEM_writeLE16(&(DElt.sequence), baseSeq); + DElt.nbBits = (BYTE)(consumed); + DElt.length = 1; + for (i = 0; i < skipSize; i++) + DTable[i] = DElt; + } + + /* fill DTable */ + { U32 s; for (s=0; s= 1 */ + + rankVal[weight] += length; + } } +} + +typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1]; +typedef rankValCol_t rankVal_t[HUF_TABLELOG_MAX]; + +static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog, + const sortedSymbol_t* sortedList, const U32 sortedListSize, + const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight, + const U32 nbBitsBaseline) +{ + U32 rankVal[HUF_TABLELOG_MAX + 1]; + const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ + const U32 minBits = nbBitsBaseline - maxWeight; + U32 s; + + memcpy(rankVal, rankValOrigin, sizeof(rankVal)); + + /* fill DTable */ + for (s=0; s= minBits) { /* enough room for a second symbol */ + U32 sortedRank; + int minWeight = nbBits + scaleLog; + if (minWeight < 1) minWeight = 1; + sortedRank = rankStart[minWeight]; + HUF_fillDTableX4Level2(DTable+start, targetLog-nbBits, nbBits, + rankValOrigin[nbBits], minWeight, + sortedList+sortedRank, sortedListSize-sortedRank, + nbBitsBaseline, symbol); + } else { + HUF_DEltX4 DElt; + MEM_writeLE16(&(DElt.sequence), symbol); + DElt.nbBits = (BYTE)(nbBits); + DElt.length = 1; + { U32 const end = start + length; + U32 u; + for (u = start; u < end; u++) DTable[u] = DElt; + } } + rankVal[weight] += length; + } +} + +size_t HUF_readDTableX4_wksp(HUF_DTable* DTable, const void* src, + size_t srcSize, void* workSpace, + size_t wkspSize) +{ + U32 tableLog, maxW, sizeOfSort, nbSymbols; + DTableDesc dtd = HUF_getDTableDesc(DTable); + U32 const maxTableLog = dtd.maxTableLog; + size_t iSize; + void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */ + HUF_DEltX4* const dt = (HUF_DEltX4*)dtPtr; + U32 *rankStart; + + rankValCol_t* rankVal; + U32* rankStats; + U32* rankStart0; + sortedSymbol_t* sortedSymbol; + BYTE* weightList; + size_t spaceUsed32 = 0; + + rankVal = (rankValCol_t *)((U32 *)workSpace + spaceUsed32); + spaceUsed32 += (sizeof(rankValCol_t) * HUF_TABLELOG_MAX) >> 2; + rankStats = (U32 *)workSpace + spaceUsed32; + spaceUsed32 += HUF_TABLELOG_MAX + 1; + rankStart0 = (U32 *)workSpace + spaceUsed32; + spaceUsed32 += HUF_TABLELOG_MAX + 2; + sortedSymbol = (sortedSymbol_t *)workSpace + (spaceUsed32 * sizeof(U32)) / sizeof(sortedSymbol_t); + spaceUsed32 += HUF_ALIGN(sizeof(sortedSymbol_t) * (HUF_SYMBOLVALUE_MAX + 1), sizeof(U32)) >> 2; + weightList = (BYTE *)((U32 *)workSpace + spaceUsed32); + spaceUsed32 += HUF_ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2; + + if ((spaceUsed32 << 2) > wkspSize) + return ERROR(tableLog_tooLarge); + workSpace = (U32 *)workSpace + spaceUsed32; + wkspSize -= (spaceUsed32 << 2); + + rankStart = rankStart0 + 1; + memset(rankStats, 0, sizeof(U32) * (2 * HUF_TABLELOG_MAX + 2 + 1)); + + HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */ + if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + /* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); + if (HUF_isError(iSize)) return iSize; + + /* check result */ + if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ + + /* find maxWeight */ + for (maxW = tableLog; rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ + + /* Get start index of each weight */ + { U32 w, nextRankStart = 0; + for (w=1; w> consumed; + } } } } + + HUF_fillDTableX4(dt, maxTableLog, + sortedSymbol, sizeOfSort, + rankStart0, rankVal, maxW, + tableLog+1); + + dtd.tableLog = (BYTE)maxTableLog; + dtd.tableType = 1; + memcpy(DTable, &dtd, sizeof(dtd)); + return iSize; +} + +size_t HUF_readDTableX4(HUF_DTable* DTable, const void* src, size_t srcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_readDTableX4_wksp(DTable, src, srcSize, + workSpace, sizeof(workSpace)); +} + +static U32 HUF_decodeSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + memcpy(op, dt+val, 2); + BIT_skipBits(DStream, dt[val].nbBits); + return dt[val].length; +} + +static U32 HUF_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + memcpy(op, dt+val, 1); + if (dt[val].length==1) BIT_skipBits(DStream, dt[val].nbBits); + else { + if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) { + BIT_skipBits(DStream, dt[val].nbBits); + if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) + /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ + DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); + } } + return 1; +} + + +#define HUF_DECODE_SYMBOLX4_0(ptr, DStreamPtr) \ + ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX4_2(ptr, DStreamPtr) \ + if (MEM_64bits()) \ + ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) + +HINT_INLINE size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const HUF_DEltX4* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 8 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) { + HUF_DECODE_SYMBOLX4_2(p, bitDPtr); + HUF_DECODE_SYMBOLX4_1(p, bitDPtr); + HUF_DECODE_SYMBOLX4_2(p, bitDPtr); + HUF_DECODE_SYMBOLX4_0(p, bitDPtr); + } + + /* closer to end : up to 2 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2)) + HUF_DECODE_SYMBOLX4_0(p, bitDPtr); + + while (p <= pEnd-2) + HUF_DECODE_SYMBOLX4_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ + + if (p < pEnd) + p += HUF_decodeLastSymbolX4(p, bitDPtr, dt, dtLog); + + return p-pStart; +} + + +static size_t HUF_decompress1X4_usingDTable_internal( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + BIT_DStream_t bitD; + + /* Init */ + { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); + if (HUF_isError(errorCode)) return errorCode; + } + + /* decode */ + { BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */ + const HUF_DEltX4* const dt = (const HUF_DEltX4*)dtPtr; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + HUF_decodeStreamX4(ostart, &bitD, oend, dt, dtd.tableLog); + } + + /* check */ + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + + /* decoded size */ + return dstSize; +} + +size_t HUF_decompress1X4_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 1) return ERROR(GENERIC); + return HUF_decompress1X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); +} + +size_t HUF_decompress1X4_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX4_wksp(DCtx, cSrc, cSrcSize, + workSpace, wkspSize); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress1X4_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx); +} + + +size_t HUF_decompress1X4_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_decompress1X4_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize, + workSpace, sizeof(workSpace)); +} + +size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_TABLELOG_MAX); + return HUF_decompress1X4_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); +} + +static size_t HUF_decompress4X4_usingDTable_internal( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable+1; + const HUF_DEltX4* const dt = (const HUF_DEltX4*)dtPtr; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + size_t const segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + U32 endSignal; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + { size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); + if (HUF_isError(errorCode)) return errorCode; } + + /* 16-32 symbols per loop (4-8 symbols per stream) */ + endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + for ( ; (endSignal==BIT_DStream_unfinished) & (op4<(oend-(sizeof(bitD4.bitContainer)-1))) ; ) { + HUF_DECODE_SYMBOLX4_2(op1, &bitD1); + HUF_DECODE_SYMBOLX4_2(op2, &bitD2); + HUF_DECODE_SYMBOLX4_2(op3, &bitD3); + HUF_DECODE_SYMBOLX4_2(op4, &bitD4); + HUF_DECODE_SYMBOLX4_1(op1, &bitD1); + HUF_DECODE_SYMBOLX4_1(op2, &bitD2); + HUF_DECODE_SYMBOLX4_1(op3, &bitD3); + HUF_DECODE_SYMBOLX4_1(op4, &bitD4); + HUF_DECODE_SYMBOLX4_2(op1, &bitD1); + HUF_DECODE_SYMBOLX4_2(op2, &bitD2); + HUF_DECODE_SYMBOLX4_2(op3, &bitD3); + HUF_DECODE_SYMBOLX4_2(op4, &bitD4); + HUF_DECODE_SYMBOLX4_0(op1, &bitD1); + HUF_DECODE_SYMBOLX4_0(op2, &bitD2); + HUF_DECODE_SYMBOLX4_0(op3, &bitD3); + HUF_DECODE_SYMBOLX4_0(op4, &bitD4); + + endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + } + + /* check corruption */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX4(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX4(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX4(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX4(op4, &bitD4, oend, dt, dtLog); + + /* check */ + { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) return ERROR(corruption_detected); } + + /* decoded size */ + return dstSize; + } +} + + +size_t HUF_decompress4X4_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 1) return ERROR(GENERIC); + return HUF_decompress4X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); +} + + +size_t HUF_decompress4X4_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t hSize = HUF_readDTableX4_wksp(dctx, cSrc, cSrcSize, + workSpace, wkspSize); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress4X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx); +} + + +size_t HUF_decompress4X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_decompress4X4_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, + workSpace, sizeof(workSpace)); +} + +size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_TABLELOG_MAX); + return HUF_decompress4X4_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); +} + + +/* ********************************/ +/* Generic decompression selector */ +/* ********************************/ + +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); + return dtd.tableType ? HUF_decompress1X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) : + HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable); +} + +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); + return dtd.tableType ? HUF_decompress4X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) : + HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable); +} + + +typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; +static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] = +{ + /* single, double, quad */ + {{0,0}, {1,1}, {2,2}}, /* Q==0 : impossible */ + {{0,0}, {1,1}, {2,2}}, /* Q==1 : impossible */ + {{ 38,130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */ + {{ 448,128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */ + {{ 556,128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */ + {{ 714,128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */ + {{ 883,128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */ + {{ 897,128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */ + {{ 926,128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */ + {{ 947,128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */ + {{1107,128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */ + {{1177,128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */ + {{1242,128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */ + {{1349,128}, {2644,106}, {5260,106}}, /* Q ==13 : 81-87% */ + {{1455,128}, {2422,124}, {4174,124}}, /* Q ==14 : 87-93% */ + {{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */ +}; + +/** HUF_selectDecoder() : +* Tells which decoder is likely to decode faster, +* based on a set of pre-determined metrics. +* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 . +* Assumption : 0 < cSrcSize, dstSize <= 128 KB */ +U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize) +{ + /* decoder timing evaluation */ + U32 const Q = cSrcSize >= dstSize ? 15 : (U32)(cSrcSize * 16 / dstSize); /* Q < 16 */ + U32 const D256 = (U32)(dstSize >> 8); + U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256); + U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256); + DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, for cache eviction */ + + return DTime1 < DTime0; +} + + +typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); + +size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + static const decompressionAlgo decompress[2] = { HUF_decompress4X2, HUF_decompress4X4 }; + + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); + return decompress[algoNb](dst, dstSize, cSrc, cSrcSize); + } +} + +size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); + return algoNb ? HUF_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : + HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; + } +} + +size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_decompress4X_hufOnly_wksp(dctx, dst, dstSize, cSrc, cSrcSize, + workSpace, sizeof(workSpace)); +} + + +size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, + size_t dstSize, const void* cSrc, + size_t cSrcSize, void* workSpace, + size_t wkspSize) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize == 0) return ERROR(corruption_detected); + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); + return algoNb ? HUF_decompress4X4_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize): + HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); + } +} + +size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); + return algoNb ? HUF_decompress1X4_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize): + HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize); + } +} + +size_t HUF_decompress1X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize) +{ + U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + return HUF_decompress1X_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, + workSpace, sizeof(workSpace)); +} diff --git a/src/borg/algorithms/zstd/lib/decompress/zstd_decompress.c b/src/borg/algorithms/zstd/lib/decompress/zstd_decompress.c new file mode 100644 index 0000000000..96fc609089 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/decompress/zstd_decompress.c @@ -0,0 +1,2655 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* *************************************************************** +* Tuning parameters +*****************************************************************/ +/*! + * HEAPMODE : + * Select how default decompression function ZSTD_decompress() will allocate memory, + * in memory stack (0), or in memory heap (1, requires malloc()) + */ +#ifndef ZSTD_HEAPMODE +# define ZSTD_HEAPMODE 1 +#endif + +/*! +* LEGACY_SUPPORT : +* if set to 1, ZSTD_decompress() can decode older formats (v0.1+) +*/ +#ifndef ZSTD_LEGACY_SUPPORT +# define ZSTD_LEGACY_SUPPORT 0 +#endif + +/*! +* MAXWINDOWSIZE_DEFAULT : +* maximum window size accepted by DStream, by default. +* Frames requiring more memory will be rejected. +*/ +#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT +# define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_DEFAULTMAX) + 1) +#endif + + +/*-******************************************************* +* Dependencies +*********************************************************/ +#include /* memcpy, memmove, memset */ +#include "mem.h" /* low level memory routines */ +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" +#include "zstd_internal.h" + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) +# include "zstd_legacy.h" +#endif + + +/*-************************************* +* Errors +***************************************/ +#define ZSTD_isError ERR_isError /* for inlining */ +#define FSE_isError ERR_isError +#define HUF_isError ERR_isError + + +/*_******************************************************* +* Memory operations +**********************************************************/ +static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); } + + +/*-************************************************************* +* Context management +***************************************************************/ +typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, + ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock, + ZSTDds_decompressLastBlock, ZSTDds_checkChecksum, + ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage; + +typedef enum { zdss_init=0, zdss_loadHeader, + zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage; + +typedef struct { + FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)]; + FSE_DTable OFTable[FSE_DTABLE_SIZE_U32(OffFSELog)]; + FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)]; + HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */ + U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; + U32 rep[ZSTD_REP_NUM]; +} ZSTD_entropyDTables_t; + +struct ZSTD_DCtx_s +{ + const FSE_DTable* LLTptr; + const FSE_DTable* MLTptr; + const FSE_DTable* OFTptr; + const HUF_DTable* HUFptr; + ZSTD_entropyDTables_t entropy; + const void* previousDstEnd; /* detect continuity */ + const void* base; /* start of current segment */ + const void* vBase; /* virtual start of previous segment if it was just before current one */ + const void* dictEnd; /* end of previous segment */ + size_t expected; + ZSTD_frameHeader fParams; + U64 decodedSize; + blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */ + ZSTD_dStage stage; + U32 litEntropy; + U32 fseEntropy; + XXH64_state_t xxhState; + size_t headerSize; + U32 dictID; + ZSTD_format_e format; + const BYTE* litPtr; + ZSTD_customMem customMem; + size_t litSize; + size_t rleSize; + size_t staticSize; + + /* streaming */ + ZSTD_DDict* ddictLocal; + const ZSTD_DDict* ddict; + ZSTD_dStreamStage streamStage; + char* inBuff; + size_t inBuffSize; + size_t inPos; + size_t maxWindowSize; + char* outBuff; + size_t outBuffSize; + size_t outStart; + size_t outEnd; + size_t lhSize; + void* legacyContext; + U32 previousLegacyVersion; + U32 legacyVersion; + U32 hostageByte; + + /* workspace */ + BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH]; + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; +}; /* typedef'd to ZSTD_DCtx within "zstd.h" */ + +size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) +{ + if (dctx==NULL) return 0; /* support sizeof NULL */ + return sizeof(*dctx) + + ZSTD_sizeof_DDict(dctx->ddictLocal) + + dctx->inBuffSize + dctx->outBuffSize; +} + +size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); } + + +static size_t ZSTD_startingInputLength(ZSTD_format_e format) +{ + size_t const startingInputLength = (format==ZSTD_f_zstd1_magicless) ? + ZSTD_frameHeaderSize_prefix - ZSTD_frameIdSize : + ZSTD_frameHeaderSize_prefix; + ZSTD_STATIC_ASSERT(ZSTD_FRAMEHEADERSIZE_PREFIX >= ZSTD_FRAMEIDSIZE); + /* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */ + assert( (format == ZSTD_f_zstd1) || (format == ZSTD_f_zstd1_magicless) ); + return startingInputLength; +} + +static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) +{ + dctx->format = ZSTD_f_zstd1; /* ZSTD_decompressBegin() invokes ZSTD_startingInputLength() with argument dctx->format */ + dctx->staticSize = 0; + dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; + dctx->ddict = NULL; + dctx->ddictLocal = NULL; + dctx->inBuff = NULL; + dctx->inBuffSize = 0; + dctx->outBuffSize = 0; + dctx->streamStage = zdss_init; +} + +ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize) +{ + ZSTD_DCtx* const dctx = (ZSTD_DCtx*) workspace; + + if ((size_t)workspace & 7) return NULL; /* 8-aligned */ + if (workspaceSize < sizeof(ZSTD_DCtx)) return NULL; /* minimum size */ + + ZSTD_initDCtx_internal(dctx); + dctx->staticSize = workspaceSize; + dctx->inBuff = (char*)(dctx+1); + return dctx; +} + +ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) +{ + if (!customMem.customAlloc ^ !customMem.customFree) return NULL; + + { ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_malloc(sizeof(*dctx), customMem); + if (!dctx) return NULL; + dctx->customMem = customMem; + dctx->legacyContext = NULL; + dctx->previousLegacyVersion = 0; + ZSTD_initDCtx_internal(dctx); + return dctx; + } +} + +ZSTD_DCtx* ZSTD_createDCtx(void) +{ + return ZSTD_createDCtx_advanced(ZSTD_defaultCMem); +} + +size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) +{ + if (dctx==NULL) return 0; /* support free on NULL */ + if (dctx->staticSize) return ERROR(memory_allocation); /* not compatible with static DCtx */ + { ZSTD_customMem const cMem = dctx->customMem; + ZSTD_freeDDict(dctx->ddictLocal); + dctx->ddictLocal = NULL; + ZSTD_free(dctx->inBuff, cMem); + dctx->inBuff = NULL; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (dctx->legacyContext) + ZSTD_freeLegacyStreamContext(dctx->legacyContext, dctx->previousLegacyVersion); +#endif + ZSTD_free(dctx, cMem); + return 0; + } +} + +/* no longer useful */ +void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) +{ + size_t const toCopy = (size_t)((char*)(&dstDCtx->inBuff) - (char*)dstDCtx); + memcpy(dstDCtx, srcDCtx, toCopy); /* no need to copy workspace */ +} + + +/*-************************************************************* +* Decompression section +***************************************************************/ + +/*! ZSTD_isFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. + * Note 3 : Skippable Frame Identifiers are considered valid. */ +unsigned ZSTD_isFrame(const void* buffer, size_t size) +{ + if (size < ZSTD_frameIdSize) return 0; + { U32 const magic = MEM_readLE32(buffer); + if (magic == ZSTD_MAGICNUMBER) return 1; + if ((magic & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) return 1; + } +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(buffer, size)) return 1; +#endif + return 0; +} + +/** ZSTD_frameHeaderSize_internal() : + * srcSize must be large enough to reach header size fields. + * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless + * @return : size of the Frame Header + * or an error code, which can be tested with ZSTD_isError() */ +static size_t ZSTD_frameHeaderSize_internal(const void* src, size_t srcSize, ZSTD_format_e format) +{ + size_t const minInputSize = ZSTD_startingInputLength(format); + if (srcSize < minInputSize) return ERROR(srcSize_wrong); + + { BYTE const fhd = ((const BYTE*)src)[minInputSize-1]; + U32 const dictID= fhd & 3; + U32 const singleSegment = (fhd >> 5) & 1; + U32 const fcsId = fhd >> 6; + return minInputSize + !singleSegment + + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId] + + (singleSegment && !fcsId); + } +} + +/** ZSTD_frameHeaderSize() : + * srcSize must be >= ZSTD_frameHeaderSize_prefix. + * @return : size of the Frame Header */ +size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) +{ + return ZSTD_frameHeaderSize_internal(src, srcSize, ZSTD_f_zstd1); +} + + +/** ZSTD_getFrameHeader_internal() : + * decode Frame Header, or require larger `srcSize`. + * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless + * @return : 0, `zfhPtr` is correctly filled, + * >0, `srcSize` is too small, value is wanted `srcSize` amount, + * or an error code, which can be tested using ZSTD_isError() */ +static size_t ZSTD_getFrameHeader_internal(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format) +{ + const BYTE* ip = (const BYTE*)src; + size_t const minInputSize = ZSTD_startingInputLength(format); + + if (srcSize < minInputSize) return minInputSize; + + if ( (format != ZSTD_f_zstd1_magicless) + && (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) { + if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { + /* skippable frame */ + if (srcSize < ZSTD_skippableHeaderSize) + return ZSTD_skippableHeaderSize; /* magic number + frame length */ + memset(zfhPtr, 0, sizeof(*zfhPtr)); + zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_frameIdSize); + zfhPtr->frameType = ZSTD_skippableFrame; + return 0; + } + return ERROR(prefix_unknown); + } + + /* ensure there is enough `srcSize` to fully read/decode frame header */ + { size_t const fhsize = ZSTD_frameHeaderSize_internal(src, srcSize, format); + if (srcSize < fhsize) return fhsize; + zfhPtr->headerSize = (U32)fhsize; + } + + { BYTE const fhdByte = ip[minInputSize-1]; + size_t pos = minInputSize; + U32 const dictIDSizeCode = fhdByte&3; + U32 const checksumFlag = (fhdByte>>2)&1; + U32 const singleSegment = (fhdByte>>5)&1; + U32 const fcsID = fhdByte>>6; + U64 windowSize = 0; + U32 dictID = 0; + U64 frameContentSize = ZSTD_CONTENTSIZE_UNKNOWN; + if ((fhdByte & 0x08) != 0) + return ERROR(frameParameter_unsupported); /* reserved bits, must be zero */ + + if (!singleSegment) { + BYTE const wlByte = ip[pos++]; + U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; + if (windowLog > ZSTD_WINDOWLOG_MAX) + return ERROR(frameParameter_windowTooLarge); + windowSize = (1ULL << windowLog); + windowSize += (windowSize >> 3) * (wlByte&7); + } + switch(dictIDSizeCode) + { + default: assert(0); /* impossible */ + case 0 : break; + case 1 : dictID = ip[pos]; pos++; break; + case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break; + case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break; + } + switch(fcsID) + { + default: assert(0); /* impossible */ + case 0 : if (singleSegment) frameContentSize = ip[pos]; break; + case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break; + case 2 : frameContentSize = MEM_readLE32(ip+pos); break; + case 3 : frameContentSize = MEM_readLE64(ip+pos); break; + } + if (singleSegment) windowSize = frameContentSize; + + zfhPtr->frameType = ZSTD_frame; + zfhPtr->frameContentSize = frameContentSize; + zfhPtr->windowSize = windowSize; + zfhPtr->blockSizeMax = (unsigned) MIN(windowSize, ZSTD_BLOCKSIZE_MAX); + zfhPtr->dictID = dictID; + zfhPtr->checksumFlag = checksumFlag; + } + return 0; +} + +/** ZSTD_getFrameHeader() : + * decode Frame Header, or require larger `srcSize`. + * note : this function does not consume input, it only reads it. + * @return : 0, `zfhPtr` is correctly filled, + * >0, `srcSize` is too small, value is wanted `srcSize` amount, + * or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize) +{ + return ZSTD_getFrameHeader_internal(zfhPtr, src, srcSize, ZSTD_f_zstd1); +} + + +/** ZSTD_getFrameContentSize() : + * compatible with legacy mode + * @return : decompressed size of the single frame pointed to be `src` if known, otherwise + * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined + * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */ +unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) +{ +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(src, srcSize)) { + unsigned long long const ret = ZSTD_getDecompressedSize_legacy(src, srcSize); + return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret; + } +#endif + { ZSTD_frameHeader zfh; + if (ZSTD_getFrameHeader(&zfh, src, srcSize) != 0) + return ZSTD_CONTENTSIZE_ERROR; + if (zfh.frameType == ZSTD_skippableFrame) { + return 0; + } else { + return zfh.frameContentSize; + } } +} + +/** ZSTD_findDecompressedSize() : + * compatible with legacy mode + * `srcSize` must be the exact length of some number of ZSTD compressed and/or + * skippable frames + * @return : decompressed size of the frames contained */ +unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) +{ + unsigned long long totalDstSize = 0; + + while (srcSize >= ZSTD_frameHeaderSize_prefix) { + U32 const magicNumber = MEM_readLE32(src); + + if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { + size_t skippableSize; + if (srcSize < ZSTD_skippableHeaderSize) + return ERROR(srcSize_wrong); + skippableSize = MEM_readLE32((const BYTE *)src + ZSTD_frameIdSize) + + ZSTD_skippableHeaderSize; + if (srcSize < skippableSize) { + return ZSTD_CONTENTSIZE_ERROR; + } + + src = (const BYTE *)src + skippableSize; + srcSize -= skippableSize; + continue; + } + + { unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); + if (ret >= ZSTD_CONTENTSIZE_ERROR) return ret; + + /* check for overflow */ + if (totalDstSize + ret < totalDstSize) return ZSTD_CONTENTSIZE_ERROR; + totalDstSize += ret; + } + { size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize); + if (ZSTD_isError(frameSrcSize)) { + return ZSTD_CONTENTSIZE_ERROR; + } + + src = (const BYTE *)src + frameSrcSize; + srcSize -= frameSrcSize; + } + } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */ + + if (srcSize) return ZSTD_CONTENTSIZE_ERROR; + + return totalDstSize; +} + +/** ZSTD_getDecompressedSize() : +* compatible with legacy mode +* @return : decompressed size if known, 0 otherwise + note : 0 can mean any of the following : + - frame content is empty + - decompressed size field is not present in frame header + - frame header unknown / not supported + - frame header not complete (`srcSize` too small) */ +unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize) +{ + unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_ERROR < ZSTD_CONTENTSIZE_UNKNOWN); + return (ret >= ZSTD_CONTENTSIZE_ERROR) ? 0 : ret; +} + + +/** ZSTD_decodeFrameHeader() : +* `headerSize` must be the size provided by ZSTD_frameHeaderSize(). +* @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */ +static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize) +{ + size_t const result = ZSTD_getFrameHeader_internal(&(dctx->fParams), src, headerSize, dctx->format); + if (ZSTD_isError(result)) return result; /* invalid header */ + if (result>0) return ERROR(srcSize_wrong); /* headerSize too small */ + if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) + return ERROR(dictionary_wrong); + if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0); + return 0; +} + + +/*! ZSTD_getcBlockSize() : +* Provides the size of compressed block from block header `src` */ +size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, + blockProperties_t* bpPtr) +{ + if (srcSize < ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); + { U32 const cBlockHeader = MEM_readLE24(src); + U32 const cSize = cBlockHeader >> 3; + bpPtr->lastBlock = cBlockHeader & 1; + bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3); + bpPtr->origSize = cSize; /* only useful for RLE */ + if (bpPtr->blockType == bt_rle) return 1; + if (bpPtr->blockType == bt_reserved) return ERROR(corruption_detected); + return cSize; + } +} + + +static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall); + memcpy(dst, src, srcSize); + return srcSize; +} + + +static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + size_t regenSize) +{ + if (srcSize != 1) return ERROR(srcSize_wrong); + if (regenSize > dstCapacity) return ERROR(dstSize_tooSmall); + memset(dst, *(const BYTE*)src, regenSize); + return regenSize; +} + +/*! ZSTD_decodeLiteralsBlock() : + * @return : nb of bytes read from src (< srcSize ) + * note : symbol not declared but exposed for fullbench */ +size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, + const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */ +{ + if (srcSize < MIN_CBLOCK_SIZE) return ERROR(corruption_detected); + + { const BYTE* const istart = (const BYTE*) src; + symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3); + + switch(litEncType) + { + case set_repeat: + if (dctx->litEntropy==0) return ERROR(dictionary_corrupted); + /* fall-through */ + case set_compressed: + if (srcSize < 5) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3 */ + { size_t lhSize, litSize, litCSize; + U32 singleStream=0; + U32 const lhlCode = (istart[0] >> 2) & 3; + U32 const lhc = MEM_readLE32(istart); + switch(lhlCode) + { + case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */ + /* 2 - 2 - 10 - 10 */ + singleStream = !lhlCode; + lhSize = 3; + litSize = (lhc >> 4) & 0x3FF; + litCSize = (lhc >> 14) & 0x3FF; + break; + case 2: + /* 2 - 2 - 14 - 14 */ + lhSize = 4; + litSize = (lhc >> 4) & 0x3FFF; + litCSize = lhc >> 18; + break; + case 3: + /* 2 - 2 - 18 - 18 */ + lhSize = 5; + litSize = (lhc >> 4) & 0x3FFFF; + litCSize = (lhc >> 22) + (istart[4] << 10); + break; + } + if (litSize > ZSTD_BLOCKSIZE_MAX) return ERROR(corruption_detected); + if (litCSize + lhSize > srcSize) return ERROR(corruption_detected); + + if (HUF_isError((litEncType==set_repeat) ? + ( singleStream ? + HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) : + HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) ) : + ( singleStream ? + HUF_decompress1X2_DCtx_wksp(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize, + dctx->entropy.workspace, sizeof(dctx->entropy.workspace)) : + HUF_decompress4X_hufOnly_wksp(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize, + dctx->entropy.workspace, sizeof(dctx->entropy.workspace))))) + return ERROR(corruption_detected); + + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + dctx->litEntropy = 1; + if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable; + memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); + return litCSize + lhSize; + } + + case set_basic: + { size_t litSize, lhSize; + U32 const lhlCode = ((istart[0]) >> 2) & 3; + switch(lhlCode) + { + case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ + lhSize = 1; + litSize = istart[0] >> 3; + break; + case 1: + lhSize = 2; + litSize = MEM_readLE16(istart) >> 4; + break; + case 3: + lhSize = 3; + litSize = MEM_readLE24(istart) >> 4; + break; + } + + if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ + if (litSize+lhSize > srcSize) return ERROR(corruption_detected); + memcpy(dctx->litBuffer, istart+lhSize, litSize); + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); + return lhSize+litSize; + } + /* direct reference into compressed stream */ + dctx->litPtr = istart+lhSize; + dctx->litSize = litSize; + return lhSize+litSize; + } + + case set_rle: + { U32 const lhlCode = ((istart[0]) >> 2) & 3; + size_t litSize, lhSize; + switch(lhlCode) + { + case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ + lhSize = 1; + litSize = istart[0] >> 3; + break; + case 1: + lhSize = 2; + litSize = MEM_readLE16(istart) >> 4; + break; + case 3: + lhSize = 3; + litSize = MEM_readLE24(istart) >> 4; + if (srcSize<4) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */ + break; + } + if (litSize > ZSTD_BLOCKSIZE_MAX) return ERROR(corruption_detected); + memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH); + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + return lhSize+1; + } + default: + return ERROR(corruption_detected); /* impossible */ + } + } +} + + +typedef union { + FSE_decode_t realData; + U32 alignedBy4; +} FSE_decode_t4; + +/* Default FSE distribution table for Literal Lengths */ +static const FSE_decode_t4 LL_defaultDTable[(1< max) return ERROR(corruption_detected); + FSE_buildDTable_rle(DTableSpace, *(const BYTE*)src); + *DTablePtr = DTableSpace; + return 1; + case set_basic : + *DTablePtr = (const FSE_DTable*)tmpPtr; + return 0; + case set_repeat: + if (!flagRepeatTable) return ERROR(corruption_detected); + return 0; + default : /* impossible */ + case set_compressed : + { U32 tableLog; + S16 norm[MaxSeq+1]; + size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize); + if (FSE_isError(headerSize)) return ERROR(corruption_detected); + if (tableLog > maxLog) return ERROR(corruption_detected); + FSE_buildDTable(DTableSpace, norm, max, tableLog); + *DTablePtr = DTableSpace; + return headerSize; + } } +} + +size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, + const void* src, size_t srcSize) +{ + const BYTE* const istart = (const BYTE* const)src; + const BYTE* const iend = istart + srcSize; + const BYTE* ip = istart; + DEBUGLOG(5, "ZSTD_decodeSeqHeaders"); + + /* check */ + if (srcSize < MIN_SEQUENCES_SIZE) return ERROR(srcSize_wrong); + + /* SeqHead */ + { int nbSeq = *ip++; + if (!nbSeq) { *nbSeqPtr=0; return 1; } + if (nbSeq > 0x7F) { + if (nbSeq == 0xFF) { + if (ip+2 > iend) return ERROR(srcSize_wrong); + nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2; + } else { + if (ip >= iend) return ERROR(srcSize_wrong); + nbSeq = ((nbSeq-0x80)<<8) + *ip++; + } + } + *nbSeqPtr = nbSeq; + } + + /* FSE table descriptors */ + if (ip+4 > iend) return ERROR(srcSize_wrong); /* minimum possible size */ + { symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); + symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); + symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); + ip++; + + /* Build DTables */ + { size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr, + LLtype, MaxLL, LLFSELog, + ip, iend-ip, LL_defaultDTable, dctx->fseEntropy); + if (ZSTD_isError(llhSize)) return ERROR(corruption_detected); + ip += llhSize; + } + { size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr, + OFtype, MaxOff, OffFSELog, + ip, iend-ip, OF_defaultDTable, dctx->fseEntropy); + if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected); + ip += ofhSize; + } + { size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr, + MLtype, MaxML, MLFSELog, + ip, iend-ip, ML_defaultDTable, dctx->fseEntropy); + if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected); + ip += mlhSize; + } + } + + return ip-istart; +} + + +typedef struct { + size_t litLength; + size_t matchLength; + size_t offset; + const BYTE* match; +} seq_t; + +typedef struct { + BIT_DStream_t DStream; + FSE_DState_t stateLL; + FSE_DState_t stateOffb; + FSE_DState_t stateML; + size_t prevOffset[ZSTD_REP_NUM]; + const BYTE* base; + size_t pos; + uPtrDiff gotoDict; +} seqState_t; + + +FORCE_NOINLINE +size_t ZSTD_execSequenceLast7(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + /* check */ + if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ + if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */ + if (oLitEnd <= oend_w) return ERROR(GENERIC); /* Precondition */ + + /* copy literals */ + if (op < oend_w) { + ZSTD_wildcopy(op, *litPtr, oend_w - op); + *litPtr += oend_w - op; + op = oend_w; + } + while (op < oLitEnd) *op++ = *(*litPtr)++; + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - base)) { + /* offset beyond prefix */ + if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected); + match = dictEnd - (base-match); + if (match + sequence.matchLength <= dictEnd) { + memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = base; + } } + while (op < oMatchEnd) *op++ = *match++; + return sequenceLength; +} + + +typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset=1 } ZSTD_longOffset_e; + +/* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum + * offset bits. But we can only read at most (STREAM_ACCUMULATOR_MIN_32 - 1) + * bits before reloading. This value is the maximum number of bytes we read + * after reloading when we are decoding long offets. + */ +#define LONG_OFFSETS_MAX_EXTRA_BITS_32 \ + (ZSTD_WINDOWLOG_MAX_32 > STREAM_ACCUMULATOR_MIN_32 \ + ? ZSTD_WINDOWLOG_MAX_32 - STREAM_ACCUMULATOR_MIN_32 \ + : 0) + +static seq_t ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets) +{ + seq_t seq; + + U32 const llCode = FSE_peekSymbol(&seqState->stateLL); + U32 const mlCode = FSE_peekSymbol(&seqState->stateML); + U32 const ofCode = FSE_peekSymbol(&seqState->stateOffb); /* <= MaxOff, by table construction */ + + U32 const llBits = LL_bits[llCode]; + U32 const mlBits = ML_bits[mlCode]; + U32 const ofBits = ofCode; + U32 const totalBits = llBits+mlBits+ofBits; + + static const U32 LL_base[MaxLL+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 18, 20, 22, 24, 28, 32, 40, + 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, + 0x2000, 0x4000, 0x8000, 0x10000 }; + + static const U32 ML_base[MaxML+1] = { + 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 33, 34, + 35, 37, 39, 41, 43, 47, 51, 59, + 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, + 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; + + static const U32 OF_base[MaxOff+1] = { + 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, + 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, + 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, + 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD }; + + /* sequence */ + { size_t offset; + if (!ofCode) + offset = 0; + else { + ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); + ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5); + assert(ofBits <= MaxOff); + if (MEM_32bits() && longOffsets) { + U32 const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN_32-1); + offset = OF_base[ofCode] + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); + if (MEM_32bits() || extraBits) BIT_reloadDStream(&seqState->DStream); + if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits); + } else { + offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); + } + } + + if (ofCode <= 1) { + offset += (llCode==0); + if (offset) { + size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; + temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ + if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset = temp; + } else { + offset = seqState->prevOffset[0]; + } + } else { + seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset; + } + seq.offset = offset; + } + + seq.matchLength = ML_base[mlCode] + + ((mlCode>31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */ + if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32)) + BIT_reloadDStream(&seqState->DStream); + if (MEM_64bits() && (totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) + BIT_reloadDStream(&seqState->DStream); + /* Verify that there is enough bits to read the rest of the data in 64-bit mode. */ + ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); + + seq.litLength = LL_base[llCode] + + ((llCode>15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */ + if (MEM_32bits()) + BIT_reloadDStream(&seqState->DStream); + + DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u", + (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); + + /* ANS state update */ + FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ + FSE_updateState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ + FSE_updateState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ + + return seq; +} + + +HINT_INLINE +size_t ZSTD_execSequence(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + /* check */ + if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ + if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */ + if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd); + + /* copy Literals */ + ZSTD_copy8(op, *litPtr); + if (sequence.litLength > 8) + ZSTD_wildcopy(op+8, (*litPtr)+8, sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ + op = oLitEnd; + *litPtr = iLitEnd; /* update for next sequence */ + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - base)) { + /* offset beyond prefix -> go into extDict */ + if (sequence.offset > (size_t)(oLitEnd - vBase)) + return ERROR(corruption_detected); + match = dictEnd + (match - base); + if (match + sequence.matchLength <= dictEnd) { + memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = base; + if (op > oend_w || sequence.matchLength < MINMATCH) { + U32 i; + for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i]; + return sequenceLength; + } + } } + /* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */ + + /* match within prefix */ + if (sequence.offset < 8) { + /* close range match, overlap */ + static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ + static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */ + int const sub2 = dec64table[sequence.offset]; + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += dec32table[sequence.offset]; + ZSTD_copy4(op+4, match); + match -= sub2; + } else { + ZSTD_copy8(op, match); + } + op += 8; match += 8; + + if (oMatchEnd > oend-(16-MINMATCH)) { + if (op < oend_w) { + ZSTD_wildcopy(op, match, oend_w - op); + match += oend_w - op; + op = oend_w; + } + while (op < oMatchEnd) *op++ = *match++; + } else { + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */ + } + return sequenceLength; +} + + +static size_t ZSTD_decompressSequences( + ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, + const ZSTD_longOffset_e isLongOffset) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE* const)dst; + BYTE* const oend = ostart + maxDstSize; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* const base = (const BYTE*) (dctx->base); + const BYTE* const vBase = (const BYTE*) (dctx->vBase); + const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); + int nbSeq; + DEBUGLOG(5, "ZSTD_decompressSequences"); + + /* Build Decoding Tables */ + { size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize); + DEBUGLOG(5, "ZSTD_decodeSeqHeaders: size=%u, nbSeq=%i", + (U32)seqHSize, nbSeq); + if (ZSTD_isError(seqHSize)) return seqHSize; + ip += seqHSize; + } + + /* Regen sequences */ + if (nbSeq) { + seqState_t seqState; + dctx->fseEntropy = 1; + { U32 i; for (i=0; ientropy.rep[i]; } + CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected); + FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + + for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) { + nbSeq--; + { seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd); + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + op += oneSeqSize; + } } + + /* check if reached exact end */ + DEBUGLOG(5, "after decode loop, remaining nbSeq : %i", nbSeq); + if (nbSeq) return ERROR(corruption_detected); + /* save reps for next block */ + { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + { size_t const lastLLSize = litEnd - litPtr; + if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall); + memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } + + return op-ostart; +} + + +HINT_INLINE +seq_t ZSTD_decodeSequenceLong(seqState_t* seqState, ZSTD_longOffset_e const longOffsets) +{ + seq_t seq; + + U32 const llCode = FSE_peekSymbol(&seqState->stateLL); + U32 const mlCode = FSE_peekSymbol(&seqState->stateML); + U32 const ofCode = FSE_peekSymbol(&seqState->stateOffb); /* <= MaxOff, by table construction */ + + U32 const llBits = LL_bits[llCode]; + U32 const mlBits = ML_bits[mlCode]; + U32 const ofBits = ofCode; + U32 const totalBits = llBits+mlBits+ofBits; + + static const U32 LL_base[MaxLL+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 18, 20, 22, 24, 28, 32, 40, + 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, + 0x2000, 0x4000, 0x8000, 0x10000 }; + + static const U32 ML_base[MaxML+1] = { + 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 33, 34, + 35, 37, 39, 41, 43, 47, 51, 59, + 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, + 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; + + static const U32 OF_base[MaxOff+1] = { + 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, + 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, + 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, + 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD }; + + /* sequence */ + { size_t offset; + if (!ofCode) + offset = 0; + else { + ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); + ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5); + assert(ofBits <= MaxOff); + if (MEM_32bits() && longOffsets) { + U32 const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN_32-1); + offset = OF_base[ofCode] + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); + if (MEM_32bits() || extraBits) BIT_reloadDStream(&seqState->DStream); + if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits); + } else { + offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); + } + } + + if (ofCode <= 1) { + offset += (llCode==0); + if (offset) { + size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; + temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ + if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset = temp; + } else { + offset = seqState->prevOffset[0]; + } + } else { + seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset; + } + seq.offset = offset; + } + + seq.matchLength = ML_base[mlCode] + ((mlCode>31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */ + if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32)) + BIT_reloadDStream(&seqState->DStream); + if (MEM_64bits() && (totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) + BIT_reloadDStream(&seqState->DStream); + /* Verify that there is enough bits to read the rest of the data in 64-bit mode. */ + ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); + + seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */ + if (MEM_32bits()) + BIT_reloadDStream(&seqState->DStream); + + { size_t const pos = seqState->pos + seq.litLength; + seq.match = seqState->base + pos - seq.offset; /* single memory segment */ + if (seq.offset > pos) seq.match += seqState->gotoDict; /* separate memory segment */ + seqState->pos = pos + seq.matchLength; + } + + /* ANS state update */ + FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ + FSE_updateState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ + FSE_updateState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ + + return seq; +} + + +HINT_INLINE +size_t ZSTD_execSequenceLong(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = sequence.match; + + /* check */ + if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ + if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */ + if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd); + + /* copy Literals */ + ZSTD_copy8(op, *litPtr); + if (sequence.litLength > 8) + ZSTD_wildcopy(op+8, (*litPtr)+8, sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ + op = oLitEnd; + *litPtr = iLitEnd; /* update for next sequence */ + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - base)) { + /* offset beyond prefix */ + if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected); + if (match + sequence.matchLength <= dictEnd) { + memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = base; + if (op > oend_w || sequence.matchLength < MINMATCH) { + U32 i; + for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i]; + return sequenceLength; + } + } } + assert(op <= oend_w); + assert(sequence.matchLength >= MINMATCH); + + /* match within prefix */ + if (sequence.offset < 8) { + /* close range match, overlap */ + static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ + static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */ + int const sub2 = dec64table[sequence.offset]; + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += dec32table[sequence.offset]; + ZSTD_copy4(op+4, match); + match -= sub2; + } else { + ZSTD_copy8(op, match); + } + op += 8; match += 8; + + if (oMatchEnd > oend-(16-MINMATCH)) { + if (op < oend_w) { + ZSTD_wildcopy(op, match, oend_w - op); + match += oend_w - op; + op = oend_w; + } + while (op < oMatchEnd) *op++ = *match++; + } else { + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */ + } + return sequenceLength; +} + +static size_t ZSTD_decompressSequencesLong( + ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, + const ZSTD_longOffset_e isLongOffset) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE* const)dst; + BYTE* const oend = ostart + maxDstSize; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* const base = (const BYTE*) (dctx->base); + const BYTE* const vBase = (const BYTE*) (dctx->vBase); + const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); + int nbSeq; + + /* Build Decoding Tables */ + { size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize); + if (ZSTD_isError(seqHSize)) return seqHSize; + ip += seqHSize; + } + + /* Regen sequences */ + if (nbSeq) { +#define STORED_SEQS 4 +#define STOSEQ_MASK (STORED_SEQS-1) +#define ADVANCED_SEQS 4 + seq_t sequences[STORED_SEQS]; + int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS); + seqState_t seqState; + int seqNb; + dctx->fseEntropy = 1; + { U32 i; for (i=0; ientropy.rep[i]; } + seqState.base = base; + seqState.pos = (size_t)(op-base); + seqState.gotoDict = (uPtrDiff)dictEnd - (uPtrDiff)base; /* cast to avoid undefined behaviour */ + CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected); + FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + + /* prepare in advance */ + for (seqNb=0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && seqNbentropy.rep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + { size_t const lastLLSize = litEnd - litPtr; + if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall); + memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } + + return op-ostart; +} + + +static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, const int frame) +{ /* blockType == blockCompressed */ + const BYTE* ip = (const BYTE*)src; + /* isLongOffset must be true if there are long offsets. + * Offsets are long if they are larger than 2^STREAM_ACCUMULATOR_MIN. + * We don't expect that to be the case in 64-bit mode. + * If we are in block mode we don't know the window size, so we have to be + * conservative. + */ + ZSTD_longOffset_e const isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (!frame || dctx->fParams.windowSize > (1ULL << STREAM_ACCUMULATOR_MIN))); + /* windowSize could be any value at this point, since it is only validated + * in the streaming API. + */ + DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize); + + if (srcSize >= ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); + + /* Decode literals section */ + { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize); + DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : %u", (U32)litCSize); + if (ZSTD_isError(litCSize)) return litCSize; + ip += litCSize; + srcSize -= litCSize; + } + if (frame && dctx->fParams.windowSize > (1<<23)) + return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, isLongOffset); + return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, isLongOffset); +} + + +static void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst) +{ + if (dst != dctx->previousDstEnd) { /* not contiguous */ + dctx->dictEnd = dctx->previousDstEnd; + dctx->vBase = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base)); + dctx->base = dst; + dctx->previousDstEnd = dst; + } +} + +size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t dSize; + ZSTD_checkContinuity(dctx, dst); + dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 0); + dctx->previousDstEnd = (char*)dst + dSize; + return dSize; +} + + +/** ZSTD_insertBlock() : + insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ +ZSTDLIB_API size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize) +{ + ZSTD_checkContinuity(dctx, blockStart); + dctx->previousDstEnd = (const char*)blockStart + blockSize; + return blockSize; +} + + +static size_t ZSTD_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length) +{ + if (length > dstCapacity) return ERROR(dstSize_tooSmall); + memset(dst, byte, length); + return length; +} + +/** ZSTD_findFrameCompressedSize() : + * compatible with legacy mode + * `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame + * `srcSize` must be at least as large as the frame contained + * @return : the compressed size of the frame starting at `src` */ +size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) +{ +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(src, srcSize)) + return ZSTD_findFrameCompressedSizeLegacy(src, srcSize); +#endif + if ( (srcSize >= ZSTD_skippableHeaderSize) + && (MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START ) { + return ZSTD_skippableHeaderSize + MEM_readLE32((const BYTE*)src + ZSTD_frameIdSize); + } else { + const BYTE* ip = (const BYTE*)src; + const BYTE* const ipstart = ip; + size_t remainingSize = srcSize; + ZSTD_frameHeader zfh; + + /* Extract Frame Header */ + { size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize); + if (ZSTD_isError(ret)) return ret; + if (ret > 0) return ERROR(srcSize_wrong); + } + + ip += zfh.headerSize; + remainingSize -= zfh.headerSize; + + /* Loop on each block */ + while (1) { + blockProperties_t blockProperties; + size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + + if (ZSTD_blockHeaderSize + cBlockSize > remainingSize) + return ERROR(srcSize_wrong); + + ip += ZSTD_blockHeaderSize + cBlockSize; + remainingSize -= ZSTD_blockHeaderSize + cBlockSize; + + if (blockProperties.lastBlock) break; + } + + if (zfh.checksumFlag) { /* Final frame content checksum */ + if (remainingSize < 4) return ERROR(srcSize_wrong); + ip += 4; + remainingSize -= 4; + } + + return ip - ipstart; + } +} + +/*! ZSTD_decompressFrame() : +* @dctx must be properly initialized */ +static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void** srcPtr, size_t *srcSizePtr) +{ + const BYTE* ip = (const BYTE*)(*srcPtr); + BYTE* const ostart = (BYTE* const)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart; + size_t remainingSize = *srcSizePtr; + + /* check */ + if (remainingSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) + return ERROR(srcSize_wrong); + + /* Frame Header */ + { size_t const frameHeaderSize = ZSTD_frameHeaderSize(ip, ZSTD_frameHeaderSize_prefix); + if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize; + if (remainingSize < frameHeaderSize+ZSTD_blockHeaderSize) + return ERROR(srcSize_wrong); + CHECK_F( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) ); + ip += frameHeaderSize; remainingSize -= frameHeaderSize; + } + + /* Loop on each block */ + while (1) { + size_t decodedSize; + blockProperties_t blockProperties; + size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + + ip += ZSTD_blockHeaderSize; + remainingSize -= ZSTD_blockHeaderSize; + if (cBlockSize > remainingSize) return ERROR(srcSize_wrong); + + switch(blockProperties.blockType) + { + case bt_compressed: + decodedSize = ZSTD_decompressBlock_internal(dctx, op, oend-op, ip, cBlockSize, /* frame */ 1); + break; + case bt_raw : + decodedSize = ZSTD_copyRawBlock(op, oend-op, ip, cBlockSize); + break; + case bt_rle : + decodedSize = ZSTD_generateNxBytes(op, oend-op, *ip, blockProperties.origSize); + break; + case bt_reserved : + default: + return ERROR(corruption_detected); + } + + if (ZSTD_isError(decodedSize)) return decodedSize; + if (dctx->fParams.checksumFlag) + XXH64_update(&dctx->xxhState, op, decodedSize); + op += decodedSize; + ip += cBlockSize; + remainingSize -= cBlockSize; + if (blockProperties.lastBlock) break; + } + + if (dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) { + if ((U64)(op-ostart) != dctx->fParams.frameContentSize) { + return ERROR(corruption_detected); + } } + if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */ + U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState); + U32 checkRead; + if (remainingSize<4) return ERROR(checksum_wrong); + checkRead = MEM_readLE32(ip); + if (checkRead != checkCalc) return ERROR(checksum_wrong); + ip += 4; + remainingSize -= 4; + } + + /* Allow caller to get size read */ + *srcPtr = ip; + *srcSizePtr = remainingSize; + return op-ostart; +} + +static const void* ZSTD_DDictDictContent(const ZSTD_DDict* ddict); +static size_t ZSTD_DDictDictSize(const ZSTD_DDict* ddict); + +static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + const ZSTD_DDict* ddict) +{ + void* const dststart = dst; + assert(dict==NULL || ddict==NULL); /* either dict or ddict set, not both */ + + if (ddict) { + dict = ZSTD_DDictDictContent(ddict); + dictSize = ZSTD_DDictDictSize(ddict); + } + + while (srcSize >= ZSTD_frameHeaderSize_prefix) { + U32 magicNumber; + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(src, srcSize)) { + size_t decodedSize; + size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize); + if (ZSTD_isError(frameSize)) return frameSize; + /* legacy support is not compatible with static dctx */ + if (dctx->staticSize) return ERROR(memory_allocation); + + decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize); + + dst = (BYTE*)dst + decodedSize; + dstCapacity -= decodedSize; + + src = (const BYTE*)src + frameSize; + srcSize -= frameSize; + + continue; + } +#endif + + magicNumber = MEM_readLE32(src); + DEBUGLOG(4, "reading magic number %08X (expecting %08X)", + (U32)magicNumber, (U32)ZSTD_MAGICNUMBER); + if (magicNumber != ZSTD_MAGICNUMBER) { + if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { + size_t skippableSize; + if (srcSize < ZSTD_skippableHeaderSize) + return ERROR(srcSize_wrong); + skippableSize = MEM_readLE32((const BYTE*)src + ZSTD_frameIdSize) + + ZSTD_skippableHeaderSize; + if (srcSize < skippableSize) return ERROR(srcSize_wrong); + + src = (const BYTE *)src + skippableSize; + srcSize -= skippableSize; + continue; + } + return ERROR(prefix_unknown); + } + + if (ddict) { + /* we were called from ZSTD_decompress_usingDDict */ + CHECK_F(ZSTD_decompressBegin_usingDDict(dctx, ddict)); + } else { + /* this will initialize correctly with no dict if dict == NULL, so + * use this in all cases but ddict */ + CHECK_F(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize)); + } + ZSTD_checkContinuity(dctx, dst); + + { const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity, + &src, &srcSize); + if (ZSTD_isError(res)) return res; + /* no need to bound check, ZSTD_decompressFrame already has */ + dst = (BYTE*)dst + res; + dstCapacity -= res; + } + } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */ + + if (srcSize) return ERROR(srcSize_wrong); /* input not entirely consumed */ + + return (BYTE*)dst - (BYTE*)dststart; +} + +size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize) +{ + return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL); +} + + +size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + return ZSTD_decompress_usingDict(dctx, dst, dstCapacity, src, srcSize, NULL, 0); +} + + +size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ +#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1) + size_t regenSize; + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + if (dctx==NULL) return ERROR(memory_allocation); + regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize); + ZSTD_freeDCtx(dctx); + return regenSize; +#else /* stack mode */ + ZSTD_DCtx dctx; + return ZSTD_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize); +#endif +} + + +/*-************************************** +* Advanced Streaming Decompression API +* Bufferless and synchronous +****************************************/ +size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; } + +ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { + switch(dctx->stage) + { + default: /* should not happen */ + assert(0); + case ZSTDds_getFrameHeaderSize: + case ZSTDds_decodeFrameHeader: + return ZSTDnit_frameHeader; + case ZSTDds_decodeBlockHeader: + return ZSTDnit_blockHeader; + case ZSTDds_decompressBlock: + return ZSTDnit_block; + case ZSTDds_decompressLastBlock: + return ZSTDnit_lastBlock; + case ZSTDds_checkChecksum: + return ZSTDnit_checksum; + case ZSTDds_decodeSkippableHeader: + case ZSTDds_skipFrame: + return ZSTDnit_skippableFrame; + } +} + +static int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) { return dctx->stage == ZSTDds_skipFrame; } + +/** ZSTD_decompressContinue() : + * srcSize : must be the exact nb of bytes expected (see ZSTD_nextSrcSizeToDecompress()) + * @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity) + * or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_decompressContinue"); + /* Sanity check */ + if (srcSize != dctx->expected) return ERROR(srcSize_wrong); /* not allowed */ + if (dstCapacity) ZSTD_checkContinuity(dctx, dst); + + switch (dctx->stage) + { + case ZSTDds_getFrameHeaderSize : + assert(src != NULL); + if (dctx->format == ZSTD_f_zstd1) { /* allows header */ + assert(srcSize >= ZSTD_frameIdSize); /* to read skippable magic number */ + if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ + memcpy(dctx->headerBuffer, src, srcSize); + dctx->expected = ZSTD_skippableHeaderSize - srcSize; /* remaining to load to get full skippable frame header */ + dctx->stage = ZSTDds_decodeSkippableHeader; + return 0; + } } + dctx->headerSize = ZSTD_frameHeaderSize_internal(src, srcSize, dctx->format); + if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; + memcpy(dctx->headerBuffer, src, srcSize); + dctx->expected = dctx->headerSize - srcSize; + dctx->stage = ZSTDds_decodeFrameHeader; + return 0; + + case ZSTDds_decodeFrameHeader: + assert(src != NULL); + memcpy(dctx->headerBuffer + (dctx->headerSize - srcSize), src, srcSize); + CHECK_F(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize)); + dctx->expected = ZSTD_blockHeaderSize; + dctx->stage = ZSTDds_decodeBlockHeader; + return 0; + + case ZSTDds_decodeBlockHeader: + { blockProperties_t bp; + size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + dctx->expected = cBlockSize; + dctx->bType = bp.blockType; + dctx->rleSize = bp.origSize; + if (cBlockSize) { + dctx->stage = bp.lastBlock ? ZSTDds_decompressLastBlock : ZSTDds_decompressBlock; + return 0; + } + /* empty block */ + if (bp.lastBlock) { + if (dctx->fParams.checksumFlag) { + dctx->expected = 4; + dctx->stage = ZSTDds_checkChecksum; + } else { + dctx->expected = 0; /* end of frame */ + dctx->stage = ZSTDds_getFrameHeaderSize; + } + } else { + dctx->expected = ZSTD_blockHeaderSize; /* jump to next header */ + dctx->stage = ZSTDds_decodeBlockHeader; + } + return 0; + } + + case ZSTDds_decompressLastBlock: + case ZSTDds_decompressBlock: + DEBUGLOG(5, "case ZSTDds_decompressBlock"); + { size_t rSize; + switch(dctx->bType) + { + case bt_compressed: + DEBUGLOG(5, "case bt_compressed"); + rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 1); + break; + case bt_raw : + rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize); + break; + case bt_rle : + rSize = ZSTD_setRleBlock(dst, dstCapacity, src, srcSize, dctx->rleSize); + break; + case bt_reserved : /* should never happen */ + default: + return ERROR(corruption_detected); + } + if (ZSTD_isError(rSize)) return rSize; + DEBUGLOG(5, "decoded size from block : %u", (U32)rSize); + dctx->decodedSize += rSize; + if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize); + + if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */ + DEBUGLOG(4, "decoded size from frame : %u", (U32)dctx->decodedSize); + if (dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) { + if (dctx->decodedSize != dctx->fParams.frameContentSize) { + return ERROR(corruption_detected); + } } + if (dctx->fParams.checksumFlag) { /* another round for frame checksum */ + dctx->expected = 4; + dctx->stage = ZSTDds_checkChecksum; + } else { + dctx->expected = 0; /* ends here */ + dctx->stage = ZSTDds_getFrameHeaderSize; + } + } else { + dctx->stage = ZSTDds_decodeBlockHeader; + dctx->expected = ZSTD_blockHeaderSize; + dctx->previousDstEnd = (char*)dst + rSize; + } + return rSize; + } + + case ZSTDds_checkChecksum: + assert(srcSize == 4); /* guaranteed by dctx->expected */ + { U32 const h32 = (U32)XXH64_digest(&dctx->xxhState); + U32 const check32 = MEM_readLE32(src); + DEBUGLOG(4, "checksum : calculated %08X :: %08X read", h32, check32); + if (check32 != h32) return ERROR(checksum_wrong); + dctx->expected = 0; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + } + + case ZSTDds_decodeSkippableHeader: + assert(src != NULL); + assert(srcSize <= ZSTD_skippableHeaderSize); + memcpy(dctx->headerBuffer + (ZSTD_skippableHeaderSize - srcSize), src, srcSize); /* complete skippable header */ + dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_frameIdSize); /* note : dctx->expected can grow seriously large, beyond local buffer size */ + dctx->stage = ZSTDds_skipFrame; + return 0; + + case ZSTDds_skipFrame: + dctx->expected = 0; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + + default: + return ERROR(GENERIC); /* impossible */ + } +} + + +static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + dctx->dictEnd = dctx->previousDstEnd; + dctx->vBase = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base)); + dctx->base = dict; + dctx->previousDstEnd = (const char*)dict + dictSize; + return 0; +} + +/* ZSTD_loadEntropy() : + * dict : must point at beginning of a valid zstd dictionary + * @return : size of entropy tables read */ +static size_t ZSTD_loadEntropy(ZSTD_entropyDTables_t* entropy, const void* const dict, size_t const dictSize) +{ + const BYTE* dictPtr = (const BYTE*)dict; + const BYTE* const dictEnd = dictPtr + dictSize; + + if (dictSize <= 8) return ERROR(dictionary_corrupted); + dictPtr += 8; /* skip header = magic + dictID */ + + + { size_t const hSize = HUF_readDTableX4_wksp( + entropy->hufTable, dictPtr, dictEnd - dictPtr, + entropy->workspace, sizeof(entropy->workspace)); + if (HUF_isError(hSize)) return ERROR(dictionary_corrupted); + dictPtr += hSize; + } + + { short offcodeNCount[MaxOff+1]; + U32 offcodeMaxValue = MaxOff, offcodeLog; + size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); + if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); + if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted); + CHECK_E(FSE_buildDTable(entropy->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted); + dictPtr += offcodeHeaderSize; + } + + { short matchlengthNCount[MaxML+1]; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; + size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); + if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted); + CHECK_E(FSE_buildDTable(entropy->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted); + dictPtr += matchlengthHeaderSize; + } + + { short litlengthNCount[MaxLL+1]; + unsigned litlengthMaxValue = MaxLL, litlengthLog; + size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); + if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted); + CHECK_E(FSE_buildDTable(entropy->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted); + dictPtr += litlengthHeaderSize; + } + + if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted); + { int i; + size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12)); + for (i=0; i<3; i++) { + U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4; + if (rep==0 || rep >= dictContentSize) return ERROR(dictionary_corrupted); + entropy->rep[i] = rep; + } } + + return dictPtr - (const BYTE*)dict; +} + +static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + if (dictSize < 8) return ZSTD_refDictContent(dctx, dict, dictSize); + { U32 const magic = MEM_readLE32(dict); + if (magic != ZSTD_MAGIC_DICTIONARY) { + return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */ + } } + dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_frameIdSize); + + /* load entropy tables */ + { size_t const eSize = ZSTD_loadEntropy(&dctx->entropy, dict, dictSize); + if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted); + dict = (const char*)dict + eSize; + dictSize -= eSize; + } + dctx->litEntropy = dctx->fseEntropy = 1; + + /* reference dictionary content */ + return ZSTD_refDictContent(dctx, dict, dictSize); +} + +/* Note : this function cannot fail */ +size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) +{ + assert(dctx != NULL); + dctx->expected = ZSTD_startingInputLength(dctx->format); /* dctx->format must be properly set */ + dctx->stage = ZSTDds_getFrameHeaderSize; + dctx->decodedSize = 0; + dctx->previousDstEnd = NULL; + dctx->base = NULL; + dctx->vBase = NULL; + dctx->dictEnd = NULL; + dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ + dctx->litEntropy = dctx->fseEntropy = 0; + dctx->dictID = 0; + ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue)); + memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */ + dctx->LLTptr = dctx->entropy.LLTable; + dctx->MLTptr = dctx->entropy.MLTable; + dctx->OFTptr = dctx->entropy.OFTable; + dctx->HUFptr = dctx->entropy.hufTable; + return 0; +} + +size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + CHECK_F( ZSTD_decompressBegin(dctx) ); + if (dict && dictSize) + CHECK_E(ZSTD_decompress_insertDictionary(dctx, dict, dictSize), dictionary_corrupted); + return 0; +} + + +/* ====== ZSTD_DDict ====== */ + +struct ZSTD_DDict_s { + void* dictBuffer; + const void* dictContent; + size_t dictSize; + ZSTD_entropyDTables_t entropy; + U32 dictID; + U32 entropyPresent; + ZSTD_customMem cMem; +}; /* typedef'd to ZSTD_DDict within "zstd.h" */ + +static const void* ZSTD_DDictDictContent(const ZSTD_DDict* ddict) +{ + return ddict->dictContent; +} + +static size_t ZSTD_DDictDictSize(const ZSTD_DDict* ddict) +{ + return ddict->dictSize; +} + +size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddict) +{ + CHECK_F( ZSTD_decompressBegin(dstDCtx) ); + if (ddict) { /* support begin on NULL */ + dstDCtx->dictID = ddict->dictID; + dstDCtx->base = ddict->dictContent; + dstDCtx->vBase = ddict->dictContent; + dstDCtx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize; + dstDCtx->previousDstEnd = dstDCtx->dictEnd; + if (ddict->entropyPresent) { + dstDCtx->litEntropy = 1; + dstDCtx->fseEntropy = 1; + dstDCtx->LLTptr = ddict->entropy.LLTable; + dstDCtx->MLTptr = ddict->entropy.MLTable; + dstDCtx->OFTptr = ddict->entropy.OFTable; + dstDCtx->HUFptr = ddict->entropy.hufTable; + dstDCtx->entropy.rep[0] = ddict->entropy.rep[0]; + dstDCtx->entropy.rep[1] = ddict->entropy.rep[1]; + dstDCtx->entropy.rep[2] = ddict->entropy.rep[2]; + } else { + dstDCtx->litEntropy = 0; + dstDCtx->fseEntropy = 0; + } + } + return 0; +} + +static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict) +{ + ddict->dictID = 0; + ddict->entropyPresent = 0; + if (ddict->dictSize < 8) return 0; + { U32 const magic = MEM_readLE32(ddict->dictContent); + if (magic != ZSTD_MAGIC_DICTIONARY) return 0; /* pure content mode */ + } + ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_frameIdSize); + + /* load entropy tables */ + CHECK_E( ZSTD_loadEntropy(&ddict->entropy, ddict->dictContent, ddict->dictSize), dictionary_corrupted ); + ddict->entropyPresent = 1; + return 0; +} + + +static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod) +{ + if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) { + ddict->dictBuffer = NULL; + ddict->dictContent = dict; + } else { + void* const internalBuffer = ZSTD_malloc(dictSize, ddict->cMem); + ddict->dictBuffer = internalBuffer; + ddict->dictContent = internalBuffer; + if (!internalBuffer) return ERROR(memory_allocation); + memcpy(internalBuffer, dict, dictSize); + } + ddict->dictSize = dictSize; + ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ + + /* parse dictionary content */ + CHECK_F( ZSTD_loadEntropy_inDDict(ddict) ); + + return 0; +} + +ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_customMem customMem) +{ + if (!customMem.customAlloc ^ !customMem.customFree) return NULL; + + { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem); + if (!ddict) return NULL; + ddict->cMem = customMem; + + if (ZSTD_isError( ZSTD_initDDict_internal(ddict, dict, dictSize, dictLoadMethod) )) { + ZSTD_freeDDict(ddict); + return NULL; + } + + return ddict; + } +} + +/*! ZSTD_createDDict() : +* Create a digested dictionary, to start decompression without startup delay. +* `dict` content is copied inside DDict. +* Consequently, `dict` can be released after `ZSTD_DDict` creation */ +ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, allocator); +} + +/*! ZSTD_createDDict_byReference() : + * Create a digested dictionary, to start decompression without startup delay. + * Dictionary content is simply referenced, it will be accessed during decompression. + * Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */ +ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, allocator); +} + + +ZSTD_DDict* ZSTD_initStaticDDict(void* workspace, size_t workspaceSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod) +{ + size_t const neededSpace = + sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); + ZSTD_DDict* const ddict = (ZSTD_DDict*)workspace; + assert(workspace != NULL); + assert(dict != NULL); + if ((size_t)workspace & 7) return NULL; /* 8-aligned */ + if (workspaceSize < neededSpace) return NULL; + if (dictLoadMethod == ZSTD_dlm_byCopy) { + memcpy(ddict+1, dict, dictSize); /* local copy */ + dict = ddict+1; + } + if (ZSTD_isError( ZSTD_initDDict_internal(ddict, dict, dictSize, ZSTD_dlm_byRef) )) + return NULL; + return ddict; +} + + +size_t ZSTD_freeDDict(ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; /* support free on NULL */ + { ZSTD_customMem const cMem = ddict->cMem; + ZSTD_free(ddict->dictBuffer, cMem); + ZSTD_free(ddict, cMem); + return 0; + } +} + +/*! ZSTD_estimateDDictSize() : + * Estimate amount of memory that will be needed to create a dictionary for decompression. + * Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */ +size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod) +{ + return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); +} + +size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; /* support sizeof on NULL */ + return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ; +} + +/*! ZSTD_getDictID_fromDict() : + * Provides the dictID stored within dictionary. + * if @return == 0, the dictionary is not conformant with Zstandard specification. + * It can still be loaded, but as a content-only dictionary. */ +unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) +{ + if (dictSize < 8) return 0; + if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) return 0; + return MEM_readLE32((const char*)dict + ZSTD_frameIdSize); +} + +/*! ZSTD_getDictID_fromDDict() : + * Provides the dictID of the dictionary loaded into `ddict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; + return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize); +} + +/*! ZSTD_getDictID_fromFrame() : + * Provides the dictID required to decompresse frame stored within `src`. + * If @return == 0, the dictID could not be decoded. + * This could for one of the following reasons : + * - The frame does not require a dictionary (most common case). + * - The frame was built with dictID intentionally removed. + * Needed dictionary is a hidden information. + * Note : this use case also happens when using a non-conformant dictionary. + * - `srcSize` is too small, and as a result, frame header could not be decoded. + * Note : possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`. + * - This is not a Zstandard frame. + * When identifying the exact failure cause, it's possible to use + * ZSTD_getFrameHeader(), which will provide a more precise error code. */ +unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize) +{ + ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0 }; + size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize); + if (ZSTD_isError(hError)) return 0; + return zfp.dictID; +} + + +/*! ZSTD_decompress_usingDDict() : +* Decompression using a pre-digested Dictionary +* Use dictionary without significant overhead. */ +size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict) +{ + /* pass content and size in case legacy frames are encountered */ + return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, + NULL, 0, + ddict); +} + + +/*===================================== +* Streaming decompression +*====================================*/ + +ZSTD_DStream* ZSTD_createDStream(void) +{ + return ZSTD_createDStream_advanced(ZSTD_defaultCMem); +} + +ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize) +{ + return ZSTD_initStaticDCtx(workspace, workspaceSize); +} + +ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem) +{ + return ZSTD_createDCtx_advanced(customMem); +} + +size_t ZSTD_freeDStream(ZSTD_DStream* zds) +{ + return ZSTD_freeDCtx(zds); +} + + +/* *** Initialization *** */ + +size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; } +size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; } + +size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize) +{ + zds->streamStage = zdss_loadHeader; + zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; + ZSTD_freeDDict(zds->ddictLocal); + if (dict && dictSize >= 8) { + zds->ddictLocal = ZSTD_createDDict(dict, dictSize); + if (zds->ddictLocal == NULL) return ERROR(memory_allocation); + } else zds->ddictLocal = NULL; + zds->ddict = zds->ddictLocal; + zds->legacyVersion = 0; + zds->hostageByte = 0; + return ZSTD_frameHeaderSize_prefix; +} + +/* note : this variant can't fail */ +size_t ZSTD_initDStream(ZSTD_DStream* zds) +{ + return ZSTD_initDStream_usingDict(zds, NULL, 0); +} + +/* ZSTD_initDStream_usingDDict() : + * ddict will just be referenced, and must outlive decompression session + * this function cannot fail */ +size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict) +{ + size_t const initResult = ZSTD_initDStream(zds); + zds->ddict = ddict; + return initResult; +} + +size_t ZSTD_resetDStream(ZSTD_DStream* zds) +{ + zds->streamStage = zdss_loadHeader; + zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; + zds->legacyVersion = 0; + zds->hostageByte = 0; + return ZSTD_frameHeaderSize_prefix; +} + +size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, + ZSTD_DStreamParameter_e paramType, unsigned paramValue) +{ + ZSTD_STATIC_ASSERT((unsigned)zdss_loadHeader >= (unsigned)zdss_init); + if ((unsigned)zds->streamStage > (unsigned)zdss_loadHeader) + return ERROR(stage_wrong); + switch(paramType) + { + default : return ERROR(parameter_unsupported); + case DStream_p_maxWindowSize : + DEBUGLOG(4, "setting maxWindowSize = %u KB", paramValue >> 10); + zds->maxWindowSize = paramValue ? paramValue : (U32)(-1); + break; + } + return 0; +} + +size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize) +{ + ZSTD_STATIC_ASSERT((unsigned)zdss_loadHeader >= (unsigned)zdss_init); + if ((unsigned)dctx->streamStage > (unsigned)zdss_loadHeader) + return ERROR(stage_wrong); + dctx->maxWindowSize = maxWindowSize; + return 0; +} + +size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format) +{ + DEBUGLOG(4, "ZSTD_DCtx_setFormat : %u", (unsigned)format); + ZSTD_STATIC_ASSERT((unsigned)zdss_loadHeader >= (unsigned)zdss_init); + if ((unsigned)dctx->streamStage > (unsigned)zdss_loadHeader) + return ERROR(stage_wrong); + dctx->format = format; + return 0; +} + + +size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds) +{ + return ZSTD_sizeof_DCtx(zds); +} + +size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize) +{ + size_t const blockSize = (size_t) MIN(windowSize, ZSTD_BLOCKSIZE_MAX); + unsigned long long const neededRBSize = windowSize + blockSize + (WILDCOPY_OVERLENGTH * 2); + unsigned long long const neededSize = MIN(frameContentSize, neededRBSize); + size_t const minRBSize = (size_t) neededSize; + if ((unsigned long long)minRBSize != neededSize) return ERROR(frameParameter_windowTooLarge); + return minRBSize; +} + +size_t ZSTD_estimateDStreamSize(size_t windowSize) +{ + size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX); + size_t const inBuffSize = blockSize; /* no block can be larger */ + size_t const outBuffSize = ZSTD_decodingBufferSize_min(windowSize, ZSTD_CONTENTSIZE_UNKNOWN); + return ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize; +} + +size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize) +{ + U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; /* note : should be user-selectable */ + ZSTD_frameHeader zfh; + size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize); + if (ZSTD_isError(err)) return err; + if (err>0) return ERROR(srcSize_wrong); + if (zfh.windowSize > windowSizeMax) + return ERROR(frameParameter_windowTooLarge); + return ZSTD_estimateDStreamSize((size_t)zfh.windowSize); +} + + +/* ***** Decompression ***** */ + +MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const length = MIN(dstCapacity, srcSize); + memcpy(dst, src, length); + return length; +} + + +size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input) +{ + const char* const istart = (const char*)(input->src) + input->pos; + const char* const iend = (const char*)(input->src) + input->size; + const char* ip = istart; + char* const ostart = (char*)(output->dst) + output->pos; + char* const oend = (char*)(output->dst) + output->size; + char* op = ostart; + U32 someMoreWork = 1; + + DEBUGLOG(5, "ZSTD_decompressStream"); + if (input->pos > input->size) { /* forbidden */ + DEBUGLOG(5, "in: pos: %u vs size: %u", + (U32)input->pos, (U32)input->size); + return ERROR(srcSize_wrong); + } + if (output->pos > output->size) { /* forbidden */ + DEBUGLOG(5, "out: pos: %u vs size: %u", + (U32)output->pos, (U32)output->size); + return ERROR(dstSize_tooSmall); + } + DEBUGLOG(5, "input size : %u", (U32)(input->size - input->pos)); + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + if (zds->legacyVersion) { + /* legacy support is incompatible with static dctx */ + if (zds->staticSize) return ERROR(memory_allocation); + return ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input); + } +#endif + + while (someMoreWork) { + switch(zds->streamStage) + { + case zdss_init : + ZSTD_resetDStream(zds); /* transparent reset on starting decoding a new frame */ + /* fall-through */ + + case zdss_loadHeader : + DEBUGLOG(5, "stage zdss_loadHeader (srcSize : %u)", (U32)(iend - ip)); + { size_t const hSize = ZSTD_getFrameHeader_internal(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format); + DEBUGLOG(5, "header size : %u", (U32)hSize); + if (ZSTD_isError(hSize)) { +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart); + if (legacyVersion) { + const void* const dict = zds->ddict ? zds->ddict->dictContent : NULL; + size_t const dictSize = zds->ddict ? zds->ddict->dictSize : 0; + /* legacy support is incompatible with static dctx */ + if (zds->staticSize) return ERROR(memory_allocation); + CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext, + zds->previousLegacyVersion, legacyVersion, + dict, dictSize)); + zds->legacyVersion = zds->previousLegacyVersion = legacyVersion; + return ZSTD_decompressLegacyStream(zds->legacyContext, legacyVersion, output, input); + } +#endif + return hSize; /* error */ + } + if (hSize != 0) { /* need more input */ + size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */ + if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */ + if (iend-ip > 0) { + memcpy(zds->headerBuffer + zds->lhSize, ip, iend-ip); + zds->lhSize += iend-ip; + } + input->pos = input->size; + return (MAX(ZSTD_frameHeaderSize_min, hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ + } + assert(ip != NULL); + memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad; + break; + } } + + /* check for single-pass mode opportunity */ + if (zds->fParams.frameContentSize && zds->fParams.windowSize /* skippable frame if == 0 */ + && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) { + size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart); + if (cSize <= (size_t)(iend-istart)) { + size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, oend-op, istart, cSize, zds->ddict); + if (ZSTD_isError(decompressedSize)) return decompressedSize; + ip = istart + cSize; + op += decompressedSize; + zds->expected = 0; + zds->streamStage = zdss_init; + someMoreWork = 0; + break; + } } + + /* Consume header (see ZSTDds_decodeFrameHeader) */ + DEBUGLOG(4, "Consume header"); + CHECK_F(ZSTD_decompressBegin_usingDDict(zds, zds->ddict)); + + if ((MEM_readLE32(zds->headerBuffer) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ + zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_frameIdSize); + zds->stage = ZSTDds_skipFrame; + } else { + CHECK_F(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize)); + zds->expected = ZSTD_blockHeaderSize; + zds->stage = ZSTDds_decodeBlockHeader; + } + + /* control buffer memory usage */ + DEBUGLOG(4, "Control max buffer memory usage (max %u KB)", + (U32)(zds->maxWindowSize >> 10)); + zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); + if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge); + + /* Adapt buffer sizes to frame header instructions */ + { size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */); + size_t const neededOutBuffSize = ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize); + if ((zds->inBuffSize < neededInBuffSize) || (zds->outBuffSize < neededOutBuffSize)) { + size_t const bufferSize = neededInBuffSize + neededOutBuffSize; + DEBUGLOG(4, "inBuff : from %u to %u", + (U32)zds->inBuffSize, (U32)neededInBuffSize); + DEBUGLOG(4, "outBuff : from %u to %u", + (U32)zds->outBuffSize, (U32)neededOutBuffSize); + if (zds->staticSize) { /* static DCtx */ + DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize); + assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* controlled at init */ + if (bufferSize > zds->staticSize - sizeof(ZSTD_DCtx)) + return ERROR(memory_allocation); + } else { + ZSTD_free(zds->inBuff, zds->customMem); + zds->inBuffSize = 0; + zds->outBuffSize = 0; + zds->inBuff = (char*)ZSTD_malloc(bufferSize, zds->customMem); + if (zds->inBuff == NULL) return ERROR(memory_allocation); + } + zds->inBuffSize = neededInBuffSize; + zds->outBuff = zds->inBuff + zds->inBuffSize; + zds->outBuffSize = neededOutBuffSize; + } } + zds->streamStage = zdss_read; + /* fall-through */ + + case zdss_read: + DEBUGLOG(5, "stage zdss_read"); + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds); + DEBUGLOG(5, "neededInSize = %u", (U32)neededInSize); + if (neededInSize==0) { /* end of frame */ + zds->streamStage = zdss_init; + someMoreWork = 0; + break; + } + if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ + int const isSkipFrame = ZSTD_isSkipFrame(zds); + size_t const decodedSize = ZSTD_decompressContinue(zds, + zds->outBuff + zds->outStart, (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart), + ip, neededInSize); + if (ZSTD_isError(decodedSize)) return decodedSize; + ip += neededInSize; + if (!decodedSize && !isSkipFrame) break; /* this was just a header */ + zds->outEnd = zds->outStart + decodedSize; + zds->streamStage = zdss_flush; + break; + } } + if (ip==iend) { someMoreWork = 0; break; } /* no more input */ + zds->streamStage = zdss_load; + /* fall-through */ + case zdss_load: + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds); + size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */ + size_t loadedSize; + if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected); /* should never happen */ + loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend-ip); + ip += loadedSize; + zds->inPos += loadedSize; + if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ + + /* decode loaded input */ + { const int isSkipFrame = ZSTD_isSkipFrame(zds); + size_t const decodedSize = ZSTD_decompressContinue(zds, + zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart, + zds->inBuff, neededInSize); + if (ZSTD_isError(decodedSize)) return decodedSize; + zds->inPos = 0; /* input is consumed */ + if (!decodedSize && !isSkipFrame) { zds->streamStage = zdss_read; break; } /* this was just a header */ + zds->outEnd = zds->outStart + decodedSize; + } } + zds->streamStage = zdss_flush; + /* fall-through */ + case zdss_flush: + { size_t const toFlushSize = zds->outEnd - zds->outStart; + size_t const flushedSize = ZSTD_limitCopy(op, oend-op, zds->outBuff + zds->outStart, toFlushSize); + op += flushedSize; + zds->outStart += flushedSize; + if (flushedSize == toFlushSize) { /* flush completed */ + zds->streamStage = zdss_read; + if ( (zds->outBuffSize < zds->fParams.frameContentSize) + && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) { + DEBUGLOG(5, "restart filling outBuff from beginning (left:%i, needed:%u)", + (int)(zds->outBuffSize - zds->outStart), + (U32)zds->fParams.blockSizeMax); + zds->outStart = zds->outEnd = 0; + } + break; + } } + /* cannot complete flush */ + someMoreWork = 0; + break; + + default: return ERROR(GENERIC); /* impossible */ + } } + + /* result */ + input->pos += (size_t)(ip-istart); + output->pos += (size_t)(op-ostart); + { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds); + if (!nextSrcSizeHint) { /* frame fully decoded */ + if (zds->outEnd == zds->outStart) { /* output fully flushed */ + if (zds->hostageByte) { + if (input->pos >= input->size) { + /* can't release hostage (not present) */ + zds->streamStage = zdss_read; + return 1; + } + input->pos++; /* release hostage */ + } /* zds->hostageByte */ + return 0; + } /* zds->outEnd == zds->outStart */ + if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */ + input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */ + zds->hostageByte=1; + } + return 1; + } /* nextSrcSizeHint==0 */ + nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds) == ZSTDnit_block); /* preload header of next block */ + if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */ + nextSrcSizeHint -= zds->inPos; /* already loaded*/ + return nextSrcSizeHint; + } +} + + +size_t ZSTD_decompress_generic(ZSTD_DCtx* dctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input) +{ + return ZSTD_decompressStream(dctx, output, input); +} + +size_t ZSTD_decompress_generic_simpleArgs ( + ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos) +{ + ZSTD_outBuffer output = { dst, dstCapacity, *dstPos }; + ZSTD_inBuffer input = { src, srcSize, *srcPos }; + /* ZSTD_compress_generic() will check validity of dstPos and srcPos */ + size_t const cErr = ZSTD_decompress_generic(dctx, &output, &input); + *dstPos = output.pos; + *srcPos = input.pos; + return cErr; +} + +void ZSTD_DCtx_reset(ZSTD_DCtx* dctx) +{ + (void)ZSTD_initDStream(dctx); + dctx->format = ZSTD_f_zstd1; + dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; +} diff --git a/src/borg/algorithms/zstd/lib/deprecated/zbuff.h b/src/borg/algorithms/zstd/lib/deprecated/zbuff.h new file mode 100644 index 0000000000..a93115da4a --- /dev/null +++ b/src/borg/algorithms/zstd/lib/deprecated/zbuff.h @@ -0,0 +1,213 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* *************************************************************** +* NOTES/WARNINGS +******************************************************************/ +/* The streaming API defined here is deprecated. + * Consider migrating towards ZSTD_compressStream() API in `zstd.h` + * See 'lib/README.md'. + *****************************************************************/ + + +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef ZSTD_BUFFERED_H_23987 +#define ZSTD_BUFFERED_H_23987 + +/* ************************************* +* Dependencies +***************************************/ +#include /* size_t */ +#include "zstd.h" /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */ + + +/* *************************************************************** +* Compiler specifics +*****************************************************************/ +/* Deprecation warnings */ +/* Should these warnings be a problem, + it is generally possible to disable them, + typically with -Wno-deprecated-declarations for gcc + or _CRT_SECURE_NO_WARNINGS in Visual. + Otherwise, it's also possible to define ZBUFF_DISABLE_DEPRECATE_WARNINGS */ +#ifdef ZBUFF_DISABLE_DEPRECATE_WARNINGS +# define ZBUFF_DEPRECATED(message) ZSTDLIB_API /* disable deprecation warnings */ +#else +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# define ZBUFF_DEPRECATED(message) [[deprecated(message)]] ZSTDLIB_API +# elif (defined(__GNUC__) && (__GNUC__ >= 5)) || defined(__clang__) +# define ZBUFF_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated(message))) +# elif defined(__GNUC__) && (__GNUC__ >= 3) +# define ZBUFF_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define ZBUFF_DEPRECATED(message) ZSTDLIB_API __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement ZBUFF_DEPRECATED for this compiler") +# define ZBUFF_DEPRECATED(message) ZSTDLIB_API +# endif +#endif /* ZBUFF_DISABLE_DEPRECATE_WARNINGS */ + + +/* ************************************* +* Streaming functions +***************************************/ +/* This is the easier "buffered" streaming API, +* using an internal buffer to lift all restrictions on user-provided buffers +* which can be any size, any place, for both input and output. +* ZBUFF and ZSTD are 100% interoperable, +* frames created by one can be decoded by the other one */ + +typedef ZSTD_CStream ZBUFF_CCtx; +ZBUFF_DEPRECATED("use ZSTD_createCStream") ZBUFF_CCtx* ZBUFF_createCCtx(void); +ZBUFF_DEPRECATED("use ZSTD_freeCStream") size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx); + +ZBUFF_DEPRECATED("use ZSTD_initCStream") size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel); +ZBUFF_DEPRECATED("use ZSTD_initCStream_usingDict") size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); + +ZBUFF_DEPRECATED("use ZSTD_compressStream") size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr); +ZBUFF_DEPRECATED("use ZSTD_flushStream") size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr); +ZBUFF_DEPRECATED("use ZSTD_endStream") size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr); + +/*-************************************************* +* Streaming compression - howto +* +* A ZBUFF_CCtx object is required to track streaming operation. +* Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources. +* ZBUFF_CCtx objects can be reused multiple times. +* +* Start by initializing ZBUF_CCtx. +* Use ZBUFF_compressInit() to start a new compression operation. +* Use ZBUFF_compressInitDictionary() for a compression which requires a dictionary. +* +* Use ZBUFF_compressContinue() repetitively to consume input stream. +* *srcSizePtr and *dstCapacityPtr can be any size. +* The function will report how many bytes were read or written within *srcSizePtr and *dstCapacityPtr. +* Note that it may not consume the entire input, in which case it's up to the caller to present again remaining data. +* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each call, so save its content if it matters or change @dst . +* @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency) +* or an error code, which can be tested using ZBUFF_isError(). +* +* At any moment, it's possible to flush whatever data remains within buffer, using ZBUFF_compressFlush(). +* The nb of bytes written into `dst` will be reported into *dstCapacityPtr. +* Note that the function cannot output more than *dstCapacityPtr, +* therefore, some content might still be left into internal buffer if *dstCapacityPtr is too small. +* @return : nb of bytes still present into internal buffer (0 if it's empty) +* or an error code, which can be tested using ZBUFF_isError(). +* +* ZBUFF_compressEnd() instructs to finish a frame. +* It will perform a flush and write frame epilogue. +* The epilogue is required for decoders to consider a frame completed. +* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *dstCapacityPtr is too small. +* In which case, call again ZBUFF_compressFlush() to complete the flush. +* @return : nb of bytes still present into internal buffer (0 if it's empty) +* or an error code, which can be tested using ZBUFF_isError(). +* +* Hint : _recommended buffer_ sizes (not compulsory) : ZBUFF_recommendedCInSize() / ZBUFF_recommendedCOutSize() +* input : ZBUFF_recommendedCInSize==128 KB block size is the internal unit, use this value to reduce intermediate stages (better latency) +* output : ZBUFF_recommendedCOutSize==ZSTD_compressBound(128 KB) + 3 + 3 : ensures it's always possible to write/flush/end a full block. Skip some buffering. +* By using both, it ensures that input will be entirely consumed, and output will always contain the result, reducing intermediate buffering. +* **************************************************/ + + +typedef ZSTD_DStream ZBUFF_DCtx; +ZBUFF_DEPRECATED("use ZSTD_createDStream") ZBUFF_DCtx* ZBUFF_createDCtx(void); +ZBUFF_DEPRECATED("use ZSTD_freeDStream") size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx); + +ZBUFF_DEPRECATED("use ZSTD_initDStream") size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx); +ZBUFF_DEPRECATED("use ZSTD_initDStream_usingDict") size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* dctx, const void* dict, size_t dictSize); + +ZBUFF_DEPRECATED("use ZSTD_decompressStream") size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, + void* dst, size_t* dstCapacityPtr, + const void* src, size_t* srcSizePtr); + +/*-*************************************************************************** +* Streaming decompression howto +* +* A ZBUFF_DCtx object is required to track streaming operations. +* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources. +* Use ZBUFF_decompressInit() to start a new decompression operation, +* or ZBUFF_decompressInitDictionary() if decompression requires a dictionary. +* Note that ZBUFF_DCtx objects can be re-init multiple times. +* +* Use ZBUFF_decompressContinue() repetitively to consume your input. +* *srcSizePtr and *dstCapacityPtr can be any size. +* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. +* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. +* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`. +* @return : 0 when a frame is completely decoded and fully flushed, +* 1 when there is still some data left within internal buffer to flush, +* >1 when more data is expected, with value being a suggested next input size (it's just a hint, which helps latency), +* or an error code, which can be tested using ZBUFF_isError(). +* +* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize() +* output : ZBUFF_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. +* input : ZBUFF_recommendedDInSize == 128KB + 3; +* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . +* *******************************************************************************/ + + +/* ************************************* +* Tool functions +***************************************/ +ZBUFF_DEPRECATED("use ZSTD_isError") unsigned ZBUFF_isError(size_t errorCode); +ZBUFF_DEPRECATED("use ZSTD_getErrorName") const char* ZBUFF_getErrorName(size_t errorCode); + +/** Functions below provide recommended buffer sizes for Compression or Decompression operations. +* These sizes are just hints, they tend to offer better latency */ +ZBUFF_DEPRECATED("use ZSTD_CStreamInSize") size_t ZBUFF_recommendedCInSize(void); +ZBUFF_DEPRECATED("use ZSTD_CStreamOutSize") size_t ZBUFF_recommendedCOutSize(void); +ZBUFF_DEPRECATED("use ZSTD_DStreamInSize") size_t ZBUFF_recommendedDInSize(void); +ZBUFF_DEPRECATED("use ZSTD_DStreamOutSize") size_t ZBUFF_recommendedDOutSize(void); + +#endif /* ZSTD_BUFFERED_H_23987 */ + + +#ifdef ZBUFF_STATIC_LINKING_ONLY +#ifndef ZBUFF_STATIC_H_30298098432 +#define ZBUFF_STATIC_H_30298098432 + +/* ==================================================================================== + * The definitions in this section are considered experimental. + * They should never be used in association with a dynamic library, as they may change in the future. + * They are provided for advanced usages. + * Use them only in association with static linking. + * ==================================================================================== */ + +/*--- Dependency ---*/ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters, ZSTD_customMem */ +#include "zstd.h" + + +/*--- Custom memory allocator ---*/ +/*! ZBUFF_createCCtx_advanced() : + * Create a ZBUFF compression context using external alloc and free functions */ +ZBUFF_DEPRECATED("use ZSTD_createCStream_advanced") ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem); + +/*! ZBUFF_createDCtx_advanced() : + * Create a ZBUFF decompression context using external alloc and free functions */ +ZBUFF_DEPRECATED("use ZSTD_createDStream_advanced") ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem); + + +/*--- Advanced Streaming Initialization ---*/ +ZBUFF_DEPRECATED("use ZSTD_initDStream_usingDict") size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, + const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pledgedSrcSize); + + +#endif /* ZBUFF_STATIC_H_30298098432 */ +#endif /* ZBUFF_STATIC_LINKING_ONLY */ + + +#if defined (__cplusplus) +} +#endif diff --git a/src/borg/algorithms/zstd/lib/deprecated/zbuff_common.c b/src/borg/algorithms/zstd/lib/deprecated/zbuff_common.c new file mode 100644 index 0000000000..661b9b0e18 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/deprecated/zbuff_common.c @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/*-************************************* +* Dependencies +***************************************/ +#include "error_private.h" +#include "zbuff.h" + +/*-**************************************** +* ZBUFF Error Management (deprecated) +******************************************/ + +/*! ZBUFF_isError() : +* tells if a return value is an error code */ +unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); } +/*! ZBUFF_getErrorName() : +* provides error code string from function result (useful for debugging) */ +const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } diff --git a/src/borg/algorithms/zstd/lib/deprecated/zbuff_compress.c b/src/borg/algorithms/zstd/lib/deprecated/zbuff_compress.c new file mode 100644 index 0000000000..8adbaec260 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/deprecated/zbuff_compress.c @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + + +/* ************************************* +* Dependencies +***************************************/ +#define ZBUFF_STATIC_LINKING_ONLY +#include "zbuff.h" + + +/*-*********************************************************** +* Streaming compression +* +* A ZBUFF_CCtx object is required to track streaming operation. +* Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources. +* Use ZBUFF_compressInit() to start a new compression operation. +* ZBUFF_CCtx objects can be reused multiple times. +* +* Use ZBUFF_compressContinue() repetitively to consume your input. +* *srcSizePtr and *dstCapacityPtr can be any size. +* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. +* Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input. +* The content of dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change dst . +* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency) +* or an error code, which can be tested using ZBUFF_isError(). +* +* ZBUFF_compressFlush() can be used to instruct ZBUFF to compress and output whatever remains within its buffer. +* Note that it will not output more than *dstCapacityPtr. +* Therefore, some content might still be left into its internal buffer if dst buffer is too small. +* @return : nb of bytes still present into internal buffer (0 if it's empty) +* or an error code, which can be tested using ZBUFF_isError(). +* +* ZBUFF_compressEnd() instructs to finish a frame. +* It will perform a flush and write frame epilogue. +* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *dstCapacityPtr is too small. +* @return : nb of bytes still present into internal buffer (0 if it's empty) +* or an error code, which can be tested using ZBUFF_isError(). +* +* Hint : recommended buffer sizes (not compulsory) +* input : ZSTD_BLOCKSIZE_MAX (128 KB), internal unit size, it improves latency to use this value. +* output : ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize : ensures it's always possible to write/flush/end a full block at best speed. +* ***********************************************************/ + +ZBUFF_CCtx* ZBUFF_createCCtx(void) +{ + return ZSTD_createCStream(); +} + +ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem) +{ + return ZSTD_createCStream_advanced(customMem); +} + +size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc) +{ + return ZSTD_freeCStream(zbc); +} + + +/* ====== Initialization ====== */ + +size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, + const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pledgedSrcSize) +{ + return ZSTD_initCStream_advanced(zbc, dict, dictSize, params, pledgedSrcSize); +} + + +size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel) +{ + return ZSTD_initCStream_usingDict(zbc, dict, dictSize, compressionLevel); +} + +size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel) +{ + return ZSTD_initCStream(zbc, compressionLevel); +} + +/* ====== Compression ====== */ + + +size_t ZBUFF_compressContinue(ZBUFF_CCtx* zbc, + void* dst, size_t* dstCapacityPtr, + const void* src, size_t* srcSizePtr) +{ + size_t result; + ZSTD_outBuffer outBuff; + ZSTD_inBuffer inBuff; + outBuff.dst = dst; + outBuff.pos = 0; + outBuff.size = *dstCapacityPtr; + inBuff.src = src; + inBuff.pos = 0; + inBuff.size = *srcSizePtr; + result = ZSTD_compressStream(zbc, &outBuff, &inBuff); + *dstCapacityPtr = outBuff.pos; + *srcSizePtr = inBuff.pos; + return result; +} + + + +/* ====== Finalize ====== */ + +size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr) +{ + size_t result; + ZSTD_outBuffer outBuff; + outBuff.dst = dst; + outBuff.pos = 0; + outBuff.size = *dstCapacityPtr; + result = ZSTD_flushStream(zbc, &outBuff); + *dstCapacityPtr = outBuff.pos; + return result; +} + + +size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr) +{ + size_t result; + ZSTD_outBuffer outBuff; + outBuff.dst = dst; + outBuff.pos = 0; + outBuff.size = *dstCapacityPtr; + result = ZSTD_endStream(zbc, &outBuff); + *dstCapacityPtr = outBuff.pos; + return result; +} + + + +/* ************************************* +* Tool functions +***************************************/ +size_t ZBUFF_recommendedCInSize(void) { return ZSTD_CStreamInSize(); } +size_t ZBUFF_recommendedCOutSize(void) { return ZSTD_CStreamOutSize(); } diff --git a/src/borg/algorithms/zstd/lib/deprecated/zbuff_decompress.c b/src/borg/algorithms/zstd/lib/deprecated/zbuff_decompress.c new file mode 100644 index 0000000000..923c22b73c --- /dev/null +++ b/src/borg/algorithms/zstd/lib/deprecated/zbuff_decompress.c @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + + +/* ************************************* +* Dependencies +***************************************/ +#define ZBUFF_STATIC_LINKING_ONLY +#include "zbuff.h" + + +ZBUFF_DCtx* ZBUFF_createDCtx(void) +{ + return ZSTD_createDStream(); +} + +ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem) +{ + return ZSTD_createDStream_advanced(customMem); +} + +size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd) +{ + return ZSTD_freeDStream(zbd); +} + + +/* *** Initialization *** */ + +size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbd, const void* dict, size_t dictSize) +{ + return ZSTD_initDStream_usingDict(zbd, dict, dictSize); +} + +size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbd) +{ + return ZSTD_initDStream(zbd); +} + + +/* *** Decompression *** */ + +size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, + void* dst, size_t* dstCapacityPtr, + const void* src, size_t* srcSizePtr) +{ + ZSTD_outBuffer outBuff; + ZSTD_inBuffer inBuff; + size_t result; + outBuff.dst = dst; + outBuff.pos = 0; + outBuff.size = *dstCapacityPtr; + inBuff.src = src; + inBuff.pos = 0; + inBuff.size = *srcSizePtr; + result = ZSTD_decompressStream(zbd, &outBuff, &inBuff); + *dstCapacityPtr = outBuff.pos; + *srcSizePtr = inBuff.pos; + return result; +} + + +/* ************************************* +* Tool functions +***************************************/ +size_t ZBUFF_recommendedDInSize(void) { return ZSTD_DStreamInSize(); } +size_t ZBUFF_recommendedDOutSize(void) { return ZSTD_DStreamOutSize(); } diff --git a/src/borg/algorithms/zstd/lib/dictBuilder/cover.c b/src/borg/algorithms/zstd/lib/dictBuilder/cover.c new file mode 100644 index 0000000000..efdffddbf9 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/dictBuilder/cover.c @@ -0,0 +1,1045 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* ***************************************************************************** + * Constructs a dictionary using a heuristic based on the following paper: + * + * Liao, Petri, Moffat, Wirth + * Effective Construction of Relative Lempel-Ziv Dictionaries + * Published in WWW 2016. + * + * Adapted from code originally written by @ot (Giuseppe Ottaviano). + ******************************************************************************/ + +/*-************************************* +* Dependencies +***************************************/ +#include /* fprintf */ +#include /* malloc, free, qsort */ +#include /* memset */ +#include /* clock */ + +#include "mem.h" /* read */ +#include "pool.h" +#include "threading.h" +#include "zstd_internal.h" /* includes zstd.h */ +#ifndef ZDICT_STATIC_LINKING_ONLY +#define ZDICT_STATIC_LINKING_ONLY +#endif +#include "zdict.h" + +/*-************************************* +* Constants +***************************************/ +#define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB)) + +/*-************************************* +* Console display +***************************************/ +static int g_displayLevel = 2; +#define DISPLAY(...) \ + { \ + fprintf(stderr, __VA_ARGS__); \ + fflush(stderr); \ + } +#define LOCALDISPLAYLEVEL(displayLevel, l, ...) \ + if (displayLevel >= l) { \ + DISPLAY(__VA_ARGS__); \ + } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */ +#define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__) + +#define LOCALDISPLAYUPDATE(displayLevel, l, ...) \ + if (displayLevel >= l) { \ + if ((clock() - g_time > refreshRate) || (displayLevel >= 4)) { \ + g_time = clock(); \ + DISPLAY(__VA_ARGS__); \ + } \ + } +#define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__) +static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; +static clock_t g_time = 0; + +/*-************************************* +* Hash table +*************************************** +* A small specialized hash map for storing activeDmers. +* The map does not resize, so if it becomes full it will loop forever. +* Thus, the map must be large enough to store every value. +* The map implements linear probing and keeps its load less than 0.5. +*/ + +#define MAP_EMPTY_VALUE ((U32)-1) +typedef struct COVER_map_pair_t_s { + U32 key; + U32 value; +} COVER_map_pair_t; + +typedef struct COVER_map_s { + COVER_map_pair_t *data; + U32 sizeLog; + U32 size; + U32 sizeMask; +} COVER_map_t; + +/** + * Clear the map. + */ +static void COVER_map_clear(COVER_map_t *map) { + memset(map->data, MAP_EMPTY_VALUE, map->size * sizeof(COVER_map_pair_t)); +} + +/** + * Initializes a map of the given size. + * Returns 1 on success and 0 on failure. + * The map must be destroyed with COVER_map_destroy(). + * The map is only guaranteed to be large enough to hold size elements. + */ +static int COVER_map_init(COVER_map_t *map, U32 size) { + map->sizeLog = ZSTD_highbit32(size) + 2; + map->size = (U32)1 << map->sizeLog; + map->sizeMask = map->size - 1; + map->data = (COVER_map_pair_t *)malloc(map->size * sizeof(COVER_map_pair_t)); + if (!map->data) { + map->sizeLog = 0; + map->size = 0; + return 0; + } + COVER_map_clear(map); + return 1; +} + +/** + * Internal hash function + */ +static const U32 prime4bytes = 2654435761U; +static U32 COVER_map_hash(COVER_map_t *map, U32 key) { + return (key * prime4bytes) >> (32 - map->sizeLog); +} + +/** + * Helper function that returns the index that a key should be placed into. + */ +static U32 COVER_map_index(COVER_map_t *map, U32 key) { + const U32 hash = COVER_map_hash(map, key); + U32 i; + for (i = hash;; i = (i + 1) & map->sizeMask) { + COVER_map_pair_t *pos = &map->data[i]; + if (pos->value == MAP_EMPTY_VALUE) { + return i; + } + if (pos->key == key) { + return i; + } + } +} + +/** + * Returns the pointer to the value for key. + * If key is not in the map, it is inserted and the value is set to 0. + * The map must not be full. + */ +static U32 *COVER_map_at(COVER_map_t *map, U32 key) { + COVER_map_pair_t *pos = &map->data[COVER_map_index(map, key)]; + if (pos->value == MAP_EMPTY_VALUE) { + pos->key = key; + pos->value = 0; + } + return &pos->value; +} + +/** + * Deletes key from the map if present. + */ +static void COVER_map_remove(COVER_map_t *map, U32 key) { + U32 i = COVER_map_index(map, key); + COVER_map_pair_t *del = &map->data[i]; + U32 shift = 1; + if (del->value == MAP_EMPTY_VALUE) { + return; + } + for (i = (i + 1) & map->sizeMask;; i = (i + 1) & map->sizeMask) { + COVER_map_pair_t *const pos = &map->data[i]; + /* If the position is empty we are done */ + if (pos->value == MAP_EMPTY_VALUE) { + del->value = MAP_EMPTY_VALUE; + return; + } + /* If pos can be moved to del do so */ + if (((i - COVER_map_hash(map, pos->key)) & map->sizeMask) >= shift) { + del->key = pos->key; + del->value = pos->value; + del = pos; + shift = 1; + } else { + ++shift; + } + } +} + +/** + * Destroyes a map that is inited with COVER_map_init(). + */ +static void COVER_map_destroy(COVER_map_t *map) { + if (map->data) { + free(map->data); + } + map->data = NULL; + map->size = 0; +} + +/*-************************************* +* Context +***************************************/ + +typedef struct { + const BYTE *samples; + size_t *offsets; + const size_t *samplesSizes; + size_t nbSamples; + U32 *suffix; + size_t suffixSize; + U32 *freqs; + U32 *dmerAt; + unsigned d; +} COVER_ctx_t; + +/* We need a global context for qsort... */ +static COVER_ctx_t *g_ctx = NULL; + +/*-************************************* +* Helper functions +***************************************/ + +/** + * Returns the sum of the sample sizes. + */ +static size_t COVER_sum(const size_t *samplesSizes, unsigned nbSamples) { + size_t sum = 0; + size_t i; + for (i = 0; i < nbSamples; ++i) { + sum += samplesSizes[i]; + } + return sum; +} + +/** + * Returns -1 if the dmer at lp is less than the dmer at rp. + * Return 0 if the dmers at lp and rp are equal. + * Returns 1 if the dmer at lp is greater than the dmer at rp. + */ +static int COVER_cmp(COVER_ctx_t *ctx, const void *lp, const void *rp) { + U32 const lhs = *(U32 const *)lp; + U32 const rhs = *(U32 const *)rp; + return memcmp(ctx->samples + lhs, ctx->samples + rhs, ctx->d); +} +/** + * Faster version for d <= 8. + */ +static int COVER_cmp8(COVER_ctx_t *ctx, const void *lp, const void *rp) { + U64 const mask = (ctx->d == 8) ? (U64)-1 : (((U64)1 << (8 * ctx->d)) - 1); + U64 const lhs = MEM_readLE64(ctx->samples + *(U32 const *)lp) & mask; + U64 const rhs = MEM_readLE64(ctx->samples + *(U32 const *)rp) & mask; + if (lhs < rhs) { + return -1; + } + return (lhs > rhs); +} + +/** + * Same as COVER_cmp() except ties are broken by pointer value + * NOTE: g_ctx must be set to call this function. A global is required because + * qsort doesn't take an opaque pointer. + */ +static int COVER_strict_cmp(const void *lp, const void *rp) { + int result = COVER_cmp(g_ctx, lp, rp); + if (result == 0) { + result = lp < rp ? -1 : 1; + } + return result; +} +/** + * Faster version for d <= 8. + */ +static int COVER_strict_cmp8(const void *lp, const void *rp) { + int result = COVER_cmp8(g_ctx, lp, rp); + if (result == 0) { + result = lp < rp ? -1 : 1; + } + return result; +} + +/** + * Returns the first pointer in [first, last) whose element does not compare + * less than value. If no such element exists it returns last. + */ +static const size_t *COVER_lower_bound(const size_t *first, const size_t *last, + size_t value) { + size_t count = last - first; + while (count != 0) { + size_t step = count / 2; + const size_t *ptr = first; + ptr += step; + if (*ptr < value) { + first = ++ptr; + count -= step + 1; + } else { + count = step; + } + } + return first; +} + +/** + * Generic groupBy function. + * Groups an array sorted by cmp into groups with equivalent values. + * Calls grp for each group. + */ +static void +COVER_groupBy(const void *data, size_t count, size_t size, COVER_ctx_t *ctx, + int (*cmp)(COVER_ctx_t *, const void *, const void *), + void (*grp)(COVER_ctx_t *, const void *, const void *)) { + const BYTE *ptr = (const BYTE *)data; + size_t num = 0; + while (num < count) { + const BYTE *grpEnd = ptr + size; + ++num; + while (num < count && cmp(ctx, ptr, grpEnd) == 0) { + grpEnd += size; + ++num; + } + grp(ctx, ptr, grpEnd); + ptr = grpEnd; + } +} + +/*-************************************* +* Cover functions +***************************************/ + +/** + * Called on each group of positions with the same dmer. + * Counts the frequency of each dmer and saves it in the suffix array. + * Fills `ctx->dmerAt`. + */ +static void COVER_group(COVER_ctx_t *ctx, const void *group, + const void *groupEnd) { + /* The group consists of all the positions with the same first d bytes. */ + const U32 *grpPtr = (const U32 *)group; + const U32 *grpEnd = (const U32 *)groupEnd; + /* The dmerId is how we will reference this dmer. + * This allows us to map the whole dmer space to a much smaller space, the + * size of the suffix array. + */ + const U32 dmerId = (U32)(grpPtr - ctx->suffix); + /* Count the number of samples this dmer shows up in */ + U32 freq = 0; + /* Details */ + const size_t *curOffsetPtr = ctx->offsets; + const size_t *offsetsEnd = ctx->offsets + ctx->nbSamples; + /* Once *grpPtr >= curSampleEnd this occurrence of the dmer is in a + * different sample than the last. + */ + size_t curSampleEnd = ctx->offsets[0]; + for (; grpPtr != grpEnd; ++grpPtr) { + /* Save the dmerId for this position so we can get back to it. */ + ctx->dmerAt[*grpPtr] = dmerId; + /* Dictionaries only help for the first reference to the dmer. + * After that zstd can reference the match from the previous reference. + * So only count each dmer once for each sample it is in. + */ + if (*grpPtr < curSampleEnd) { + continue; + } + freq += 1; + /* Binary search to find the end of the sample *grpPtr is in. + * In the common case that grpPtr + 1 == grpEnd we can skip the binary + * search because the loop is over. + */ + if (grpPtr + 1 != grpEnd) { + const size_t *sampleEndPtr = + COVER_lower_bound(curOffsetPtr, offsetsEnd, *grpPtr); + curSampleEnd = *sampleEndPtr; + curOffsetPtr = sampleEndPtr + 1; + } + } + /* At this point we are never going to look at this segment of the suffix + * array again. We take advantage of this fact to save memory. + * We store the frequency of the dmer in the first position of the group, + * which is dmerId. + */ + ctx->suffix[dmerId] = freq; +} + +/** + * A segment is a range in the source as well as the score of the segment. + */ +typedef struct { + U32 begin; + U32 end; + U32 score; +} COVER_segment_t; + +/** + * Selects the best segment in an epoch. + * Segments of are scored according to the function: + * + * Let F(d) be the frequency of dmer d. + * Let S_i be the dmer at position i of segment S which has length k. + * + * Score(S) = F(S_1) + F(S_2) + ... + F(S_{k-d+1}) + * + * Once the dmer d is in the dictionay we set F(d) = 0. + */ +static COVER_segment_t COVER_selectSegment(const COVER_ctx_t *ctx, U32 *freqs, + COVER_map_t *activeDmers, U32 begin, + U32 end, + ZDICT_cover_params_t parameters) { + /* Constants */ + const U32 k = parameters.k; + const U32 d = parameters.d; + const U32 dmersInK = k - d + 1; + /* Try each segment (activeSegment) and save the best (bestSegment) */ + COVER_segment_t bestSegment = {0, 0, 0}; + COVER_segment_t activeSegment; + /* Reset the activeDmers in the segment */ + COVER_map_clear(activeDmers); + /* The activeSegment starts at the beginning of the epoch. */ + activeSegment.begin = begin; + activeSegment.end = begin; + activeSegment.score = 0; + /* Slide the activeSegment through the whole epoch. + * Save the best segment in bestSegment. + */ + while (activeSegment.end < end) { + /* The dmerId for the dmer at the next position */ + U32 newDmer = ctx->dmerAt[activeSegment.end]; + /* The entry in activeDmers for this dmerId */ + U32 *newDmerOcc = COVER_map_at(activeDmers, newDmer); + /* If the dmer isn't already present in the segment add its score. */ + if (*newDmerOcc == 0) { + /* The paper suggest using the L-0.5 norm, but experiments show that it + * doesn't help. + */ + activeSegment.score += freqs[newDmer]; + } + /* Add the dmer to the segment */ + activeSegment.end += 1; + *newDmerOcc += 1; + + /* If the window is now too large, drop the first position */ + if (activeSegment.end - activeSegment.begin == dmersInK + 1) { + U32 delDmer = ctx->dmerAt[activeSegment.begin]; + U32 *delDmerOcc = COVER_map_at(activeDmers, delDmer); + activeSegment.begin += 1; + *delDmerOcc -= 1; + /* If this is the last occurence of the dmer, subtract its score */ + if (*delDmerOcc == 0) { + COVER_map_remove(activeDmers, delDmer); + activeSegment.score -= freqs[delDmer]; + } + } + + /* If this segment is the best so far save it */ + if (activeSegment.score > bestSegment.score) { + bestSegment = activeSegment; + } + } + { + /* Trim off the zero frequency head and tail from the segment. */ + U32 newBegin = bestSegment.end; + U32 newEnd = bestSegment.begin; + U32 pos; + for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) { + U32 freq = freqs[ctx->dmerAt[pos]]; + if (freq != 0) { + newBegin = MIN(newBegin, pos); + newEnd = pos + 1; + } + } + bestSegment.begin = newBegin; + bestSegment.end = newEnd; + } + { + /* Zero out the frequency of each dmer covered by the chosen segment. */ + U32 pos; + for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) { + freqs[ctx->dmerAt[pos]] = 0; + } + } + return bestSegment; +} + +/** + * Check the validity of the parameters. + * Returns non-zero if the parameters are valid and 0 otherwise. + */ +static int COVER_checkParameters(ZDICT_cover_params_t parameters, + size_t maxDictSize) { + /* k and d are required parameters */ + if (parameters.d == 0 || parameters.k == 0) { + return 0; + } + /* k <= maxDictSize */ + if (parameters.k > maxDictSize) { + return 0; + } + /* d <= k */ + if (parameters.d > parameters.k) { + return 0; + } + return 1; +} + +/** + * Clean up a context initialized with `COVER_ctx_init()`. + */ +static void COVER_ctx_destroy(COVER_ctx_t *ctx) { + if (!ctx) { + return; + } + if (ctx->suffix) { + free(ctx->suffix); + ctx->suffix = NULL; + } + if (ctx->freqs) { + free(ctx->freqs); + ctx->freqs = NULL; + } + if (ctx->dmerAt) { + free(ctx->dmerAt); + ctx->dmerAt = NULL; + } + if (ctx->offsets) { + free(ctx->offsets); + ctx->offsets = NULL; + } +} + +/** + * Prepare a context for dictionary building. + * The context is only dependent on the parameter `d` and can used multiple + * times. + * Returns 1 on success or zero on error. + * The context must be destroyed with `COVER_ctx_destroy()`. + */ +static int COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, + const size_t *samplesSizes, unsigned nbSamples, + unsigned d) { + const BYTE *const samples = (const BYTE *)samplesBuffer; + const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples); + /* Checks */ + if (totalSamplesSize < MAX(d, sizeof(U64)) || + totalSamplesSize >= (size_t)COVER_MAX_SAMPLES_SIZE) { + DISPLAYLEVEL(1, "Total samples size is too large, maximum size is %u MB\n", + (COVER_MAX_SAMPLES_SIZE >> 20)); + return 0; + } + /* Zero the context */ + memset(ctx, 0, sizeof(*ctx)); + DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbSamples, + (U32)totalSamplesSize); + ctx->samples = samples; + ctx->samplesSizes = samplesSizes; + ctx->nbSamples = nbSamples; + /* Partial suffix array */ + ctx->suffixSize = totalSamplesSize - MAX(d, sizeof(U64)) + 1; + ctx->suffix = (U32 *)malloc(ctx->suffixSize * sizeof(U32)); + /* Maps index to the dmerID */ + ctx->dmerAt = (U32 *)malloc(ctx->suffixSize * sizeof(U32)); + /* The offsets of each file */ + ctx->offsets = (size_t *)malloc((nbSamples + 1) * sizeof(size_t)); + if (!ctx->suffix || !ctx->dmerAt || !ctx->offsets) { + DISPLAYLEVEL(1, "Failed to allocate scratch buffers\n"); + COVER_ctx_destroy(ctx); + return 0; + } + ctx->freqs = NULL; + ctx->d = d; + + /* Fill offsets from the samlesSizes */ + { + U32 i; + ctx->offsets[0] = 0; + for (i = 1; i <= nbSamples; ++i) { + ctx->offsets[i] = ctx->offsets[i - 1] + samplesSizes[i - 1]; + } + } + DISPLAYLEVEL(2, "Constructing partial suffix array\n"); + { + /* suffix is a partial suffix array. + * It only sorts suffixes by their first parameters.d bytes. + * The sort is stable, so each dmer group is sorted by position in input. + */ + U32 i; + for (i = 0; i < ctx->suffixSize; ++i) { + ctx->suffix[i] = i; + } + /* qsort doesn't take an opaque pointer, so pass as a global */ + g_ctx = ctx; + qsort(ctx->suffix, ctx->suffixSize, sizeof(U32), + (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp)); + } + DISPLAYLEVEL(2, "Computing frequencies\n"); + /* For each dmer group (group of positions with the same first d bytes): + * 1. For each position we set dmerAt[position] = dmerID. The dmerID is + * (groupBeginPtr - suffix). This allows us to go from position to + * dmerID so we can look up values in freq. + * 2. We calculate how many samples the dmer occurs in and save it in + * freqs[dmerId]. + */ + COVER_groupBy(ctx->suffix, ctx->suffixSize, sizeof(U32), ctx, + (ctx->d <= 8 ? &COVER_cmp8 : &COVER_cmp), &COVER_group); + ctx->freqs = ctx->suffix; + ctx->suffix = NULL; + return 1; +} + +/** + * Given the prepared context build the dictionary. + */ +static size_t COVER_buildDictionary(const COVER_ctx_t *ctx, U32 *freqs, + COVER_map_t *activeDmers, void *dictBuffer, + size_t dictBufferCapacity, + ZDICT_cover_params_t parameters) { + BYTE *const dict = (BYTE *)dictBuffer; + size_t tail = dictBufferCapacity; + /* Divide the data up into epochs of equal size. + * We will select at least one segment from each epoch. + */ + const U32 epochs = (U32)(dictBufferCapacity / parameters.k); + const U32 epochSize = (U32)(ctx->suffixSize / epochs); + size_t epoch; + DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n", epochs, + epochSize); + /* Loop through the epochs until there are no more segments or the dictionary + * is full. + */ + for (epoch = 0; tail > 0; epoch = (epoch + 1) % epochs) { + const U32 epochBegin = (U32)(epoch * epochSize); + const U32 epochEnd = epochBegin + epochSize; + size_t segmentSize; + /* Select a segment */ + COVER_segment_t segment = COVER_selectSegment( + ctx, freqs, activeDmers, epochBegin, epochEnd, parameters); + /* If the segment covers no dmers, then we are out of content */ + if (segment.score == 0) { + break; + } + /* Trim the segment if necessary and if it is too small then we are done */ + segmentSize = MIN(segment.end - segment.begin + parameters.d - 1, tail); + if (segmentSize < parameters.d) { + break; + } + /* We fill the dictionary from the back to allow the best segments to be + * referenced with the smallest offsets. + */ + tail -= segmentSize; + memcpy(dict + tail, ctx->samples + segment.begin, segmentSize); + DISPLAYUPDATE( + 2, "\r%u%% ", + (U32)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity)); + } + DISPLAYLEVEL(2, "\r%79s\r", ""); + return tail; +} + +ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover( + void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, + const size_t *samplesSizes, unsigned nbSamples, + ZDICT_cover_params_t parameters) { + BYTE *const dict = (BYTE *)dictBuffer; + COVER_ctx_t ctx; + COVER_map_t activeDmers; + /* Checks */ + if (!COVER_checkParameters(parameters, dictBufferCapacity)) { + DISPLAYLEVEL(1, "Cover parameters incorrect\n"); + return ERROR(GENERIC); + } + if (nbSamples == 0) { + DISPLAYLEVEL(1, "Cover must have at least one input file\n"); + return ERROR(GENERIC); + } + if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) { + DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n", + ZDICT_DICTSIZE_MIN); + return ERROR(dstSize_tooSmall); + } + /* Initialize global data */ + g_displayLevel = parameters.zParams.notificationLevel; + /* Initialize context and activeDmers */ + if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, + parameters.d)) { + return ERROR(GENERIC); + } + if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) { + DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n"); + COVER_ctx_destroy(&ctx); + return ERROR(GENERIC); + } + + DISPLAYLEVEL(2, "Building dictionary\n"); + { + const size_t tail = + COVER_buildDictionary(&ctx, ctx.freqs, &activeDmers, dictBuffer, + dictBufferCapacity, parameters); + const size_t dictionarySize = ZDICT_finalizeDictionary( + dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail, + samplesBuffer, samplesSizes, nbSamples, parameters.zParams); + if (!ZSTD_isError(dictionarySize)) { + DISPLAYLEVEL(2, "Constructed dictionary of size %u\n", + (U32)dictionarySize); + } + COVER_ctx_destroy(&ctx); + COVER_map_destroy(&activeDmers); + return dictionarySize; + } +} + +/** + * COVER_best_t is used for two purposes: + * 1. Synchronizing threads. + * 2. Saving the best parameters and dictionary. + * + * All of the methods except COVER_best_init() are thread safe if zstd is + * compiled with multithreaded support. + */ +typedef struct COVER_best_s { + ZSTD_pthread_mutex_t mutex; + ZSTD_pthread_cond_t cond; + size_t liveJobs; + void *dict; + size_t dictSize; + ZDICT_cover_params_t parameters; + size_t compressedSize; +} COVER_best_t; + +/** + * Initialize the `COVER_best_t`. + */ +static void COVER_best_init(COVER_best_t *best) { + if (best==NULL) return; /* compatible with init on NULL */ + (void)ZSTD_pthread_mutex_init(&best->mutex, NULL); + (void)ZSTD_pthread_cond_init(&best->cond, NULL); + best->liveJobs = 0; + best->dict = NULL; + best->dictSize = 0; + best->compressedSize = (size_t)-1; + memset(&best->parameters, 0, sizeof(best->parameters)); +} + +/** + * Wait until liveJobs == 0. + */ +static void COVER_best_wait(COVER_best_t *best) { + if (!best) { + return; + } + ZSTD_pthread_mutex_lock(&best->mutex); + while (best->liveJobs != 0) { + ZSTD_pthread_cond_wait(&best->cond, &best->mutex); + } + ZSTD_pthread_mutex_unlock(&best->mutex); +} + +/** + * Call COVER_best_wait() and then destroy the COVER_best_t. + */ +static void COVER_best_destroy(COVER_best_t *best) { + if (!best) { + return; + } + COVER_best_wait(best); + if (best->dict) { + free(best->dict); + } + ZSTD_pthread_mutex_destroy(&best->mutex); + ZSTD_pthread_cond_destroy(&best->cond); +} + +/** + * Called when a thread is about to be launched. + * Increments liveJobs. + */ +static void COVER_best_start(COVER_best_t *best) { + if (!best) { + return; + } + ZSTD_pthread_mutex_lock(&best->mutex); + ++best->liveJobs; + ZSTD_pthread_mutex_unlock(&best->mutex); +} + +/** + * Called when a thread finishes executing, both on error or success. + * Decrements liveJobs and signals any waiting threads if liveJobs == 0. + * If this dictionary is the best so far save it and its parameters. + */ +static void COVER_best_finish(COVER_best_t *best, size_t compressedSize, + ZDICT_cover_params_t parameters, void *dict, + size_t dictSize) { + if (!best) { + return; + } + { + size_t liveJobs; + ZSTD_pthread_mutex_lock(&best->mutex); + --best->liveJobs; + liveJobs = best->liveJobs; + /* If the new dictionary is better */ + if (compressedSize < best->compressedSize) { + /* Allocate space if necessary */ + if (!best->dict || best->dictSize < dictSize) { + if (best->dict) { + free(best->dict); + } + best->dict = malloc(dictSize); + if (!best->dict) { + best->compressedSize = ERROR(GENERIC); + best->dictSize = 0; + return; + } + } + /* Save the dictionary, parameters, and size */ + memcpy(best->dict, dict, dictSize); + best->dictSize = dictSize; + best->parameters = parameters; + best->compressedSize = compressedSize; + } + ZSTD_pthread_mutex_unlock(&best->mutex); + if (liveJobs == 0) { + ZSTD_pthread_cond_broadcast(&best->cond); + } + } +} + +/** + * Parameters for COVER_tryParameters(). + */ +typedef struct COVER_tryParameters_data_s { + const COVER_ctx_t *ctx; + COVER_best_t *best; + size_t dictBufferCapacity; + ZDICT_cover_params_t parameters; +} COVER_tryParameters_data_t; + +/** + * Tries a set of parameters and upates the COVER_best_t with the results. + * This function is thread safe if zstd is compiled with multithreaded support. + * It takes its parameters as an *OWNING* opaque pointer to support threading. + */ +static void COVER_tryParameters(void *opaque) { + /* Save parameters as local variables */ + COVER_tryParameters_data_t *const data = (COVER_tryParameters_data_t *)opaque; + const COVER_ctx_t *const ctx = data->ctx; + const ZDICT_cover_params_t parameters = data->parameters; + size_t dictBufferCapacity = data->dictBufferCapacity; + size_t totalCompressedSize = ERROR(GENERIC); + /* Allocate space for hash table, dict, and freqs */ + COVER_map_t activeDmers; + BYTE *const dict = (BYTE * const)malloc(dictBufferCapacity); + U32 *freqs = (U32 *)malloc(ctx->suffixSize * sizeof(U32)); + if (!COVER_map_init(&activeDmers, parameters.k - parameters.d + 1)) { + DISPLAYLEVEL(1, "Failed to allocate dmer map: out of memory\n"); + goto _cleanup; + } + if (!dict || !freqs) { + DISPLAYLEVEL(1, "Failed to allocate buffers: out of memory\n"); + goto _cleanup; + } + /* Copy the frequencies because we need to modify them */ + memcpy(freqs, ctx->freqs, ctx->suffixSize * sizeof(U32)); + /* Build the dictionary */ + { + const size_t tail = COVER_buildDictionary(ctx, freqs, &activeDmers, dict, + dictBufferCapacity, parameters); + dictBufferCapacity = ZDICT_finalizeDictionary( + dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail, + ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbSamples, + parameters.zParams); + if (ZDICT_isError(dictBufferCapacity)) { + DISPLAYLEVEL(1, "Failed to finalize dictionary\n"); + goto _cleanup; + } + } + /* Check total compressed size */ + { + /* Pointers */ + ZSTD_CCtx *cctx; + ZSTD_CDict *cdict; + void *dst; + /* Local variables */ + size_t dstCapacity; + size_t i; + /* Allocate dst with enough space to compress the maximum sized sample */ + { + size_t maxSampleSize = 0; + for (i = 0; i < ctx->nbSamples; ++i) { + maxSampleSize = MAX(ctx->samplesSizes[i], maxSampleSize); + } + dstCapacity = ZSTD_compressBound(maxSampleSize); + dst = malloc(dstCapacity); + } + /* Create the cctx and cdict */ + cctx = ZSTD_createCCtx(); + cdict = ZSTD_createCDict(dict, dictBufferCapacity, + parameters.zParams.compressionLevel); + if (!dst || !cctx || !cdict) { + goto _compressCleanup; + } + /* Compress each sample and sum their sizes (or error) */ + totalCompressedSize = dictBufferCapacity; + for (i = 0; i < ctx->nbSamples; ++i) { + const size_t size = ZSTD_compress_usingCDict( + cctx, dst, dstCapacity, ctx->samples + ctx->offsets[i], + ctx->samplesSizes[i], cdict); + if (ZSTD_isError(size)) { + totalCompressedSize = ERROR(GENERIC); + goto _compressCleanup; + } + totalCompressedSize += size; + } + _compressCleanup: + ZSTD_freeCCtx(cctx); + ZSTD_freeCDict(cdict); + if (dst) { + free(dst); + } + } + +_cleanup: + COVER_best_finish(data->best, totalCompressedSize, parameters, dict, + dictBufferCapacity); + free(data); + COVER_map_destroy(&activeDmers); + if (dict) { + free(dict); + } + if (freqs) { + free(freqs); + } +} + +ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover( + void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, + const size_t *samplesSizes, unsigned nbSamples, + ZDICT_cover_params_t *parameters) { + /* constants */ + const unsigned nbThreads = parameters->nbThreads; + const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d; + const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d; + const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k; + const unsigned kMaxK = parameters->k == 0 ? 2000 : parameters->k; + const unsigned kSteps = parameters->steps == 0 ? 40 : parameters->steps; + const unsigned kStepSize = MAX((kMaxK - kMinK) / kSteps, 1); + const unsigned kIterations = + (1 + (kMaxD - kMinD) / 2) * (1 + (kMaxK - kMinK) / kStepSize); + /* Local variables */ + const int displayLevel = parameters->zParams.notificationLevel; + unsigned iteration = 1; + unsigned d; + unsigned k; + COVER_best_t best; + POOL_ctx *pool = NULL; + /* Checks */ + if (kMinK < kMaxD || kMaxK < kMinK) { + LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n"); + return ERROR(GENERIC); + } + if (nbSamples == 0) { + DISPLAYLEVEL(1, "Cover must have at least one input file\n"); + return ERROR(GENERIC); + } + if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) { + DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n", + ZDICT_DICTSIZE_MIN); + return ERROR(dstSize_tooSmall); + } + if (nbThreads > 1) { + pool = POOL_create(nbThreads, 1); + if (!pool) { + return ERROR(memory_allocation); + } + } + /* Initialization */ + COVER_best_init(&best); + /* Turn down global display level to clean up display at level 2 and below */ + g_displayLevel = displayLevel == 0 ? 0 : displayLevel - 1; + /* Loop through d first because each new value needs a new context */ + LOCALDISPLAYLEVEL(displayLevel, 2, "Trying %u different sets of parameters\n", + kIterations); + for (d = kMinD; d <= kMaxD; d += 2) { + /* Initialize the context for this value of d */ + COVER_ctx_t ctx; + LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d); + if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d)) { + LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n"); + COVER_best_destroy(&best); + POOL_free(pool); + return ERROR(GENERIC); + } + /* Loop through k reusing the same context */ + for (k = kMinK; k <= kMaxK; k += kStepSize) { + /* Prepare the arguments */ + COVER_tryParameters_data_t *data = (COVER_tryParameters_data_t *)malloc( + sizeof(COVER_tryParameters_data_t)); + LOCALDISPLAYLEVEL(displayLevel, 3, "k=%u\n", k); + if (!data) { + LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to allocate parameters\n"); + COVER_best_destroy(&best); + COVER_ctx_destroy(&ctx); + POOL_free(pool); + return ERROR(GENERIC); + } + data->ctx = &ctx; + data->best = &best; + data->dictBufferCapacity = dictBufferCapacity; + data->parameters = *parameters; + data->parameters.k = k; + data->parameters.d = d; + data->parameters.steps = kSteps; + data->parameters.zParams.notificationLevel = g_displayLevel; + /* Check the parameters */ + if (!COVER_checkParameters(data->parameters, dictBufferCapacity)) { + DISPLAYLEVEL(1, "Cover parameters incorrect\n"); + free(data); + continue; + } + /* Call the function and pass ownership of data to it */ + COVER_best_start(&best); + if (pool) { + POOL_add(pool, &COVER_tryParameters, data); + } else { + COVER_tryParameters(data); + } + /* Print status */ + LOCALDISPLAYUPDATE(displayLevel, 2, "\r%u%% ", + (U32)((iteration * 100) / kIterations)); + ++iteration; + } + COVER_best_wait(&best); + COVER_ctx_destroy(&ctx); + } + LOCALDISPLAYLEVEL(displayLevel, 2, "\r%79s\r", ""); + /* Fill the output buffer and parameters with output of the best parameters */ + { + const size_t dictSize = best.dictSize; + if (ZSTD_isError(best.compressedSize)) { + const size_t compressedSize = best.compressedSize; + COVER_best_destroy(&best); + POOL_free(pool); + return compressedSize; + } + *parameters = best.parameters; + memcpy(dictBuffer, best.dict, dictSize); + COVER_best_destroy(&best); + POOL_free(pool); + return dictSize; + } +} diff --git a/src/borg/algorithms/zstd/lib/dictBuilder/divsufsort.c b/src/borg/algorithms/zstd/lib/dictBuilder/divsufsort.c new file mode 100644 index 0000000000..60cceb0883 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/dictBuilder/divsufsort.c @@ -0,0 +1,1913 @@ +/* + * divsufsort.c for libdivsufsort-lite + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/*- Compiler specifics -*/ +#ifdef __clang__ +#pragma clang diagnostic ignored "-Wshorten-64-to-32" +#endif + +#if defined(_MSC_VER) +# pragma warning(disable : 4244) +# pragma warning(disable : 4127) /* C4127 : Condition expression is constant */ +#endif + + +/*- Dependencies -*/ +#include +#include +#include + +#include "divsufsort.h" + +/*- Constants -*/ +#if defined(INLINE) +# undef INLINE +#endif +#if !defined(INLINE) +# define INLINE __inline +#endif +#if defined(ALPHABET_SIZE) && (ALPHABET_SIZE < 1) +# undef ALPHABET_SIZE +#endif +#if !defined(ALPHABET_SIZE) +# define ALPHABET_SIZE (256) +#endif +#define BUCKET_A_SIZE (ALPHABET_SIZE) +#define BUCKET_B_SIZE (ALPHABET_SIZE * ALPHABET_SIZE) +#if defined(SS_INSERTIONSORT_THRESHOLD) +# if SS_INSERTIONSORT_THRESHOLD < 1 +# undef SS_INSERTIONSORT_THRESHOLD +# define SS_INSERTIONSORT_THRESHOLD (1) +# endif +#else +# define SS_INSERTIONSORT_THRESHOLD (8) +#endif +#if defined(SS_BLOCKSIZE) +# if SS_BLOCKSIZE < 0 +# undef SS_BLOCKSIZE +# define SS_BLOCKSIZE (0) +# elif 32768 <= SS_BLOCKSIZE +# undef SS_BLOCKSIZE +# define SS_BLOCKSIZE (32767) +# endif +#else +# define SS_BLOCKSIZE (1024) +#endif +/* minstacksize = log(SS_BLOCKSIZE) / log(3) * 2 */ +#if SS_BLOCKSIZE == 0 +# define SS_MISORT_STACKSIZE (96) +#elif SS_BLOCKSIZE <= 4096 +# define SS_MISORT_STACKSIZE (16) +#else +# define SS_MISORT_STACKSIZE (24) +#endif +#define SS_SMERGE_STACKSIZE (32) +#define TR_INSERTIONSORT_THRESHOLD (8) +#define TR_STACKSIZE (64) + + +/*- Macros -*/ +#ifndef SWAP +# define SWAP(_a, _b) do { t = (_a); (_a) = (_b); (_b) = t; } while(0) +#endif /* SWAP */ +#ifndef MIN +# define MIN(_a, _b) (((_a) < (_b)) ? (_a) : (_b)) +#endif /* MIN */ +#ifndef MAX +# define MAX(_a, _b) (((_a) > (_b)) ? (_a) : (_b)) +#endif /* MAX */ +#define STACK_PUSH(_a, _b, _c, _d)\ + do {\ + assert(ssize < STACK_SIZE);\ + stack[ssize].a = (_a), stack[ssize].b = (_b),\ + stack[ssize].c = (_c), stack[ssize++].d = (_d);\ + } while(0) +#define STACK_PUSH5(_a, _b, _c, _d, _e)\ + do {\ + assert(ssize < STACK_SIZE);\ + stack[ssize].a = (_a), stack[ssize].b = (_b),\ + stack[ssize].c = (_c), stack[ssize].d = (_d), stack[ssize++].e = (_e);\ + } while(0) +#define STACK_POP(_a, _b, _c, _d)\ + do {\ + assert(0 <= ssize);\ + if(ssize == 0) { return; }\ + (_a) = stack[--ssize].a, (_b) = stack[ssize].b,\ + (_c) = stack[ssize].c, (_d) = stack[ssize].d;\ + } while(0) +#define STACK_POP5(_a, _b, _c, _d, _e)\ + do {\ + assert(0 <= ssize);\ + if(ssize == 0) { return; }\ + (_a) = stack[--ssize].a, (_b) = stack[ssize].b,\ + (_c) = stack[ssize].c, (_d) = stack[ssize].d, (_e) = stack[ssize].e;\ + } while(0) +#define BUCKET_A(_c0) bucket_A[(_c0)] +#if ALPHABET_SIZE == 256 +#define BUCKET_B(_c0, _c1) (bucket_B[((_c1) << 8) | (_c0)]) +#define BUCKET_BSTAR(_c0, _c1) (bucket_B[((_c0) << 8) | (_c1)]) +#else +#define BUCKET_B(_c0, _c1) (bucket_B[(_c1) * ALPHABET_SIZE + (_c0)]) +#define BUCKET_BSTAR(_c0, _c1) (bucket_B[(_c0) * ALPHABET_SIZE + (_c1)]) +#endif + + +/*- Private Functions -*/ + +static const int lg_table[256]= { + -1,0,1,1,2,2,2,2,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4, + 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, + 6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6, + 6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7 +}; + +#if (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE) + +static INLINE +int +ss_ilg(int n) { +#if SS_BLOCKSIZE == 0 + return (n & 0xffff0000) ? + ((n & 0xff000000) ? + 24 + lg_table[(n >> 24) & 0xff] : + 16 + lg_table[(n >> 16) & 0xff]) : + ((n & 0x0000ff00) ? + 8 + lg_table[(n >> 8) & 0xff] : + 0 + lg_table[(n >> 0) & 0xff]); +#elif SS_BLOCKSIZE < 256 + return lg_table[n]; +#else + return (n & 0xff00) ? + 8 + lg_table[(n >> 8) & 0xff] : + 0 + lg_table[(n >> 0) & 0xff]; +#endif +} + +#endif /* (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE) */ + +#if SS_BLOCKSIZE != 0 + +static const int sqq_table[256] = { + 0, 16, 22, 27, 32, 35, 39, 42, 45, 48, 50, 53, 55, 57, 59, 61, + 64, 65, 67, 69, 71, 73, 75, 76, 78, 80, 81, 83, 84, 86, 87, 89, + 90, 91, 93, 94, 96, 97, 98, 99, 101, 102, 103, 104, 106, 107, 108, 109, +110, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, +128, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, +143, 144, 144, 145, 146, 147, 148, 149, 150, 150, 151, 152, 153, 154, 155, 155, +156, 157, 158, 159, 160, 160, 161, 162, 163, 163, 164, 165, 166, 167, 167, 168, +169, 170, 170, 171, 172, 173, 173, 174, 175, 176, 176, 177, 178, 178, 179, 180, +181, 181, 182, 183, 183, 184, 185, 185, 186, 187, 187, 188, 189, 189, 190, 191, +192, 192, 193, 193, 194, 195, 195, 196, 197, 197, 198, 199, 199, 200, 201, 201, +202, 203, 203, 204, 204, 205, 206, 206, 207, 208, 208, 209, 209, 210, 211, 211, +212, 212, 213, 214, 214, 215, 215, 216, 217, 217, 218, 218, 219, 219, 220, 221, +221, 222, 222, 223, 224, 224, 225, 225, 226, 226, 227, 227, 228, 229, 229, 230, +230, 231, 231, 232, 232, 233, 234, 234, 235, 235, 236, 236, 237, 237, 238, 238, +239, 240, 240, 241, 241, 242, 242, 243, 243, 244, 244, 245, 245, 246, 246, 247, +247, 248, 248, 249, 249, 250, 250, 251, 251, 252, 252, 253, 253, 254, 254, 255 +}; + +static INLINE +int +ss_isqrt(int x) { + int y, e; + + if(x >= (SS_BLOCKSIZE * SS_BLOCKSIZE)) { return SS_BLOCKSIZE; } + e = (x & 0xffff0000) ? + ((x & 0xff000000) ? + 24 + lg_table[(x >> 24) & 0xff] : + 16 + lg_table[(x >> 16) & 0xff]) : + ((x & 0x0000ff00) ? + 8 + lg_table[(x >> 8) & 0xff] : + 0 + lg_table[(x >> 0) & 0xff]); + + if(e >= 16) { + y = sqq_table[x >> ((e - 6) - (e & 1))] << ((e >> 1) - 7); + if(e >= 24) { y = (y + 1 + x / y) >> 1; } + y = (y + 1 + x / y) >> 1; + } else if(e >= 8) { + y = (sqq_table[x >> ((e - 6) - (e & 1))] >> (7 - (e >> 1))) + 1; + } else { + return sqq_table[x] >> 4; + } + + return (x < (y * y)) ? y - 1 : y; +} + +#endif /* SS_BLOCKSIZE != 0 */ + + +/*---------------------------------------------------------------------------*/ + +/* Compares two suffixes. */ +static INLINE +int +ss_compare(const unsigned char *T, + const int *p1, const int *p2, + int depth) { + const unsigned char *U1, *U2, *U1n, *U2n; + + for(U1 = T + depth + *p1, + U2 = T + depth + *p2, + U1n = T + *(p1 + 1) + 2, + U2n = T + *(p2 + 1) + 2; + (U1 < U1n) && (U2 < U2n) && (*U1 == *U2); + ++U1, ++U2) { + } + + return U1 < U1n ? + (U2 < U2n ? *U1 - *U2 : 1) : + (U2 < U2n ? -1 : 0); +} + + +/*---------------------------------------------------------------------------*/ + +#if (SS_BLOCKSIZE != 1) && (SS_INSERTIONSORT_THRESHOLD != 1) + +/* Insertionsort for small size groups */ +static +void +ss_insertionsort(const unsigned char *T, const int *PA, + int *first, int *last, int depth) { + int *i, *j; + int t; + int r; + + for(i = last - 2; first <= i; --i) { + for(t = *i, j = i + 1; 0 < (r = ss_compare(T, PA + t, PA + *j, depth));) { + do { *(j - 1) = *j; } while((++j < last) && (*j < 0)); + if(last <= j) { break; } + } + if(r == 0) { *j = ~*j; } + *(j - 1) = t; + } +} + +#endif /* (SS_BLOCKSIZE != 1) && (SS_INSERTIONSORT_THRESHOLD != 1) */ + + +/*---------------------------------------------------------------------------*/ + +#if (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE) + +static INLINE +void +ss_fixdown(const unsigned char *Td, const int *PA, + int *SA, int i, int size) { + int j, k; + int v; + int c, d, e; + + for(v = SA[i], c = Td[PA[v]]; (j = 2 * i + 1) < size; SA[i] = SA[k], i = k) { + d = Td[PA[SA[k = j++]]]; + if(d < (e = Td[PA[SA[j]]])) { k = j; d = e; } + if(d <= c) { break; } + } + SA[i] = v; +} + +/* Simple top-down heapsort. */ +static +void +ss_heapsort(const unsigned char *Td, const int *PA, int *SA, int size) { + int i, m; + int t; + + m = size; + if((size % 2) == 0) { + m--; + if(Td[PA[SA[m / 2]]] < Td[PA[SA[m]]]) { SWAP(SA[m], SA[m / 2]); } + } + + for(i = m / 2 - 1; 0 <= i; --i) { ss_fixdown(Td, PA, SA, i, m); } + if((size % 2) == 0) { SWAP(SA[0], SA[m]); ss_fixdown(Td, PA, SA, 0, m); } + for(i = m - 1; 0 < i; --i) { + t = SA[0], SA[0] = SA[i]; + ss_fixdown(Td, PA, SA, 0, i); + SA[i] = t; + } +} + + +/*---------------------------------------------------------------------------*/ + +/* Returns the median of three elements. */ +static INLINE +int * +ss_median3(const unsigned char *Td, const int *PA, + int *v1, int *v2, int *v3) { + int *t; + if(Td[PA[*v1]] > Td[PA[*v2]]) { SWAP(v1, v2); } + if(Td[PA[*v2]] > Td[PA[*v3]]) { + if(Td[PA[*v1]] > Td[PA[*v3]]) { return v1; } + else { return v3; } + } + return v2; +} + +/* Returns the median of five elements. */ +static INLINE +int * +ss_median5(const unsigned char *Td, const int *PA, + int *v1, int *v2, int *v3, int *v4, int *v5) { + int *t; + if(Td[PA[*v2]] > Td[PA[*v3]]) { SWAP(v2, v3); } + if(Td[PA[*v4]] > Td[PA[*v5]]) { SWAP(v4, v5); } + if(Td[PA[*v2]] > Td[PA[*v4]]) { SWAP(v2, v4); SWAP(v3, v5); } + if(Td[PA[*v1]] > Td[PA[*v3]]) { SWAP(v1, v3); } + if(Td[PA[*v1]] > Td[PA[*v4]]) { SWAP(v1, v4); SWAP(v3, v5); } + if(Td[PA[*v3]] > Td[PA[*v4]]) { return v4; } + return v3; +} + +/* Returns the pivot element. */ +static INLINE +int * +ss_pivot(const unsigned char *Td, const int *PA, int *first, int *last) { + int *middle; + int t; + + t = last - first; + middle = first + t / 2; + + if(t <= 512) { + if(t <= 32) { + return ss_median3(Td, PA, first, middle, last - 1); + } else { + t >>= 2; + return ss_median5(Td, PA, first, first + t, middle, last - 1 - t, last - 1); + } + } + t >>= 3; + first = ss_median3(Td, PA, first, first + t, first + (t << 1)); + middle = ss_median3(Td, PA, middle - t, middle, middle + t); + last = ss_median3(Td, PA, last - 1 - (t << 1), last - 1 - t, last - 1); + return ss_median3(Td, PA, first, middle, last); +} + + +/*---------------------------------------------------------------------------*/ + +/* Binary partition for substrings. */ +static INLINE +int * +ss_partition(const int *PA, + int *first, int *last, int depth) { + int *a, *b; + int t; + for(a = first - 1, b = last;;) { + for(; (++a < b) && ((PA[*a] + depth) >= (PA[*a + 1] + 1));) { *a = ~*a; } + for(; (a < --b) && ((PA[*b] + depth) < (PA[*b + 1] + 1));) { } + if(b <= a) { break; } + t = ~*b; + *b = *a; + *a = t; + } + if(first < a) { *first = ~*first; } + return a; +} + +/* Multikey introsort for medium size groups. */ +static +void +ss_mintrosort(const unsigned char *T, const int *PA, + int *first, int *last, + int depth) { +#define STACK_SIZE SS_MISORT_STACKSIZE + struct { int *a, *b, c; int d; } stack[STACK_SIZE]; + const unsigned char *Td; + int *a, *b, *c, *d, *e, *f; + int s, t; + int ssize; + int limit; + int v, x = 0; + + for(ssize = 0, limit = ss_ilg(last - first);;) { + + if((last - first) <= SS_INSERTIONSORT_THRESHOLD) { +#if 1 < SS_INSERTIONSORT_THRESHOLD + if(1 < (last - first)) { ss_insertionsort(T, PA, first, last, depth); } +#endif + STACK_POP(first, last, depth, limit); + continue; + } + + Td = T + depth; + if(limit-- == 0) { ss_heapsort(Td, PA, first, last - first); } + if(limit < 0) { + for(a = first + 1, v = Td[PA[*first]]; a < last; ++a) { + if((x = Td[PA[*a]]) != v) { + if(1 < (a - first)) { break; } + v = x; + first = a; + } + } + if(Td[PA[*first] - 1] < v) { + first = ss_partition(PA, first, a, depth); + } + if((a - first) <= (last - a)) { + if(1 < (a - first)) { + STACK_PUSH(a, last, depth, -1); + last = a, depth += 1, limit = ss_ilg(a - first); + } else { + first = a, limit = -1; + } + } else { + if(1 < (last - a)) { + STACK_PUSH(first, a, depth + 1, ss_ilg(a - first)); + first = a, limit = -1; + } else { + last = a, depth += 1, limit = ss_ilg(a - first); + } + } + continue; + } + + /* choose pivot */ + a = ss_pivot(Td, PA, first, last); + v = Td[PA[*a]]; + SWAP(*first, *a); + + /* partition */ + for(b = first; (++b < last) && ((x = Td[PA[*b]]) == v);) { } + if(((a = b) < last) && (x < v)) { + for(; (++b < last) && ((x = Td[PA[*b]]) <= v);) { + if(x == v) { SWAP(*b, *a); ++a; } + } + } + for(c = last; (b < --c) && ((x = Td[PA[*c]]) == v);) { } + if((b < (d = c)) && (x > v)) { + for(; (b < --c) && ((x = Td[PA[*c]]) >= v);) { + if(x == v) { SWAP(*c, *d); --d; } + } + } + for(; b < c;) { + SWAP(*b, *c); + for(; (++b < c) && ((x = Td[PA[*b]]) <= v);) { + if(x == v) { SWAP(*b, *a); ++a; } + } + for(; (b < --c) && ((x = Td[PA[*c]]) >= v);) { + if(x == v) { SWAP(*c, *d); --d; } + } + } + + if(a <= d) { + c = b - 1; + + if((s = a - first) > (t = b - a)) { s = t; } + for(e = first, f = b - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); } + if((s = d - c) > (t = last - d - 1)) { s = t; } + for(e = b, f = last - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); } + + a = first + (b - a), c = last - (d - c); + b = (v <= Td[PA[*a] - 1]) ? a : ss_partition(PA, a, c, depth); + + if((a - first) <= (last - c)) { + if((last - c) <= (c - b)) { + STACK_PUSH(b, c, depth + 1, ss_ilg(c - b)); + STACK_PUSH(c, last, depth, limit); + last = a; + } else if((a - first) <= (c - b)) { + STACK_PUSH(c, last, depth, limit); + STACK_PUSH(b, c, depth + 1, ss_ilg(c - b)); + last = a; + } else { + STACK_PUSH(c, last, depth, limit); + STACK_PUSH(first, a, depth, limit); + first = b, last = c, depth += 1, limit = ss_ilg(c - b); + } + } else { + if((a - first) <= (c - b)) { + STACK_PUSH(b, c, depth + 1, ss_ilg(c - b)); + STACK_PUSH(first, a, depth, limit); + first = c; + } else if((last - c) <= (c - b)) { + STACK_PUSH(first, a, depth, limit); + STACK_PUSH(b, c, depth + 1, ss_ilg(c - b)); + first = c; + } else { + STACK_PUSH(first, a, depth, limit); + STACK_PUSH(c, last, depth, limit); + first = b, last = c, depth += 1, limit = ss_ilg(c - b); + } + } + } else { + limit += 1; + if(Td[PA[*first] - 1] < v) { + first = ss_partition(PA, first, last, depth); + limit = ss_ilg(last - first); + } + depth += 1; + } + } +#undef STACK_SIZE +} + +#endif /* (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE) */ + + +/*---------------------------------------------------------------------------*/ + +#if SS_BLOCKSIZE != 0 + +static INLINE +void +ss_blockswap(int *a, int *b, int n) { + int t; + for(; 0 < n; --n, ++a, ++b) { + t = *a, *a = *b, *b = t; + } +} + +static INLINE +void +ss_rotate(int *first, int *middle, int *last) { + int *a, *b, t; + int l, r; + l = middle - first, r = last - middle; + for(; (0 < l) && (0 < r);) { + if(l == r) { ss_blockswap(first, middle, l); break; } + if(l < r) { + a = last - 1, b = middle - 1; + t = *a; + do { + *a-- = *b, *b-- = *a; + if(b < first) { + *a = t; + last = a; + if((r -= l + 1) <= l) { break; } + a -= 1, b = middle - 1; + t = *a; + } + } while(1); + } else { + a = first, b = middle; + t = *a; + do { + *a++ = *b, *b++ = *a; + if(last <= b) { + *a = t; + first = a + 1; + if((l -= r + 1) <= r) { break; } + a += 1, b = middle; + t = *a; + } + } while(1); + } + } +} + + +/*---------------------------------------------------------------------------*/ + +static +void +ss_inplacemerge(const unsigned char *T, const int *PA, + int *first, int *middle, int *last, + int depth) { + const int *p; + int *a, *b; + int len, half; + int q, r; + int x; + + for(;;) { + if(*(last - 1) < 0) { x = 1; p = PA + ~*(last - 1); } + else { x = 0; p = PA + *(last - 1); } + for(a = first, len = middle - first, half = len >> 1, r = -1; + 0 < len; + len = half, half >>= 1) { + b = a + half; + q = ss_compare(T, PA + ((0 <= *b) ? *b : ~*b), p, depth); + if(q < 0) { + a = b + 1; + half -= (len & 1) ^ 1; + } else { + r = q; + } + } + if(a < middle) { + if(r == 0) { *a = ~*a; } + ss_rotate(a, middle, last); + last -= middle - a; + middle = a; + if(first == middle) { break; } + } + --last; + if(x != 0) { while(*--last < 0) { } } + if(middle == last) { break; } + } +} + + +/*---------------------------------------------------------------------------*/ + +/* Merge-forward with internal buffer. */ +static +void +ss_mergeforward(const unsigned char *T, const int *PA, + int *first, int *middle, int *last, + int *buf, int depth) { + int *a, *b, *c, *bufend; + int t; + int r; + + bufend = buf + (middle - first) - 1; + ss_blockswap(buf, first, middle - first); + + for(t = *(a = first), b = buf, c = middle;;) { + r = ss_compare(T, PA + *b, PA + *c, depth); + if(r < 0) { + do { + *a++ = *b; + if(bufend <= b) { *bufend = t; return; } + *b++ = *a; + } while(*b < 0); + } else if(r > 0) { + do { + *a++ = *c, *c++ = *a; + if(last <= c) { + while(b < bufend) { *a++ = *b, *b++ = *a; } + *a = *b, *b = t; + return; + } + } while(*c < 0); + } else { + *c = ~*c; + do { + *a++ = *b; + if(bufend <= b) { *bufend = t; return; } + *b++ = *a; + } while(*b < 0); + + do { + *a++ = *c, *c++ = *a; + if(last <= c) { + while(b < bufend) { *a++ = *b, *b++ = *a; } + *a = *b, *b = t; + return; + } + } while(*c < 0); + } + } +} + +/* Merge-backward with internal buffer. */ +static +void +ss_mergebackward(const unsigned char *T, const int *PA, + int *first, int *middle, int *last, + int *buf, int depth) { + const int *p1, *p2; + int *a, *b, *c, *bufend; + int t; + int r; + int x; + + bufend = buf + (last - middle) - 1; + ss_blockswap(buf, middle, last - middle); + + x = 0; + if(*bufend < 0) { p1 = PA + ~*bufend; x |= 1; } + else { p1 = PA + *bufend; } + if(*(middle - 1) < 0) { p2 = PA + ~*(middle - 1); x |= 2; } + else { p2 = PA + *(middle - 1); } + for(t = *(a = last - 1), b = bufend, c = middle - 1;;) { + r = ss_compare(T, p1, p2, depth); + if(0 < r) { + if(x & 1) { do { *a-- = *b, *b-- = *a; } while(*b < 0); x ^= 1; } + *a-- = *b; + if(b <= buf) { *buf = t; break; } + *b-- = *a; + if(*b < 0) { p1 = PA + ~*b; x |= 1; } + else { p1 = PA + *b; } + } else if(r < 0) { + if(x & 2) { do { *a-- = *c, *c-- = *a; } while(*c < 0); x ^= 2; } + *a-- = *c, *c-- = *a; + if(c < first) { + while(buf < b) { *a-- = *b, *b-- = *a; } + *a = *b, *b = t; + break; + } + if(*c < 0) { p2 = PA + ~*c; x |= 2; } + else { p2 = PA + *c; } + } else { + if(x & 1) { do { *a-- = *b, *b-- = *a; } while(*b < 0); x ^= 1; } + *a-- = ~*b; + if(b <= buf) { *buf = t; break; } + *b-- = *a; + if(x & 2) { do { *a-- = *c, *c-- = *a; } while(*c < 0); x ^= 2; } + *a-- = *c, *c-- = *a; + if(c < first) { + while(buf < b) { *a-- = *b, *b-- = *a; } + *a = *b, *b = t; + break; + } + if(*b < 0) { p1 = PA + ~*b; x |= 1; } + else { p1 = PA + *b; } + if(*c < 0) { p2 = PA + ~*c; x |= 2; } + else { p2 = PA + *c; } + } + } +} + +/* D&C based merge. */ +static +void +ss_swapmerge(const unsigned char *T, const int *PA, + int *first, int *middle, int *last, + int *buf, int bufsize, int depth) { +#define STACK_SIZE SS_SMERGE_STACKSIZE +#define GETIDX(a) ((0 <= (a)) ? (a) : (~(a))) +#define MERGE_CHECK(a, b, c)\ + do {\ + if(((c) & 1) ||\ + (((c) & 2) && (ss_compare(T, PA + GETIDX(*((a) - 1)), PA + *(a), depth) == 0))) {\ + *(a) = ~*(a);\ + }\ + if(((c) & 4) && ((ss_compare(T, PA + GETIDX(*((b) - 1)), PA + *(b), depth) == 0))) {\ + *(b) = ~*(b);\ + }\ + } while(0) + struct { int *a, *b, *c; int d; } stack[STACK_SIZE]; + int *l, *r, *lm, *rm; + int m, len, half; + int ssize; + int check, next; + + for(check = 0, ssize = 0;;) { + if((last - middle) <= bufsize) { + if((first < middle) && (middle < last)) { + ss_mergebackward(T, PA, first, middle, last, buf, depth); + } + MERGE_CHECK(first, last, check); + STACK_POP(first, middle, last, check); + continue; + } + + if((middle - first) <= bufsize) { + if(first < middle) { + ss_mergeforward(T, PA, first, middle, last, buf, depth); + } + MERGE_CHECK(first, last, check); + STACK_POP(first, middle, last, check); + continue; + } + + for(m = 0, len = MIN(middle - first, last - middle), half = len >> 1; + 0 < len; + len = half, half >>= 1) { + if(ss_compare(T, PA + GETIDX(*(middle + m + half)), + PA + GETIDX(*(middle - m - half - 1)), depth) < 0) { + m += half + 1; + half -= (len & 1) ^ 1; + } + } + + if(0 < m) { + lm = middle - m, rm = middle + m; + ss_blockswap(lm, middle, m); + l = r = middle, next = 0; + if(rm < last) { + if(*rm < 0) { + *rm = ~*rm; + if(first < lm) { for(; *--l < 0;) { } next |= 4; } + next |= 1; + } else if(first < lm) { + for(; *r < 0; ++r) { } + next |= 2; + } + } + + if((l - first) <= (last - r)) { + STACK_PUSH(r, rm, last, (next & 3) | (check & 4)); + middle = lm, last = l, check = (check & 3) | (next & 4); + } else { + if((next & 2) && (r == middle)) { next ^= 6; } + STACK_PUSH(first, lm, l, (check & 3) | (next & 4)); + first = r, middle = rm, check = (next & 3) | (check & 4); + } + } else { + if(ss_compare(T, PA + GETIDX(*(middle - 1)), PA + *middle, depth) == 0) { + *middle = ~*middle; + } + MERGE_CHECK(first, last, check); + STACK_POP(first, middle, last, check); + } + } +#undef STACK_SIZE +} + +#endif /* SS_BLOCKSIZE != 0 */ + + +/*---------------------------------------------------------------------------*/ + +/* Substring sort */ +static +void +sssort(const unsigned char *T, const int *PA, + int *first, int *last, + int *buf, int bufsize, + int depth, int n, int lastsuffix) { + int *a; +#if SS_BLOCKSIZE != 0 + int *b, *middle, *curbuf; + int j, k, curbufsize, limit; +#endif + int i; + + if(lastsuffix != 0) { ++first; } + +#if SS_BLOCKSIZE == 0 + ss_mintrosort(T, PA, first, last, depth); +#else + if((bufsize < SS_BLOCKSIZE) && + (bufsize < (last - first)) && + (bufsize < (limit = ss_isqrt(last - first)))) { + if(SS_BLOCKSIZE < limit) { limit = SS_BLOCKSIZE; } + buf = middle = last - limit, bufsize = limit; + } else { + middle = last, limit = 0; + } + for(a = first, i = 0; SS_BLOCKSIZE < (middle - a); a += SS_BLOCKSIZE, ++i) { +#if SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE + ss_mintrosort(T, PA, a, a + SS_BLOCKSIZE, depth); +#elif 1 < SS_BLOCKSIZE + ss_insertionsort(T, PA, a, a + SS_BLOCKSIZE, depth); +#endif + curbufsize = last - (a + SS_BLOCKSIZE); + curbuf = a + SS_BLOCKSIZE; + if(curbufsize <= bufsize) { curbufsize = bufsize, curbuf = buf; } + for(b = a, k = SS_BLOCKSIZE, j = i; j & 1; b -= k, k <<= 1, j >>= 1) { + ss_swapmerge(T, PA, b - k, b, b + k, curbuf, curbufsize, depth); + } + } +#if SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE + ss_mintrosort(T, PA, a, middle, depth); +#elif 1 < SS_BLOCKSIZE + ss_insertionsort(T, PA, a, middle, depth); +#endif + for(k = SS_BLOCKSIZE; i != 0; k <<= 1, i >>= 1) { + if(i & 1) { + ss_swapmerge(T, PA, a - k, a, middle, buf, bufsize, depth); + a -= k; + } + } + if(limit != 0) { +#if SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE + ss_mintrosort(T, PA, middle, last, depth); +#elif 1 < SS_BLOCKSIZE + ss_insertionsort(T, PA, middle, last, depth); +#endif + ss_inplacemerge(T, PA, first, middle, last, depth); + } +#endif + + if(lastsuffix != 0) { + /* Insert last type B* suffix. */ + int PAi[2]; PAi[0] = PA[*(first - 1)], PAi[1] = n - 2; + for(a = first, i = *(first - 1); + (a < last) && ((*a < 0) || (0 < ss_compare(T, &(PAi[0]), PA + *a, depth))); + ++a) { + *(a - 1) = *a; + } + *(a - 1) = i; + } +} + + +/*---------------------------------------------------------------------------*/ + +static INLINE +int +tr_ilg(int n) { + return (n & 0xffff0000) ? + ((n & 0xff000000) ? + 24 + lg_table[(n >> 24) & 0xff] : + 16 + lg_table[(n >> 16) & 0xff]) : + ((n & 0x0000ff00) ? + 8 + lg_table[(n >> 8) & 0xff] : + 0 + lg_table[(n >> 0) & 0xff]); +} + + +/*---------------------------------------------------------------------------*/ + +/* Simple insertionsort for small size groups. */ +static +void +tr_insertionsort(const int *ISAd, int *first, int *last) { + int *a, *b; + int t, r; + + for(a = first + 1; a < last; ++a) { + for(t = *a, b = a - 1; 0 > (r = ISAd[t] - ISAd[*b]);) { + do { *(b + 1) = *b; } while((first <= --b) && (*b < 0)); + if(b < first) { break; } + } + if(r == 0) { *b = ~*b; } + *(b + 1) = t; + } +} + + +/*---------------------------------------------------------------------------*/ + +static INLINE +void +tr_fixdown(const int *ISAd, int *SA, int i, int size) { + int j, k; + int v; + int c, d, e; + + for(v = SA[i], c = ISAd[v]; (j = 2 * i + 1) < size; SA[i] = SA[k], i = k) { + d = ISAd[SA[k = j++]]; + if(d < (e = ISAd[SA[j]])) { k = j; d = e; } + if(d <= c) { break; } + } + SA[i] = v; +} + +/* Simple top-down heapsort. */ +static +void +tr_heapsort(const int *ISAd, int *SA, int size) { + int i, m; + int t; + + m = size; + if((size % 2) == 0) { + m--; + if(ISAd[SA[m / 2]] < ISAd[SA[m]]) { SWAP(SA[m], SA[m / 2]); } + } + + for(i = m / 2 - 1; 0 <= i; --i) { tr_fixdown(ISAd, SA, i, m); } + if((size % 2) == 0) { SWAP(SA[0], SA[m]); tr_fixdown(ISAd, SA, 0, m); } + for(i = m - 1; 0 < i; --i) { + t = SA[0], SA[0] = SA[i]; + tr_fixdown(ISAd, SA, 0, i); + SA[i] = t; + } +} + + +/*---------------------------------------------------------------------------*/ + +/* Returns the median of three elements. */ +static INLINE +int * +tr_median3(const int *ISAd, int *v1, int *v2, int *v3) { + int *t; + if(ISAd[*v1] > ISAd[*v2]) { SWAP(v1, v2); } + if(ISAd[*v2] > ISAd[*v3]) { + if(ISAd[*v1] > ISAd[*v3]) { return v1; } + else { return v3; } + } + return v2; +} + +/* Returns the median of five elements. */ +static INLINE +int * +tr_median5(const int *ISAd, + int *v1, int *v2, int *v3, int *v4, int *v5) { + int *t; + if(ISAd[*v2] > ISAd[*v3]) { SWAP(v2, v3); } + if(ISAd[*v4] > ISAd[*v5]) { SWAP(v4, v5); } + if(ISAd[*v2] > ISAd[*v4]) { SWAP(v2, v4); SWAP(v3, v5); } + if(ISAd[*v1] > ISAd[*v3]) { SWAP(v1, v3); } + if(ISAd[*v1] > ISAd[*v4]) { SWAP(v1, v4); SWAP(v3, v5); } + if(ISAd[*v3] > ISAd[*v4]) { return v4; } + return v3; +} + +/* Returns the pivot element. */ +static INLINE +int * +tr_pivot(const int *ISAd, int *first, int *last) { + int *middle; + int t; + + t = last - first; + middle = first + t / 2; + + if(t <= 512) { + if(t <= 32) { + return tr_median3(ISAd, first, middle, last - 1); + } else { + t >>= 2; + return tr_median5(ISAd, first, first + t, middle, last - 1 - t, last - 1); + } + } + t >>= 3; + first = tr_median3(ISAd, first, first + t, first + (t << 1)); + middle = tr_median3(ISAd, middle - t, middle, middle + t); + last = tr_median3(ISAd, last - 1 - (t << 1), last - 1 - t, last - 1); + return tr_median3(ISAd, first, middle, last); +} + + +/*---------------------------------------------------------------------------*/ + +typedef struct _trbudget_t trbudget_t; +struct _trbudget_t { + int chance; + int remain; + int incval; + int count; +}; + +static INLINE +void +trbudget_init(trbudget_t *budget, int chance, int incval) { + budget->chance = chance; + budget->remain = budget->incval = incval; +} + +static INLINE +int +trbudget_check(trbudget_t *budget, int size) { + if(size <= budget->remain) { budget->remain -= size; return 1; } + if(budget->chance == 0) { budget->count += size; return 0; } + budget->remain += budget->incval - size; + budget->chance -= 1; + return 1; +} + + +/*---------------------------------------------------------------------------*/ + +static INLINE +void +tr_partition(const int *ISAd, + int *first, int *middle, int *last, + int **pa, int **pb, int v) { + int *a, *b, *c, *d, *e, *f; + int t, s; + int x = 0; + + for(b = middle - 1; (++b < last) && ((x = ISAd[*b]) == v);) { } + if(((a = b) < last) && (x < v)) { + for(; (++b < last) && ((x = ISAd[*b]) <= v);) { + if(x == v) { SWAP(*b, *a); ++a; } + } + } + for(c = last; (b < --c) && ((x = ISAd[*c]) == v);) { } + if((b < (d = c)) && (x > v)) { + for(; (b < --c) && ((x = ISAd[*c]) >= v);) { + if(x == v) { SWAP(*c, *d); --d; } + } + } + for(; b < c;) { + SWAP(*b, *c); + for(; (++b < c) && ((x = ISAd[*b]) <= v);) { + if(x == v) { SWAP(*b, *a); ++a; } + } + for(; (b < --c) && ((x = ISAd[*c]) >= v);) { + if(x == v) { SWAP(*c, *d); --d; } + } + } + + if(a <= d) { + c = b - 1; + if((s = a - first) > (t = b - a)) { s = t; } + for(e = first, f = b - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); } + if((s = d - c) > (t = last - d - 1)) { s = t; } + for(e = b, f = last - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); } + first += (b - a), last -= (d - c); + } + *pa = first, *pb = last; +} + +static +void +tr_copy(int *ISA, const int *SA, + int *first, int *a, int *b, int *last, + int depth) { + /* sort suffixes of middle partition + by using sorted order of suffixes of left and right partition. */ + int *c, *d, *e; + int s, v; + + v = b - SA - 1; + for(c = first, d = a - 1; c <= d; ++c) { + if((0 <= (s = *c - depth)) && (ISA[s] == v)) { + *++d = s; + ISA[s] = d - SA; + } + } + for(c = last - 1, e = d + 1, d = b; e < d; --c) { + if((0 <= (s = *c - depth)) && (ISA[s] == v)) { + *--d = s; + ISA[s] = d - SA; + } + } +} + +static +void +tr_partialcopy(int *ISA, const int *SA, + int *first, int *a, int *b, int *last, + int depth) { + int *c, *d, *e; + int s, v; + int rank, lastrank, newrank = -1; + + v = b - SA - 1; + lastrank = -1; + for(c = first, d = a - 1; c <= d; ++c) { + if((0 <= (s = *c - depth)) && (ISA[s] == v)) { + *++d = s; + rank = ISA[s + depth]; + if(lastrank != rank) { lastrank = rank; newrank = d - SA; } + ISA[s] = newrank; + } + } + + lastrank = -1; + for(e = d; first <= e; --e) { + rank = ISA[*e]; + if(lastrank != rank) { lastrank = rank; newrank = e - SA; } + if(newrank != rank) { ISA[*e] = newrank; } + } + + lastrank = -1; + for(c = last - 1, e = d + 1, d = b; e < d; --c) { + if((0 <= (s = *c - depth)) && (ISA[s] == v)) { + *--d = s; + rank = ISA[s + depth]; + if(lastrank != rank) { lastrank = rank; newrank = d - SA; } + ISA[s] = newrank; + } + } +} + +static +void +tr_introsort(int *ISA, const int *ISAd, + int *SA, int *first, int *last, + trbudget_t *budget) { +#define STACK_SIZE TR_STACKSIZE + struct { const int *a; int *b, *c; int d, e; }stack[STACK_SIZE]; + int *a, *b, *c; + int t; + int v, x = 0; + int incr = ISAd - ISA; + int limit, next; + int ssize, trlink = -1; + + for(ssize = 0, limit = tr_ilg(last - first);;) { + + if(limit < 0) { + if(limit == -1) { + /* tandem repeat partition */ + tr_partition(ISAd - incr, first, first, last, &a, &b, last - SA - 1); + + /* update ranks */ + if(a < last) { + for(c = first, v = a - SA - 1; c < a; ++c) { ISA[*c] = v; } + } + if(b < last) { + for(c = a, v = b - SA - 1; c < b; ++c) { ISA[*c] = v; } + } + + /* push */ + if(1 < (b - a)) { + STACK_PUSH5(NULL, a, b, 0, 0); + STACK_PUSH5(ISAd - incr, first, last, -2, trlink); + trlink = ssize - 2; + } + if((a - first) <= (last - b)) { + if(1 < (a - first)) { + STACK_PUSH5(ISAd, b, last, tr_ilg(last - b), trlink); + last = a, limit = tr_ilg(a - first); + } else if(1 < (last - b)) { + first = b, limit = tr_ilg(last - b); + } else { + STACK_POP5(ISAd, first, last, limit, trlink); + } + } else { + if(1 < (last - b)) { + STACK_PUSH5(ISAd, first, a, tr_ilg(a - first), trlink); + first = b, limit = tr_ilg(last - b); + } else if(1 < (a - first)) { + last = a, limit = tr_ilg(a - first); + } else { + STACK_POP5(ISAd, first, last, limit, trlink); + } + } + } else if(limit == -2) { + /* tandem repeat copy */ + a = stack[--ssize].b, b = stack[ssize].c; + if(stack[ssize].d == 0) { + tr_copy(ISA, SA, first, a, b, last, ISAd - ISA); + } else { + if(0 <= trlink) { stack[trlink].d = -1; } + tr_partialcopy(ISA, SA, first, a, b, last, ISAd - ISA); + } + STACK_POP5(ISAd, first, last, limit, trlink); + } else { + /* sorted partition */ + if(0 <= *first) { + a = first; + do { ISA[*a] = a - SA; } while((++a < last) && (0 <= *a)); + first = a; + } + if(first < last) { + a = first; do { *a = ~*a; } while(*++a < 0); + next = (ISA[*a] != ISAd[*a]) ? tr_ilg(a - first + 1) : -1; + if(++a < last) { for(b = first, v = a - SA - 1; b < a; ++b) { ISA[*b] = v; } } + + /* push */ + if(trbudget_check(budget, a - first)) { + if((a - first) <= (last - a)) { + STACK_PUSH5(ISAd, a, last, -3, trlink); + ISAd += incr, last = a, limit = next; + } else { + if(1 < (last - a)) { + STACK_PUSH5(ISAd + incr, first, a, next, trlink); + first = a, limit = -3; + } else { + ISAd += incr, last = a, limit = next; + } + } + } else { + if(0 <= trlink) { stack[trlink].d = -1; } + if(1 < (last - a)) { + first = a, limit = -3; + } else { + STACK_POP5(ISAd, first, last, limit, trlink); + } + } + } else { + STACK_POP5(ISAd, first, last, limit, trlink); + } + } + continue; + } + + if((last - first) <= TR_INSERTIONSORT_THRESHOLD) { + tr_insertionsort(ISAd, first, last); + limit = -3; + continue; + } + + if(limit-- == 0) { + tr_heapsort(ISAd, first, last - first); + for(a = last - 1; first < a; a = b) { + for(x = ISAd[*a], b = a - 1; (first <= b) && (ISAd[*b] == x); --b) { *b = ~*b; } + } + limit = -3; + continue; + } + + /* choose pivot */ + a = tr_pivot(ISAd, first, last); + SWAP(*first, *a); + v = ISAd[*first]; + + /* partition */ + tr_partition(ISAd, first, first + 1, last, &a, &b, v); + if((last - first) != (b - a)) { + next = (ISA[*a] != v) ? tr_ilg(b - a) : -1; + + /* update ranks */ + for(c = first, v = a - SA - 1; c < a; ++c) { ISA[*c] = v; } + if(b < last) { for(c = a, v = b - SA - 1; c < b; ++c) { ISA[*c] = v; } } + + /* push */ + if((1 < (b - a)) && (trbudget_check(budget, b - a))) { + if((a - first) <= (last - b)) { + if((last - b) <= (b - a)) { + if(1 < (a - first)) { + STACK_PUSH5(ISAd + incr, a, b, next, trlink); + STACK_PUSH5(ISAd, b, last, limit, trlink); + last = a; + } else if(1 < (last - b)) { + STACK_PUSH5(ISAd + incr, a, b, next, trlink); + first = b; + } else { + ISAd += incr, first = a, last = b, limit = next; + } + } else if((a - first) <= (b - a)) { + if(1 < (a - first)) { + STACK_PUSH5(ISAd, b, last, limit, trlink); + STACK_PUSH5(ISAd + incr, a, b, next, trlink); + last = a; + } else { + STACK_PUSH5(ISAd, b, last, limit, trlink); + ISAd += incr, first = a, last = b, limit = next; + } + } else { + STACK_PUSH5(ISAd, b, last, limit, trlink); + STACK_PUSH5(ISAd, first, a, limit, trlink); + ISAd += incr, first = a, last = b, limit = next; + } + } else { + if((a - first) <= (b - a)) { + if(1 < (last - b)) { + STACK_PUSH5(ISAd + incr, a, b, next, trlink); + STACK_PUSH5(ISAd, first, a, limit, trlink); + first = b; + } else if(1 < (a - first)) { + STACK_PUSH5(ISAd + incr, a, b, next, trlink); + last = a; + } else { + ISAd += incr, first = a, last = b, limit = next; + } + } else if((last - b) <= (b - a)) { + if(1 < (last - b)) { + STACK_PUSH5(ISAd, first, a, limit, trlink); + STACK_PUSH5(ISAd + incr, a, b, next, trlink); + first = b; + } else { + STACK_PUSH5(ISAd, first, a, limit, trlink); + ISAd += incr, first = a, last = b, limit = next; + } + } else { + STACK_PUSH5(ISAd, first, a, limit, trlink); + STACK_PUSH5(ISAd, b, last, limit, trlink); + ISAd += incr, first = a, last = b, limit = next; + } + } + } else { + if((1 < (b - a)) && (0 <= trlink)) { stack[trlink].d = -1; } + if((a - first) <= (last - b)) { + if(1 < (a - first)) { + STACK_PUSH5(ISAd, b, last, limit, trlink); + last = a; + } else if(1 < (last - b)) { + first = b; + } else { + STACK_POP5(ISAd, first, last, limit, trlink); + } + } else { + if(1 < (last - b)) { + STACK_PUSH5(ISAd, first, a, limit, trlink); + first = b; + } else if(1 < (a - first)) { + last = a; + } else { + STACK_POP5(ISAd, first, last, limit, trlink); + } + } + } + } else { + if(trbudget_check(budget, last - first)) { + limit = tr_ilg(last - first), ISAd += incr; + } else { + if(0 <= trlink) { stack[trlink].d = -1; } + STACK_POP5(ISAd, first, last, limit, trlink); + } + } + } +#undef STACK_SIZE +} + + + +/*---------------------------------------------------------------------------*/ + +/* Tandem repeat sort */ +static +void +trsort(int *ISA, int *SA, int n, int depth) { + int *ISAd; + int *first, *last; + trbudget_t budget; + int t, skip, unsorted; + + trbudget_init(&budget, tr_ilg(n) * 2 / 3, n); +/* trbudget_init(&budget, tr_ilg(n) * 3 / 4, n); */ + for(ISAd = ISA + depth; -n < *SA; ISAd += ISAd - ISA) { + first = SA; + skip = 0; + unsorted = 0; + do { + if((t = *first) < 0) { first -= t; skip += t; } + else { + if(skip != 0) { *(first + skip) = skip; skip = 0; } + last = SA + ISA[t] + 1; + if(1 < (last - first)) { + budget.count = 0; + tr_introsort(ISA, ISAd, SA, first, last, &budget); + if(budget.count != 0) { unsorted += budget.count; } + else { skip = first - last; } + } else if((last - first) == 1) { + skip = -1; + } + first = last; + } + } while(first < (SA + n)); + if(skip != 0) { *(first + skip) = skip; } + if(unsorted == 0) { break; } + } +} + + +/*---------------------------------------------------------------------------*/ + +/* Sorts suffixes of type B*. */ +static +int +sort_typeBstar(const unsigned char *T, int *SA, + int *bucket_A, int *bucket_B, + int n, int openMP) { + int *PAb, *ISAb, *buf; +#ifdef LIBBSC_OPENMP + int *curbuf; + int l; +#endif + int i, j, k, t, m, bufsize; + int c0, c1; +#ifdef LIBBSC_OPENMP + int d0, d1; +#endif + (void)openMP; + + /* Initialize bucket arrays. */ + for(i = 0; i < BUCKET_A_SIZE; ++i) { bucket_A[i] = 0; } + for(i = 0; i < BUCKET_B_SIZE; ++i) { bucket_B[i] = 0; } + + /* Count the number of occurrences of the first one or two characters of each + type A, B and B* suffix. Moreover, store the beginning position of all + type B* suffixes into the array SA. */ + for(i = n - 1, m = n, c0 = T[n - 1]; 0 <= i;) { + /* type A suffix. */ + do { ++BUCKET_A(c1 = c0); } while((0 <= --i) && ((c0 = T[i]) >= c1)); + if(0 <= i) { + /* type B* suffix. */ + ++BUCKET_BSTAR(c0, c1); + SA[--m] = i; + /* type B suffix. */ + for(--i, c1 = c0; (0 <= i) && ((c0 = T[i]) <= c1); --i, c1 = c0) { + ++BUCKET_B(c0, c1); + } + } + } + m = n - m; +/* +note: + A type B* suffix is lexicographically smaller than a type B suffix that + begins with the same first two characters. +*/ + + /* Calculate the index of start/end point of each bucket. */ + for(c0 = 0, i = 0, j = 0; c0 < ALPHABET_SIZE; ++c0) { + t = i + BUCKET_A(c0); + BUCKET_A(c0) = i + j; /* start point */ + i = t + BUCKET_B(c0, c0); + for(c1 = c0 + 1; c1 < ALPHABET_SIZE; ++c1) { + j += BUCKET_BSTAR(c0, c1); + BUCKET_BSTAR(c0, c1) = j; /* end point */ + i += BUCKET_B(c0, c1); + } + } + + if(0 < m) { + /* Sort the type B* suffixes by their first two characters. */ + PAb = SA + n - m; ISAb = SA + m; + for(i = m - 2; 0 <= i; --i) { + t = PAb[i], c0 = T[t], c1 = T[t + 1]; + SA[--BUCKET_BSTAR(c0, c1)] = i; + } + t = PAb[m - 1], c0 = T[t], c1 = T[t + 1]; + SA[--BUCKET_BSTAR(c0, c1)] = m - 1; + + /* Sort the type B* substrings using sssort. */ +#ifdef LIBBSC_OPENMP + if (openMP) + { + buf = SA + m; + c0 = ALPHABET_SIZE - 2, c1 = ALPHABET_SIZE - 1, j = m; +#pragma omp parallel default(shared) private(bufsize, curbuf, k, l, d0, d1) + { + bufsize = (n - (2 * m)) / omp_get_num_threads(); + curbuf = buf + omp_get_thread_num() * bufsize; + k = 0; + for(;;) { + #pragma omp critical(sssort_lock) + { + if(0 < (l = j)) { + d0 = c0, d1 = c1; + do { + k = BUCKET_BSTAR(d0, d1); + if(--d1 <= d0) { + d1 = ALPHABET_SIZE - 1; + if(--d0 < 0) { break; } + } + } while(((l - k) <= 1) && (0 < (l = k))); + c0 = d0, c1 = d1, j = k; + } + } + if(l == 0) { break; } + sssort(T, PAb, SA + k, SA + l, + curbuf, bufsize, 2, n, *(SA + k) == (m - 1)); + } + } + } + else + { + buf = SA + m, bufsize = n - (2 * m); + for(c0 = ALPHABET_SIZE - 2, j = m; 0 < j; --c0) { + for(c1 = ALPHABET_SIZE - 1; c0 < c1; j = i, --c1) { + i = BUCKET_BSTAR(c0, c1); + if(1 < (j - i)) { + sssort(T, PAb, SA + i, SA + j, + buf, bufsize, 2, n, *(SA + i) == (m - 1)); + } + } + } + } +#else + buf = SA + m, bufsize = n - (2 * m); + for(c0 = ALPHABET_SIZE - 2, j = m; 0 < j; --c0) { + for(c1 = ALPHABET_SIZE - 1; c0 < c1; j = i, --c1) { + i = BUCKET_BSTAR(c0, c1); + if(1 < (j - i)) { + sssort(T, PAb, SA + i, SA + j, + buf, bufsize, 2, n, *(SA + i) == (m - 1)); + } + } + } +#endif + + /* Compute ranks of type B* substrings. */ + for(i = m - 1; 0 <= i; --i) { + if(0 <= SA[i]) { + j = i; + do { ISAb[SA[i]] = i; } while((0 <= --i) && (0 <= SA[i])); + SA[i + 1] = i - j; + if(i <= 0) { break; } + } + j = i; + do { ISAb[SA[i] = ~SA[i]] = j; } while(SA[--i] < 0); + ISAb[SA[i]] = j; + } + + /* Construct the inverse suffix array of type B* suffixes using trsort. */ + trsort(ISAb, SA, m, 1); + + /* Set the sorted order of tyoe B* suffixes. */ + for(i = n - 1, j = m, c0 = T[n - 1]; 0 <= i;) { + for(--i, c1 = c0; (0 <= i) && ((c0 = T[i]) >= c1); --i, c1 = c0) { } + if(0 <= i) { + t = i; + for(--i, c1 = c0; (0 <= i) && ((c0 = T[i]) <= c1); --i, c1 = c0) { } + SA[ISAb[--j]] = ((t == 0) || (1 < (t - i))) ? t : ~t; + } + } + + /* Calculate the index of start/end point of each bucket. */ + BUCKET_B(ALPHABET_SIZE - 1, ALPHABET_SIZE - 1) = n; /* end point */ + for(c0 = ALPHABET_SIZE - 2, k = m - 1; 0 <= c0; --c0) { + i = BUCKET_A(c0 + 1) - 1; + for(c1 = ALPHABET_SIZE - 1; c0 < c1; --c1) { + t = i - BUCKET_B(c0, c1); + BUCKET_B(c0, c1) = i; /* end point */ + + /* Move all type B* suffixes to the correct position. */ + for(i = t, j = BUCKET_BSTAR(c0, c1); + j <= k; + --i, --k) { SA[i] = SA[k]; } + } + BUCKET_BSTAR(c0, c0 + 1) = i - BUCKET_B(c0, c0) + 1; /* start point */ + BUCKET_B(c0, c0) = i; /* end point */ + } + } + + return m; +} + +/* Constructs the suffix array by using the sorted order of type B* suffixes. */ +static +void +construct_SA(const unsigned char *T, int *SA, + int *bucket_A, int *bucket_B, + int n, int m) { + int *i, *j, *k; + int s; + int c0, c1, c2; + + if(0 < m) { + /* Construct the sorted order of type B suffixes by using + the sorted order of type B* suffixes. */ + for(c1 = ALPHABET_SIZE - 2; 0 <= c1; --c1) { + /* Scan the suffix array from right to left. */ + for(i = SA + BUCKET_BSTAR(c1, c1 + 1), + j = SA + BUCKET_A(c1 + 1) - 1, k = NULL, c2 = -1; + i <= j; + --j) { + if(0 < (s = *j)) { + assert(T[s] == c1); + assert(((s + 1) < n) && (T[s] <= T[s + 1])); + assert(T[s - 1] <= T[s]); + *j = ~s; + c0 = T[--s]; + if((0 < s) && (T[s - 1] > c0)) { s = ~s; } + if(c0 != c2) { + if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; } + k = SA + BUCKET_B(c2 = c0, c1); + } + assert(k < j); + *k-- = s; + } else { + assert(((s == 0) && (T[s] == c1)) || (s < 0)); + *j = ~s; + } + } + } + } + + /* Construct the suffix array by using + the sorted order of type B suffixes. */ + k = SA + BUCKET_A(c2 = T[n - 1]); + *k++ = (T[n - 2] < c2) ? ~(n - 1) : (n - 1); + /* Scan the suffix array from left to right. */ + for(i = SA, j = SA + n; i < j; ++i) { + if(0 < (s = *i)) { + assert(T[s - 1] >= T[s]); + c0 = T[--s]; + if((s == 0) || (T[s - 1] < c0)) { s = ~s; } + if(c0 != c2) { + BUCKET_A(c2) = k - SA; + k = SA + BUCKET_A(c2 = c0); + } + assert(i < k); + *k++ = s; + } else { + assert(s < 0); + *i = ~s; + } + } +} + +/* Constructs the burrows-wheeler transformed string directly + by using the sorted order of type B* suffixes. */ +static +int +construct_BWT(const unsigned char *T, int *SA, + int *bucket_A, int *bucket_B, + int n, int m) { + int *i, *j, *k, *orig; + int s; + int c0, c1, c2; + + if(0 < m) { + /* Construct the sorted order of type B suffixes by using + the sorted order of type B* suffixes. */ + for(c1 = ALPHABET_SIZE - 2; 0 <= c1; --c1) { + /* Scan the suffix array from right to left. */ + for(i = SA + BUCKET_BSTAR(c1, c1 + 1), + j = SA + BUCKET_A(c1 + 1) - 1, k = NULL, c2 = -1; + i <= j; + --j) { + if(0 < (s = *j)) { + assert(T[s] == c1); + assert(((s + 1) < n) && (T[s] <= T[s + 1])); + assert(T[s - 1] <= T[s]); + c0 = T[--s]; + *j = ~((int)c0); + if((0 < s) && (T[s - 1] > c0)) { s = ~s; } + if(c0 != c2) { + if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; } + k = SA + BUCKET_B(c2 = c0, c1); + } + assert(k < j); + *k-- = s; + } else if(s != 0) { + *j = ~s; +#ifndef NDEBUG + } else { + assert(T[s] == c1); +#endif + } + } + } + } + + /* Construct the BWTed string by using + the sorted order of type B suffixes. */ + k = SA + BUCKET_A(c2 = T[n - 1]); + *k++ = (T[n - 2] < c2) ? ~((int)T[n - 2]) : (n - 1); + /* Scan the suffix array from left to right. */ + for(i = SA, j = SA + n, orig = SA; i < j; ++i) { + if(0 < (s = *i)) { + assert(T[s - 1] >= T[s]); + c0 = T[--s]; + *i = c0; + if((0 < s) && (T[s - 1] < c0)) { s = ~((int)T[s - 1]); } + if(c0 != c2) { + BUCKET_A(c2) = k - SA; + k = SA + BUCKET_A(c2 = c0); + } + assert(i < k); + *k++ = s; + } else if(s != 0) { + *i = ~s; + } else { + orig = i; + } + } + + return orig - SA; +} + +/* Constructs the burrows-wheeler transformed string directly + by using the sorted order of type B* suffixes. */ +static +int +construct_BWT_indexes(const unsigned char *T, int *SA, + int *bucket_A, int *bucket_B, + int n, int m, + unsigned char * num_indexes, int * indexes) { + int *i, *j, *k, *orig; + int s; + int c0, c1, c2; + + int mod = n / 8; + { + mod |= mod >> 1; mod |= mod >> 2; + mod |= mod >> 4; mod |= mod >> 8; + mod |= mod >> 16; mod >>= 1; + + *num_indexes = (unsigned char)((n - 1) / (mod + 1)); + } + + if(0 < m) { + /* Construct the sorted order of type B suffixes by using + the sorted order of type B* suffixes. */ + for(c1 = ALPHABET_SIZE - 2; 0 <= c1; --c1) { + /* Scan the suffix array from right to left. */ + for(i = SA + BUCKET_BSTAR(c1, c1 + 1), + j = SA + BUCKET_A(c1 + 1) - 1, k = NULL, c2 = -1; + i <= j; + --j) { + if(0 < (s = *j)) { + assert(T[s] == c1); + assert(((s + 1) < n) && (T[s] <= T[s + 1])); + assert(T[s - 1] <= T[s]); + + if ((s & mod) == 0) indexes[s / (mod + 1) - 1] = j - SA; + + c0 = T[--s]; + *j = ~((int)c0); + if((0 < s) && (T[s - 1] > c0)) { s = ~s; } + if(c0 != c2) { + if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; } + k = SA + BUCKET_B(c2 = c0, c1); + } + assert(k < j); + *k-- = s; + } else if(s != 0) { + *j = ~s; +#ifndef NDEBUG + } else { + assert(T[s] == c1); +#endif + } + } + } + } + + /* Construct the BWTed string by using + the sorted order of type B suffixes. */ + k = SA + BUCKET_A(c2 = T[n - 1]); + if (T[n - 2] < c2) { + if (((n - 1) & mod) == 0) indexes[(n - 1) / (mod + 1) - 1] = k - SA; + *k++ = ~((int)T[n - 2]); + } + else { + *k++ = n - 1; + } + + /* Scan the suffix array from left to right. */ + for(i = SA, j = SA + n, orig = SA; i < j; ++i) { + if(0 < (s = *i)) { + assert(T[s - 1] >= T[s]); + + if ((s & mod) == 0) indexes[s / (mod + 1) - 1] = i - SA; + + c0 = T[--s]; + *i = c0; + if(c0 != c2) { + BUCKET_A(c2) = k - SA; + k = SA + BUCKET_A(c2 = c0); + } + assert(i < k); + if((0 < s) && (T[s - 1] < c0)) { + if ((s & mod) == 0) indexes[s / (mod + 1) - 1] = k - SA; + *k++ = ~((int)T[s - 1]); + } else + *k++ = s; + } else if(s != 0) { + *i = ~s; + } else { + orig = i; + } + } + + return orig - SA; +} + + +/*---------------------------------------------------------------------------*/ + +/*- Function -*/ + +int +divsufsort(const unsigned char *T, int *SA, int n, int openMP) { + int *bucket_A, *bucket_B; + int m; + int err = 0; + + /* Check arguments. */ + if((T == NULL) || (SA == NULL) || (n < 0)) { return -1; } + else if(n == 0) { return 0; } + else if(n == 1) { SA[0] = 0; return 0; } + else if(n == 2) { m = (T[0] < T[1]); SA[m ^ 1] = 0, SA[m] = 1; return 0; } + + bucket_A = (int *)malloc(BUCKET_A_SIZE * sizeof(int)); + bucket_B = (int *)malloc(BUCKET_B_SIZE * sizeof(int)); + + /* Suffixsort. */ + if((bucket_A != NULL) && (bucket_B != NULL)) { + m = sort_typeBstar(T, SA, bucket_A, bucket_B, n, openMP); + construct_SA(T, SA, bucket_A, bucket_B, n, m); + } else { + err = -2; + } + + free(bucket_B); + free(bucket_A); + + return err; +} + +int +divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char * num_indexes, int * indexes, int openMP) { + int *B; + int *bucket_A, *bucket_B; + int m, pidx, i; + + /* Check arguments. */ + if((T == NULL) || (U == NULL) || (n < 0)) { return -1; } + else if(n <= 1) { if(n == 1) { U[0] = T[0]; } return n; } + + if((B = A) == NULL) { B = (int *)malloc((size_t)(n + 1) * sizeof(int)); } + bucket_A = (int *)malloc(BUCKET_A_SIZE * sizeof(int)); + bucket_B = (int *)malloc(BUCKET_B_SIZE * sizeof(int)); + + /* Burrows-Wheeler Transform. */ + if((B != NULL) && (bucket_A != NULL) && (bucket_B != NULL)) { + m = sort_typeBstar(T, B, bucket_A, bucket_B, n, openMP); + + if (num_indexes == NULL || indexes == NULL) { + pidx = construct_BWT(T, B, bucket_A, bucket_B, n, m); + } else { + pidx = construct_BWT_indexes(T, B, bucket_A, bucket_B, n, m, num_indexes, indexes); + } + + /* Copy to output string. */ + U[0] = T[n - 1]; + for(i = 0; i < pidx; ++i) { U[i + 1] = (unsigned char)B[i]; } + for(i += 1; i < n; ++i) { U[i] = (unsigned char)B[i]; } + pidx += 1; + } else { + pidx = -2; + } + + free(bucket_B); + free(bucket_A); + if(A == NULL) { free(B); } + + return pidx; +} diff --git a/src/borg/algorithms/zstd/lib/dictBuilder/divsufsort.h b/src/borg/algorithms/zstd/lib/dictBuilder/divsufsort.h new file mode 100644 index 0000000000..5440994af1 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/dictBuilder/divsufsort.h @@ -0,0 +1,67 @@ +/* + * divsufsort.h for libdivsufsort-lite + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef _DIVSUFSORT_H +#define _DIVSUFSORT_H 1 + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + + +/*- Prototypes -*/ + +/** + * Constructs the suffix array of a given string. + * @param T [0..n-1] The input string. + * @param SA [0..n-1] The output array of suffixes. + * @param n The length of the given string. + * @param openMP enables OpenMP optimization. + * @return 0 if no error occurred, -1 or -2 otherwise. + */ +int +divsufsort(const unsigned char *T, int *SA, int n, int openMP); + +/** + * Constructs the burrows-wheeler transformed string of a given string. + * @param T [0..n-1] The input string. + * @param U [0..n-1] The output string. (can be T) + * @param A [0..n-1] The temporary array. (can be NULL) + * @param n The length of the given string. + * @param num_indexes The length of secondary indexes array. (can be NULL) + * @param indexes The secondary indexes array. (can be NULL) + * @param openMP enables OpenMP optimization. + * @return The primary index if no error occurred, -1 or -2 otherwise. + */ +int +divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char * num_indexes, int * indexes, int openMP); + + +#ifdef __cplusplus +} /* extern "C" */ +#endif /* __cplusplus */ + +#endif /* _DIVSUFSORT_H */ diff --git a/src/borg/algorithms/zstd/lib/dictBuilder/zdict.c b/src/borg/algorithms/zstd/lib/dictBuilder/zdict.c new file mode 100644 index 0000000000..1bb8b0683d --- /dev/null +++ b/src/borg/algorithms/zstd/lib/dictBuilder/zdict.c @@ -0,0 +1,1075 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/*-************************************** +* Tuning parameters +****************************************/ +#define MINRATIO 4 /* minimum nb of apparition to be selected in dictionary */ +#define ZDICT_MAX_SAMPLES_SIZE (2000U << 20) +#define ZDICT_MIN_SAMPLES_SIZE (ZDICT_CONTENTSIZE_MIN * MINRATIO) + + +/*-************************************** +* Compiler Options +****************************************/ +/* Unix Large Files support (>4GB) */ +#define _FILE_OFFSET_BITS 64 +#if (defined(__sun__) && (!defined(__LP64__))) /* Sun Solaris 32-bits requires specific definitions */ +# define _LARGEFILE_SOURCE +#elif ! defined(__LP64__) /* No point defining Large file for 64 bit */ +# define _LARGEFILE64_SOURCE +#endif + + +/*-************************************* +* Dependencies +***************************************/ +#include /* malloc, free */ +#include /* memset */ +#include /* fprintf, fopen, ftello64 */ +#include /* clock */ + +#include "mem.h" /* read */ +#include "fse.h" /* FSE_normalizeCount, FSE_writeNCount */ +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" /* HUF_buildCTable, HUF_writeCTable */ +#include "zstd_internal.h" /* includes zstd.h */ +#include "xxhash.h" /* XXH64 */ +#include "divsufsort.h" +#ifndef ZDICT_STATIC_LINKING_ONLY +# define ZDICT_STATIC_LINKING_ONLY +#endif +#include "zdict.h" + + +/*-************************************* +* Constants +***************************************/ +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define DICTLISTSIZE_DEFAULT 10000 + +#define NOISELENGTH 32 + +static const int g_compressionLevel_default = 3; +static const U32 g_selectivity_default = 9; + + +/*-************************************* +* Console display +***************************************/ +#define DISPLAY(...) { fprintf(stderr, __VA_ARGS__); fflush( stderr ); } +#define DISPLAYLEVEL(l, ...) if (notificationLevel>=l) { DISPLAY(__VA_ARGS__); } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */ + +static clock_t ZDICT_clockSpan(clock_t nPrevious) { return clock() - nPrevious; } + +static void ZDICT_printHex(const void* ptr, size_t length) +{ + const BYTE* const b = (const BYTE*)ptr; + size_t u; + for (u=0; u126) c = '.'; /* non-printable char */ + DISPLAY("%c", c); + } +} + + +/*-******************************************************** +* Helper functions +**********************************************************/ +unsigned ZDICT_isError(size_t errorCode) { return ERR_isError(errorCode); } + +const char* ZDICT_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } + +unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize) +{ + if (dictSize < 8) return 0; + if (MEM_readLE32(dictBuffer) != ZSTD_MAGIC_DICTIONARY) return 0; + return MEM_readLE32((const char*)dictBuffer + 4); +} + + +/*-******************************************************** +* Dictionary training functions +**********************************************************/ +static unsigned ZDICT_NbCommonBytes (register size_t val) +{ + if (MEM_isLittleEndian()) { + if (MEM_64bits()) { +# if defined(_MSC_VER) && defined(_WIN64) + unsigned long r = 0; + _BitScanForward64( &r, (U64)val ); + return (unsigned)(r>>3); +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_ctzll((U64)val) >> 3); +# else + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; + return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; +# endif + } else { /* 32 bits */ +# if defined(_MSC_VER) + unsigned long r=0; + _BitScanForward( &r, (U32)val ); + return (unsigned)(r>>3); +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_ctz((U32)val) >> 3); +# else + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; +# endif + } + } else { /* Big Endian CPU */ + if (MEM_64bits()) { +# if defined(_MSC_VER) && defined(_WIN64) + unsigned long r = 0; + _BitScanReverse64( &r, val ); + return (unsigned)(r>>3); +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_clzll(val) >> 3); +# else + unsigned r; + const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */ + if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; +# endif + } else { /* 32 bits */ +# if defined(_MSC_VER) + unsigned long r = 0; + _BitScanReverse( &r, (unsigned long)val ); + return (unsigned)(r>>3); +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_clz((U32)val) >> 3); +# else + unsigned r; + if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } + r += (!val); + return r; +# endif + } } +} + + +/*! ZDICT_count() : + Count the nb of common bytes between 2 pointers. + Note : this function presumes end of buffer followed by noisy guard band. +*/ +static size_t ZDICT_count(const void* pIn, const void* pMatch) +{ + const char* const pStart = (const char*)pIn; + for (;;) { + size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn); + if (!diff) { + pIn = (const char*)pIn+sizeof(size_t); + pMatch = (const char*)pMatch+sizeof(size_t); + continue; + } + pIn = (const char*)pIn+ZDICT_NbCommonBytes(diff); + return (size_t)((const char*)pIn - pStart); + } +} + + +typedef struct { + U32 pos; + U32 length; + U32 savings; +} dictItem; + +static void ZDICT_initDictItem(dictItem* d) +{ + d->pos = 1; + d->length = 0; + d->savings = (U32)(-1); +} + + +#define LLIMIT 64 /* heuristic determined experimentally */ +#define MINMATCHLENGTH 7 /* heuristic determined experimentally */ +static dictItem ZDICT_analyzePos( + BYTE* doneMarks, + const int* suffix, U32 start, + const void* buffer, U32 minRatio, U32 notificationLevel) +{ + U32 lengthList[LLIMIT] = {0}; + U32 cumulLength[LLIMIT] = {0}; + U32 savings[LLIMIT] = {0}; + const BYTE* b = (const BYTE*)buffer; + size_t length; + size_t maxLength = LLIMIT; + size_t pos = suffix[start]; + U32 end = start; + dictItem solution; + + /* init */ + memset(&solution, 0, sizeof(solution)); + doneMarks[pos] = 1; + + /* trivial repetition cases */ + if ( (MEM_read16(b+pos+0) == MEM_read16(b+pos+2)) + ||(MEM_read16(b+pos+1) == MEM_read16(b+pos+3)) + ||(MEM_read16(b+pos+2) == MEM_read16(b+pos+4)) ) { + /* skip and mark segment */ + U16 u16 = MEM_read16(b+pos+4); + U32 u, e = 6; + while (MEM_read16(b+pos+e) == u16) e+=2 ; + if (b[pos+e] == b[pos+e-1]) e++; + for (u=1; u=MINMATCHLENGTH); + + /* look backward */ + do { + length = ZDICT_count(b + pos, b + *(suffix+start-1)); + if (length >=MINMATCHLENGTH) start--; + } while(length >= MINMATCHLENGTH); + + /* exit if not found a minimum nb of repetitions */ + if (end-start < minRatio) { + U32 idx; + for(idx=start; idx= %i at pos %7u ", (U32)(end-start), MINMATCHLENGTH, (U32)pos); + DISPLAYLEVEL(4, "\n"); + + for (searchLength = MINMATCHLENGTH ; ; searchLength++) { + BYTE currentChar = 0; + U32 currentCount = 0; + U32 currentID = refinedStart; + U32 id; + U32 selectedCount = 0; + U32 selectedID = currentID; + for (id =refinedStart; id < refinedEnd; id++) { + if (b[ suffix[id] + searchLength] != currentChar) { + if (currentCount > selectedCount) { + selectedCount = currentCount; + selectedID = currentID; + } + currentID = id; + currentChar = b[ suffix[id] + searchLength]; + currentCount = 0; + } + currentCount ++; + } + if (currentCount > selectedCount) { /* for last */ + selectedCount = currentCount; + selectedID = currentID; + } + + if (selectedCount < minRatio) + break; + refinedStart = selectedID; + refinedEnd = refinedStart + selectedCount; + } + + /* evaluate gain based on new ref */ + start = refinedStart; + pos = suffix[refinedStart]; + end = start; + memset(lengthList, 0, sizeof(lengthList)); + + /* look forward */ + do { + end++; + length = ZDICT_count(b + pos, b + suffix[end]); + if (length >= LLIMIT) length = LLIMIT-1; + lengthList[length]++; + } while (length >=MINMATCHLENGTH); + + /* look backward */ + length = MINMATCHLENGTH; + while ((length >= MINMATCHLENGTH) & (start > 0)) { + length = ZDICT_count(b + pos, b + suffix[start - 1]); + if (length >= LLIMIT) length = LLIMIT - 1; + lengthList[length]++; + if (length >= MINMATCHLENGTH) start--; + } + + /* largest useful length */ + memset(cumulLength, 0, sizeof(cumulLength)); + cumulLength[maxLength-1] = lengthList[maxLength-1]; + for (i=(int)(maxLength-2); i>=0; i--) + cumulLength[i] = cumulLength[i+1] + lengthList[i]; + + for (i=LLIMIT-1; i>=MINMATCHLENGTH; i--) if (cumulLength[i]>=minRatio) break; + maxLength = i; + + /* reduce maxLength in case of final into repetitive data */ + { U32 l = (U32)maxLength; + BYTE const c = b[pos + maxLength-1]; + while (b[pos+l-2]==c) l--; + maxLength = l; + } + if (maxLength < MINMATCHLENGTH) return solution; /* skip : no long-enough solution */ + + /* calculate savings */ + savings[5] = 0; + for (i=MINMATCHLENGTH; i<=(int)maxLength; i++) + savings[i] = savings[i-1] + (lengthList[i] * (i-3)); + + DISPLAYLEVEL(4, "Selected ref at position %u, of length %u : saves %u (ratio: %.2f) \n", + (U32)pos, (U32)maxLength, savings[maxLength], (double)savings[maxLength] / maxLength); + + solution.pos = (U32)pos; + solution.length = (U32)maxLength; + solution.savings = savings[maxLength]; + + /* mark positions done */ + { U32 id; + for (id=start; id solution.length) length = solution.length; + } + pEnd = (U32)(testedPos + length); + for (p=testedPos; ppos; + const U32 eltEnd = elt.pos + elt.length; + const char* const buf = (const char*) buffer; + + /* tail overlap */ + U32 u; for (u=1; u elt.pos) && (table[u].pos <= eltEnd)) { /* overlap, existing > new */ + /* append */ + U32 const addedLength = table[u].pos - elt.pos; + table[u].length += addedLength; + table[u].pos = elt.pos; + table[u].savings += elt.savings * addedLength / elt.length; /* rough approx */ + table[u].savings += elt.length / 8; /* rough approx bonus */ + elt = table[u]; + /* sort : improve rank */ + while ((u>1) && (table[u-1].savings < elt.savings)) + table[u] = table[u-1], u--; + table[u] = elt; + return u; + } } + + /* front overlap */ + for (u=1; u= elt.pos) && (table[u].pos < elt.pos)) { /* overlap, existing < new */ + /* append */ + int const addedLength = (int)eltEnd - (table[u].pos + table[u].length); + table[u].savings += elt.length / 8; /* rough approx bonus */ + if (addedLength > 0) { /* otherwise, elt fully included into existing */ + table[u].length += addedLength; + table[u].savings += elt.savings * addedLength / elt.length; /* rough approx */ + } + /* sort : improve rank */ + elt = table[u]; + while ((u>1) && (table[u-1].savings < elt.savings)) + table[u] = table[u-1], u--; + table[u] = elt; + return u; + } + + if (MEM_read64(buf + table[u].pos) == MEM_read64(buf + elt.pos + 1)) { + if (isIncluded(buf + table[u].pos, buf + elt.pos + 1, table[u].length)) { + size_t const addedLength = MAX( (int)elt.length - (int)table[u].length , 1 ); + table[u].pos = elt.pos; + table[u].savings += (U32)(elt.savings * addedLength / elt.length); + table[u].length = MIN(elt.length, table[u].length + 1); + return u; + } + } + } + + return 0; +} + + +static void ZDICT_removeDictItem(dictItem* table, U32 id) +{ + /* convention : table[0].pos stores nb of elts */ + U32 const max = table[0].pos; + U32 u; + if (!id) return; /* protection, should never happen */ + for (u=id; upos--; +} + + +static void ZDICT_insertDictItem(dictItem* table, U32 maxSize, dictItem elt, const void* buffer) +{ + /* merge if possible */ + U32 mergeId = ZDICT_tryMerge(table, elt, 0, buffer); + if (mergeId) { + U32 newMerge = 1; + while (newMerge) { + newMerge = ZDICT_tryMerge(table, table[mergeId], mergeId, buffer); + if (newMerge) ZDICT_removeDictItem(table, mergeId); + mergeId = newMerge; + } + return; + } + + /* insert */ + { U32 current; + U32 nextElt = table->pos; + if (nextElt >= maxSize) nextElt = maxSize-1; + current = nextElt-1; + while (table[current].savings < elt.savings) { + table[current+1] = table[current]; + current--; + } + table[current+1] = elt; + table->pos = nextElt+1; + } +} + + +static U32 ZDICT_dictSize(const dictItem* dictList) +{ + U32 u, dictSize = 0; + for (u=1; u=l) { \ + if (ZDICT_clockSpan(displayClock) > refreshRate) \ + { displayClock = clock(); DISPLAY(__VA_ARGS__); \ + if (notificationLevel>=4) fflush(stderr); } } + + /* init */ + DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */ + if (!suffix0 || !reverseSuffix || !doneMarks || !filePos) { + result = ERROR(memory_allocation); + goto _cleanup; + } + if (minRatio < MINRATIO) minRatio = MINRATIO; + memset(doneMarks, 0, bufferSize+16); + + /* limit sample set size (divsufsort limitation)*/ + if (bufferSize > ZDICT_MAX_SAMPLES_SIZE) DISPLAYLEVEL(3, "sample set too large : reduced to %u MB ...\n", (U32)(ZDICT_MAX_SAMPLES_SIZE>>20)); + while (bufferSize > ZDICT_MAX_SAMPLES_SIZE) bufferSize -= fileSizes[--nbFiles]; + + /* sort */ + DISPLAYLEVEL(2, "sorting %u files of total size %u MB ...\n", nbFiles, (U32)(bufferSize>>20)); + { int const divSuftSortResult = divsufsort((const unsigned char*)buffer, suffix, (int)bufferSize, 0); + if (divSuftSortResult != 0) { result = ERROR(GENERIC); goto _cleanup; } + } + suffix[bufferSize] = (int)bufferSize; /* leads into noise */ + suffix0[0] = (int)bufferSize; /* leads into noise */ + /* build reverse suffix sort */ + { size_t pos; + for (pos=0; pos < bufferSize; pos++) + reverseSuffix[suffix[pos]] = (U32)pos; + /* note filePos tracks borders between samples. + It's not used at this stage, but planned to become useful in a later update */ + filePos[0] = 0; + for (pos=1; pos> 21); + } +} + + +typedef struct +{ + ZSTD_CCtx* ref; + ZSTD_CCtx* zc; + void* workPlace; /* must be ZSTD_BLOCKSIZE_MAX allocated */ +} EStats_ress_t; + +#define MAXREPOFFSET 1024 + +static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params, + U32* countLit, U32* offsetcodeCount, U32* matchlengthCount, U32* litlengthCount, U32* repOffsets, + const void* src, size_t srcSize, U32 notificationLevel) +{ + size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_MAX, 1 << params.cParams.windowLog); + size_t cSize; + + if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */ + { size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref, 0); + if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; } + } + cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize); + if (ZSTD_isError(cSize)) { DISPLAYLEVEL(3, "warning : could not compress sample size %u \n", (U32)srcSize); return; } + + if (cSize) { /* if == 0; block is not compressible */ + const seqStore_t* seqStorePtr = ZSTD_getSeqStore(esr.zc); + + /* literals stats */ + { const BYTE* bytePtr; + for(bytePtr = seqStorePtr->litStart; bytePtr < seqStorePtr->lit; bytePtr++) + countLit[*bytePtr]++; + } + + /* seqStats */ + { U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + ZSTD_seqToCodes(seqStorePtr); + + { const BYTE* codePtr = seqStorePtr->ofCode; + U32 u; + for (u=0; umlCode; + U32 u; + for (u=0; ullCode; + U32 u; + for (u=0; u= 2) { /* rep offsets */ + const seqDef* const seq = seqStorePtr->sequencesStart; + U32 offset1 = seq[0].offset - 3; + U32 offset2 = seq[1].offset - 3; + if (offset1 >= MAXREPOFFSET) offset1 = 0; + if (offset2 >= MAXREPOFFSET) offset2 = 0; + repOffsets[offset1] += 3; + repOffsets[offset2] += 1; + } } } +} + +static size_t ZDICT_totalSampleSize(const size_t* fileSizes, unsigned nbFiles) +{ + size_t total=0; + unsigned u; + for (u=0; u0; u--) { + offsetCount_t tmp; + if (table[u-1].count >= table[u].count) break; + tmp = table[u-1]; + table[u-1] = table[u]; + table[u] = tmp; + } +} + + +#define OFFCODE_MAX 30 /* only applicable to first block */ +static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, + unsigned compressionLevel, + const void* srcBuffer, const size_t* fileSizes, unsigned nbFiles, + const void* dictBuffer, size_t dictBufferSize, + unsigned notificationLevel) +{ + U32 countLit[256]; + HUF_CREATE_STATIC_CTABLE(hufTable, 255); + U32 offcodeCount[OFFCODE_MAX+1]; + short offcodeNCount[OFFCODE_MAX+1]; + U32 offcodeMax = ZSTD_highbit32((U32)(dictBufferSize + 128 KB)); + U32 matchLengthCount[MaxML+1]; + short matchLengthNCount[MaxML+1]; + U32 litLengthCount[MaxLL+1]; + short litLengthNCount[MaxLL+1]; + U32 repOffset[MAXREPOFFSET]; + offsetCount_t bestRepOffset[ZSTD_REP_NUM+1]; + EStats_ress_t esr; + ZSTD_parameters params; + U32 u, huffLog = 11, Offlog = OffFSELog, mlLog = MLFSELog, llLog = LLFSELog, total; + size_t pos = 0, errorCode; + size_t eSize = 0; + size_t const totalSrcSize = ZDICT_totalSampleSize(fileSizes, nbFiles); + size_t const averageSampleSize = totalSrcSize / (nbFiles + !nbFiles); + BYTE* dstPtr = (BYTE*)dstBuffer; + + /* init */ + esr.ref = ZSTD_createCCtx(); + esr.zc = ZSTD_createCCtx(); + esr.workPlace = malloc(ZSTD_BLOCKSIZE_MAX); + if (!esr.ref || !esr.zc || !esr.workPlace) { + eSize = ERROR(memory_allocation); + DISPLAYLEVEL(1, "Not enough memory \n"); + goto _cleanup; + } + if (offcodeMax>OFFCODE_MAX) { eSize = ERROR(dictionaryCreation_failed); goto _cleanup; } /* too large dictionary */ + for (u=0; u<256; u++) countLit[u] = 1; /* any character must be described */ + for (u=0; u<=offcodeMax; u++) offcodeCount[u] = 1; + for (u=0; u<=MaxML; u++) matchLengthCount[u] = 1; + for (u=0; u<=MaxLL; u++) litLengthCount[u] = 1; + memset(repOffset, 0, sizeof(repOffset)); + repOffset[1] = repOffset[4] = repOffset[8] = 1; + memset(bestRepOffset, 0, sizeof(bestRepOffset)); + if (compressionLevel<=0) compressionLevel = g_compressionLevel_default; + params = ZSTD_getParams(compressionLevel, averageSampleSize, dictBufferSize); + { size_t const beginResult = ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params, 0); + if (ZSTD_isError(beginResult)) { + DISPLAYLEVEL(1, "error : ZSTD_compressBegin_advanced() failed : %s \n", ZSTD_getErrorName(beginResult)); + eSize = ERROR(GENERIC); + goto _cleanup; + } } + + /* collect stats on all files */ + for (u=0; u dictBufferCapacity) dictContentSize = dictBufferCapacity - hSize; + { size_t const dictSize = hSize + dictContentSize; + char* dictEnd = (char*)dictBuffer + dictSize; + memmove(dictEnd - dictContentSize, customDictContent, dictContentSize); + memcpy(dictBuffer, header, hSize); + return dictSize; + } +} + + +size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_params_t params) +{ + int const compressionLevel = (params.compressionLevel <= 0) ? g_compressionLevel_default : params.compressionLevel; + U32 const notificationLevel = params.notificationLevel; + size_t hSize = 8; + + /* calculate entropy tables */ + DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */ + DISPLAYLEVEL(2, "statistics ... \n"); + { size_t const eSize = ZDICT_analyzeEntropy((char*)dictBuffer+hSize, dictBufferCapacity-hSize, + compressionLevel, + samplesBuffer, samplesSizes, nbSamples, + (char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize, + notificationLevel); + if (ZDICT_isError(eSize)) return eSize; + hSize += eSize; + } + + /* add dictionary header (after entropy tables) */ + MEM_writeLE32(dictBuffer, ZSTD_MAGIC_DICTIONARY); + { U64 const randomID = XXH64((char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize, 0); + U32 const compliantID = (randomID % ((1U<<31)-32768)) + 32768; + U32 const dictID = params.dictID ? params.dictID : compliantID; + MEM_writeLE32((char*)dictBuffer+4, dictID); + } + + if (hSize + dictContentSize < dictBufferCapacity) + memmove((char*)dictBuffer + hSize, (char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize); + return MIN(dictBufferCapacity, hSize+dictContentSize); +} + + +/*! ZDICT_trainFromBuffer_unsafe_legacy() : +* Warning : `samplesBuffer` must be followed by noisy guard band. +* @return : size of dictionary, or an error code which can be tested with ZDICT_isError() +*/ +size_t ZDICT_trainFromBuffer_unsafe_legacy( + void* dictBuffer, size_t maxDictSize, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_legacy_params_t params) +{ + U32 const dictListSize = MAX(MAX(DICTLISTSIZE_DEFAULT, nbSamples), (U32)(maxDictSize/16)); + dictItem* const dictList = (dictItem*)malloc(dictListSize * sizeof(*dictList)); + unsigned const selectivity = params.selectivityLevel == 0 ? g_selectivity_default : params.selectivityLevel; + unsigned const minRep = (selectivity > 30) ? MINRATIO : nbSamples >> selectivity; + size_t const targetDictSize = maxDictSize; + size_t const samplesBuffSize = ZDICT_totalSampleSize(samplesSizes, nbSamples); + size_t dictSize = 0; + U32 const notificationLevel = params.zParams.notificationLevel; + + /* checks */ + if (!dictList) return ERROR(memory_allocation); + if (maxDictSize < ZDICT_DICTSIZE_MIN) { free(dictList); return ERROR(dstSize_tooSmall); } /* requested dictionary size is too small */ + if (samplesBuffSize < ZDICT_MIN_SAMPLES_SIZE) { free(dictList); return ERROR(dictionaryCreation_failed); } /* not enough source to create dictionary */ + + /* init */ + ZDICT_initDictItem(dictList); + + /* build dictionary */ + ZDICT_trainBuffer_legacy(dictList, dictListSize, + samplesBuffer, samplesBuffSize, + samplesSizes, nbSamples, + minRep, notificationLevel); + + /* display best matches */ + if (params.zParams.notificationLevel>= 3) { + U32 const nb = MIN(25, dictList[0].pos); + U32 const dictContentSize = ZDICT_dictSize(dictList); + U32 u; + DISPLAYLEVEL(3, "\n %u segments found, of total size %u \n", dictList[0].pos-1, dictContentSize); + DISPLAYLEVEL(3, "list %u best segments \n", nb-1); + for (u=1; u samplesBuffSize) || ((pos + length) > samplesBuffSize)) + return ERROR(GENERIC); /* should never happen */ + DISPLAYLEVEL(3, "%3u:%3u bytes at pos %8u, savings %7u bytes |", + u, length, pos, dictList[u].savings); + ZDICT_printHex((const char*)samplesBuffer+pos, printedLength); + DISPLAYLEVEL(3, "| \n"); + } } + + + /* create dictionary */ + { U32 dictContentSize = ZDICT_dictSize(dictList); + if (dictContentSize < ZDICT_CONTENTSIZE_MIN) { free(dictList); return ERROR(dictionaryCreation_failed); } /* dictionary content too small */ + if (dictContentSize < targetDictSize/4) { + DISPLAYLEVEL(2, "! warning : selected content significantly smaller than requested (%u < %u) \n", dictContentSize, (U32)maxDictSize); + if (samplesBuffSize < 10 * targetDictSize) + DISPLAYLEVEL(2, "! consider increasing the number of samples (total size : %u MB)\n", (U32)(samplesBuffSize>>20)); + if (minRep > MINRATIO) { + DISPLAYLEVEL(2, "! consider increasing selectivity to produce larger dictionary (-s%u) \n", selectivity+1); + DISPLAYLEVEL(2, "! note : larger dictionaries are not necessarily better, test its efficiency on samples \n"); + } + } + + if ((dictContentSize > targetDictSize*3) && (nbSamples > 2*MINRATIO) && (selectivity>1)) { + U32 proposedSelectivity = selectivity-1; + while ((nbSamples >> proposedSelectivity) <= MINRATIO) { proposedSelectivity--; } + DISPLAYLEVEL(2, "! note : calculated dictionary significantly larger than requested (%u > %u) \n", dictContentSize, (U32)maxDictSize); + DISPLAYLEVEL(2, "! consider increasing dictionary size, or produce denser dictionary (-s%u) \n", proposedSelectivity); + DISPLAYLEVEL(2, "! always test dictionary efficiency on real samples \n"); + } + + /* limit dictionary size */ + { U32 const max = dictList->pos; /* convention : nb of useful elts within dictList */ + U32 currentSize = 0; + U32 n; for (n=1; n targetDictSize) { currentSize -= dictList[n].length; break; } + } + dictList->pos = n; + dictContentSize = currentSize; + } + + /* build dict content */ + { U32 u; + BYTE* ptr = (BYTE*)dictBuffer + maxDictSize; + for (u=1; upos; u++) { + U32 l = dictList[u].length; + ptr -= l; + if (ptr<(BYTE*)dictBuffer) { free(dictList); return ERROR(GENERIC); } /* should not happen */ + memcpy(ptr, (const char*)samplesBuffer+dictList[u].pos, l); + } } + + dictSize = ZDICT_addEntropyTablesFromBuffer_advanced(dictBuffer, dictContentSize, maxDictSize, + samplesBuffer, samplesSizes, nbSamples, + params.zParams); + } + + /* clean up */ + free(dictList); + return dictSize; +} + + +/* issue : samplesBuffer need to be followed by a noisy guard band. +* work around : duplicate the buffer, and add the noise */ +size_t ZDICT_trainFromBuffer_legacy(void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_legacy_params_t params) +{ + size_t result; + void* newBuff; + size_t const sBuffSize = ZDICT_totalSampleSize(samplesSizes, nbSamples); + if (sBuffSize < ZDICT_MIN_SAMPLES_SIZE) return 0; /* not enough content => no dictionary */ + + newBuff = malloc(sBuffSize + NOISELENGTH); + if (!newBuff) return ERROR(memory_allocation); + + memcpy(newBuff, samplesBuffer, sBuffSize); + ZDICT_fillNoise((char*)newBuff + sBuffSize, NOISELENGTH); /* guard band, for end of buffer condition */ + + result = + ZDICT_trainFromBuffer_unsafe_legacy(dictBuffer, dictBufferCapacity, newBuff, + samplesSizes, nbSamples, params); + free(newBuff); + return result; +} + + +size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples) +{ + ZDICT_cover_params_t params; + memset(¶ms, 0, sizeof(params)); + params.d = 8; + params.steps = 4; + /* Default to level 6 since no compression level information is avaialble */ + params.zParams.compressionLevel = 6; + return ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, dictBufferCapacity, + samplesBuffer, samplesSizes, + nbSamples, ¶ms); +} + +size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples) +{ + ZDICT_params_t params; + memset(¶ms, 0, sizeof(params)); + return ZDICT_addEntropyTablesFromBuffer_advanced(dictBuffer, dictContentSize, dictBufferCapacity, + samplesBuffer, samplesSizes, nbSamples, + params); +} diff --git a/src/borg/algorithms/zstd/lib/dictBuilder/zdict.h b/src/borg/algorithms/zstd/lib/dictBuilder/zdict.h new file mode 100644 index 0000000000..5f0000b1c7 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/dictBuilder/zdict.h @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef DICTBUILDER_H_001 +#define DICTBUILDER_H_001 + +#if defined (__cplusplus) +extern "C" { +#endif + + +/*====== Dependencies ======*/ +#include /* size_t */ + + +/* ===== ZDICTLIB_API : control library symbols visibility ===== */ +#ifndef ZDICTLIB_VISIBILITY +# if defined(__GNUC__) && (__GNUC__ >= 4) +# define ZDICTLIB_VISIBILITY __attribute__ ((visibility ("default"))) +# else +# define ZDICTLIB_VISIBILITY +# endif +#endif +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBILITY +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZDICTLIB_API ZDICTLIB_VISIBILITY +#endif + + +/*! ZDICT_trainFromBuffer(): + * Train a dictionary from an array of samples. + * Uses ZDICT_optimizeTrainFromBuffer_cover() single-threaded, with d=8 and steps=4. + * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + * The resulting dictionary will be saved into `dictBuffer`. + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * Note: ZDICT_trainFromBuffer() requires about 9 bytes of memory for each input byte. + * Tips: In general, a reasonable dictionary has a size of ~ 100 KB. + * It's obviously possible to target smaller or larger ones, just by specifying different `dictBufferCapacity`. + * In general, it's recommended to provide a few thousands samples, but this can vary a lot. + * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. + */ +ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples); + + +/*====== Helper functions ======*/ +ZDICTLIB_API unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize); /**< extracts dictID; @return zero if error (not a valid dictionary) */ +ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode); +ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode); + + + +#ifdef ZDICT_STATIC_LINKING_ONLY + +/* ==================================================================================== + * The definitions in this section are considered experimental. + * They should never be used with a dynamic library, as they may change in the future. + * They are provided for advanced usages. + * Use them only in association with static linking. + * ==================================================================================== */ + +typedef struct { + int compressionLevel; /* 0 means default; target a specific zstd compression level */ + unsigned notificationLevel; /* Write to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */ + unsigned dictID; /* 0 means auto mode (32-bits random value); other : force dictID value */ +} ZDICT_params_t; + +/*! ZDICT_cover_params_t: + * For all values 0 means default. + * k and d are the only required parameters. + */ +typedef struct { + unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */ + unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */ + unsigned steps; /* Number of steps : Only used for optimization : 0 means default (32) : Higher means more parameters checked */ + unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */ + ZDICT_params_t zParams; +} ZDICT_cover_params_t; + + +/*! ZDICT_trainFromBuffer_cover(): + * Train a dictionary from an array of samples using the COVER algorithm. + * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + * The resulting dictionary will be saved into `dictBuffer`. + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * Note: ZDICT_trainFromBuffer_cover() requires about 9 bytes of memory for each input byte. + * Tips: In general, a reasonable dictionary has a size of ~ 100 KB. + * It's obviously possible to target smaller or larger ones, just by specifying different `dictBufferCapacity`. + * In general, it's recommended to provide a few thousands samples, but this can vary a lot. + * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. + */ +ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover( + void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, + const size_t *samplesSizes, unsigned nbSamples, + ZDICT_cover_params_t parameters); + +/*! ZDICT_optimizeTrainFromBuffer_cover(): + * The same requirements as above hold for all the parameters except `parameters`. + * This function tries many parameter combinations and picks the best parameters. + * `*parameters` is filled with the best parameters found, and the dictionary + * constructed with those parameters is stored in `dictBuffer`. + * + * All of the parameters d, k, steps are optional. + * If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8, 10, 12, 14, 16}. + * if steps is zero it defaults to its default value. + * If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [16, 2048]. + * + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * On success `*parameters` contains the parameters selected. + * Note: ZDICT_optimizeTrainFromBuffer_cover() requires about 8 bytes of memory for each input byte and additionally another 5 bytes of memory for each byte of memory for each thread. + */ +ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover( + void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, + const size_t *samplesSizes, unsigned nbSamples, + ZDICT_cover_params_t *parameters); + +/*! ZDICT_finalizeDictionary(): + * Given a custom content as a basis for dictionary, and a set of samples, + * finalize dictionary by adding headers and statistics. + * + * Samples must be stored concatenated in a flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample in order. + * + * dictContentSize must be >= ZDICT_CONTENTSIZE_MIN bytes. + * maxDictSize must be >= dictContentSize, and must be >= ZDICT_DICTSIZE_MIN bytes. + * + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`), + * or an error code, which can be tested by ZDICT_isError(). + * Note: ZDICT_finalizeDictionary() will push notifications into stderr if instructed to, using notificationLevel>0. + * Note 2: dictBuffer and dictContent can overlap + */ +#define ZDICT_CONTENTSIZE_MIN 128 +#define ZDICT_DICTSIZE_MIN 256 +ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity, + const void* dictContent, size_t dictContentSize, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_params_t parameters); + +typedef struct { + unsigned selectivityLevel; /* 0 means default; larger => select more => larger dictionary */ + ZDICT_params_t zParams; +} ZDICT_legacy_params_t; + +/*! ZDICT_trainFromBuffer_legacy(): + * Train a dictionary from an array of samples. + * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + * The resulting dictionary will be saved into `dictBuffer`. + * `parameters` is optional and can be provided with values set to 0 to mean "default". + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * Tips: In general, a reasonable dictionary has a size of ~ 100 KB. + * It's obviously possible to target smaller or larger ones, just by specifying different `dictBufferCapacity`. + * In general, it's recommended to provide a few thousands samples, but this can vary a lot. + * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. + * Note: ZDICT_trainFromBuffer_legacy() will send notifications into stderr if instructed to, using notificationLevel>0. + */ +ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy( + void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, + const size_t *samplesSizes, unsigned nbSamples, ZDICT_legacy_params_t parameters); + +/* Deprecation warnings */ +/* It is generally possible to disable deprecation warnings from compiler, + for example with -Wno-deprecated-declarations for gcc + or _CRT_SECURE_NO_WARNINGS in Visual. + Otherwise, it's also possible to manually define ZDICT_DISABLE_DEPRECATE_WARNINGS */ +#ifdef ZDICT_DISABLE_DEPRECATE_WARNINGS +# define ZDICT_DEPRECATED(message) ZDICTLIB_API /* disable deprecation warnings */ +#else +# define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# define ZDICT_DEPRECATED(message) [[deprecated(message)]] ZDICTLIB_API +# elif (ZDICT_GCC_VERSION >= 405) || defined(__clang__) +# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated(message))) +# elif (ZDICT_GCC_VERSION >= 301) +# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define ZDICT_DEPRECATED(message) ZDICTLIB_API __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement ZDICT_DEPRECATED for this compiler") +# define ZDICT_DEPRECATED(message) ZDICTLIB_API +# endif +#endif /* ZDICT_DISABLE_DEPRECATE_WARNINGS */ + +ZDICT_DEPRECATED("use ZDICT_finalizeDictionary() instead") +size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples); + + +#endif /* ZDICT_STATIC_LINKING_ONLY */ + +#if defined (__cplusplus) +} +#endif + +#endif /* DICTBUILDER_H_001 */ diff --git a/src/borg/algorithms/zstd/lib/legacy/zstd_legacy.h b/src/borg/algorithms/zstd/lib/legacy/zstd_legacy.h new file mode 100644 index 0000000000..487ff0b284 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/legacy/zstd_legacy.h @@ -0,0 +1,379 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_LEGACY_H +#define ZSTD_LEGACY_H + +#if defined (__cplusplus) +extern "C" { +#endif + +/* ************************************* +* Includes +***************************************/ +#include "mem.h" /* MEM_STATIC */ +#include "error_private.h" /* ERROR */ +#include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer */ + +#if !defined (ZSTD_LEGACY_SUPPORT) || (ZSTD_LEGACY_SUPPORT == 0) +# undef ZSTD_LEGACY_SUPPORT +# define ZSTD_LEGACY_SUPPORT 8 +#endif + +#if (ZSTD_LEGACY_SUPPORT <= 1) +# include "zstd_v01.h" +#endif +#if (ZSTD_LEGACY_SUPPORT <= 2) +# include "zstd_v02.h" +#endif +#if (ZSTD_LEGACY_SUPPORT <= 3) +# include "zstd_v03.h" +#endif +#if (ZSTD_LEGACY_SUPPORT <= 4) +# include "zstd_v04.h" +#endif +#if (ZSTD_LEGACY_SUPPORT <= 5) +# include "zstd_v05.h" +#endif +#if (ZSTD_LEGACY_SUPPORT <= 6) +# include "zstd_v06.h" +#endif +#if (ZSTD_LEGACY_SUPPORT <= 7) +# include "zstd_v07.h" +#endif + +/** ZSTD_isLegacy() : + @return : > 0 if supported by legacy decoder. 0 otherwise. + return value is the version. +*/ +MEM_STATIC unsigned ZSTD_isLegacy(const void* src, size_t srcSize) +{ + U32 magicNumberLE; + if (srcSize<4) return 0; + magicNumberLE = MEM_readLE32(src); + switch(magicNumberLE) + { +#if (ZSTD_LEGACY_SUPPORT <= 1) + case ZSTDv01_magicNumberLE:return 1; +#endif +#if (ZSTD_LEGACY_SUPPORT <= 2) + case ZSTDv02_magicNumber : return 2; +#endif +#if (ZSTD_LEGACY_SUPPORT <= 3) + case ZSTDv03_magicNumber : return 3; +#endif +#if (ZSTD_LEGACY_SUPPORT <= 4) + case ZSTDv04_magicNumber : return 4; +#endif +#if (ZSTD_LEGACY_SUPPORT <= 5) + case ZSTDv05_MAGICNUMBER : return 5; +#endif +#if (ZSTD_LEGACY_SUPPORT <= 6) + case ZSTDv06_MAGICNUMBER : return 6; +#endif +#if (ZSTD_LEGACY_SUPPORT <= 7) + case ZSTDv07_MAGICNUMBER : return 7; +#endif + default : return 0; + } +} + + +MEM_STATIC unsigned long long ZSTD_getDecompressedSize_legacy(const void* src, size_t srcSize) +{ + U32 const version = ZSTD_isLegacy(src, srcSize); + if (version < 5) return 0; /* no decompressed size in frame header, or not a legacy format */ +#if (ZSTD_LEGACY_SUPPORT <= 5) + if (version==5) { + ZSTDv05_parameters fParams; + size_t const frResult = ZSTDv05_getFrameParams(&fParams, src, srcSize); + if (frResult != 0) return 0; + return fParams.srcSize; + } +#endif +#if (ZSTD_LEGACY_SUPPORT <= 6) + if (version==6) { + ZSTDv06_frameParams fParams; + size_t const frResult = ZSTDv06_getFrameParams(&fParams, src, srcSize); + if (frResult != 0) return 0; + return fParams.frameContentSize; + } +#endif +#if (ZSTD_LEGACY_SUPPORT <= 7) + if (version==7) { + ZSTDv07_frameParams fParams; + size_t const frResult = ZSTDv07_getFrameParams(&fParams, src, srcSize); + if (frResult != 0) return 0; + return fParams.frameContentSize; + } +#endif + return 0; /* should not be possible */ +} + + +MEM_STATIC size_t ZSTD_decompressLegacy( + void* dst, size_t dstCapacity, + const void* src, size_t compressedSize, + const void* dict,size_t dictSize) +{ + U32 const version = ZSTD_isLegacy(src, compressedSize); + (void)dst; (void)dstCapacity; (void)dict; (void)dictSize; /* unused when ZSTD_LEGACY_SUPPORT >= 8 */ + switch(version) + { +#if (ZSTD_LEGACY_SUPPORT <= 1) + case 1 : + return ZSTDv01_decompress(dst, dstCapacity, src, compressedSize); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 2) + case 2 : + return ZSTDv02_decompress(dst, dstCapacity, src, compressedSize); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 3) + case 3 : + return ZSTDv03_decompress(dst, dstCapacity, src, compressedSize); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 4) + case 4 : + return ZSTDv04_decompress(dst, dstCapacity, src, compressedSize); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 5) + case 5 : + { size_t result; + ZSTDv05_DCtx* const zd = ZSTDv05_createDCtx(); + if (zd==NULL) return ERROR(memory_allocation); + result = ZSTDv05_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize); + ZSTDv05_freeDCtx(zd); + return result; + } +#endif +#if (ZSTD_LEGACY_SUPPORT <= 6) + case 6 : + { size_t result; + ZSTDv06_DCtx* const zd = ZSTDv06_createDCtx(); + if (zd==NULL) return ERROR(memory_allocation); + result = ZSTDv06_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize); + ZSTDv06_freeDCtx(zd); + return result; + } +#endif +#if (ZSTD_LEGACY_SUPPORT <= 7) + case 7 : + { size_t result; + ZSTDv07_DCtx* const zd = ZSTDv07_createDCtx(); + if (zd==NULL) return ERROR(memory_allocation); + result = ZSTDv07_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize); + ZSTDv07_freeDCtx(zd); + return result; + } +#endif + default : + return ERROR(prefix_unknown); + } +} + +MEM_STATIC size_t ZSTD_findFrameCompressedSizeLegacy(const void *src, + size_t compressedSize) +{ + U32 const version = ZSTD_isLegacy(src, compressedSize); + switch(version) + { +#if (ZSTD_LEGACY_SUPPORT <= 1) + case 1 : + return ZSTDv01_findFrameCompressedSize(src, compressedSize); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 2) + case 2 : + return ZSTDv02_findFrameCompressedSize(src, compressedSize); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 3) + case 3 : + return ZSTDv03_findFrameCompressedSize(src, compressedSize); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 4) + case 4 : + return ZSTDv04_findFrameCompressedSize(src, compressedSize); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 5) + case 5 : + return ZSTDv05_findFrameCompressedSize(src, compressedSize); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 6) + case 6 : + return ZSTDv06_findFrameCompressedSize(src, compressedSize); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 7) + case 7 : + return ZSTDv07_findFrameCompressedSize(src, compressedSize); +#endif + default : + return ERROR(prefix_unknown); + } +} + +MEM_STATIC size_t ZSTD_freeLegacyStreamContext(void* legacyContext, U32 version) +{ + switch(version) + { + default : + case 1 : + case 2 : + case 3 : + (void)legacyContext; + return ERROR(version_unsupported); +#if (ZSTD_LEGACY_SUPPORT <= 4) + case 4 : return ZBUFFv04_freeDCtx((ZBUFFv04_DCtx*)legacyContext); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 5) + case 5 : return ZBUFFv05_freeDCtx((ZBUFFv05_DCtx*)legacyContext); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 6) + case 6 : return ZBUFFv06_freeDCtx((ZBUFFv06_DCtx*)legacyContext); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 7) + case 7 : return ZBUFFv07_freeDCtx((ZBUFFv07_DCtx*)legacyContext); +#endif + } +} + + +MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U32 newVersion, + const void* dict, size_t dictSize) +{ + if (prevVersion != newVersion) ZSTD_freeLegacyStreamContext(*legacyContext, prevVersion); + switch(newVersion) + { + default : + case 1 : + case 2 : + case 3 : + (void)dict; (void)dictSize; + return 0; +#if (ZSTD_LEGACY_SUPPORT <= 4) + case 4 : + { + ZBUFFv04_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv04_createDCtx() : (ZBUFFv04_DCtx*)*legacyContext; + if (dctx==NULL) return ERROR(memory_allocation); + ZBUFFv04_decompressInit(dctx); + ZBUFFv04_decompressWithDictionary(dctx, dict, dictSize); + *legacyContext = dctx; + return 0; + } +#endif +#if (ZSTD_LEGACY_SUPPORT <= 5) + case 5 : + { + ZBUFFv05_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv05_createDCtx() : (ZBUFFv05_DCtx*)*legacyContext; + if (dctx==NULL) return ERROR(memory_allocation); + ZBUFFv05_decompressInitDictionary(dctx, dict, dictSize); + *legacyContext = dctx; + return 0; + } +#endif +#if (ZSTD_LEGACY_SUPPORT <= 6) + case 6 : + { + ZBUFFv06_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv06_createDCtx() : (ZBUFFv06_DCtx*)*legacyContext; + if (dctx==NULL) return ERROR(memory_allocation); + ZBUFFv06_decompressInitDictionary(dctx, dict, dictSize); + *legacyContext = dctx; + return 0; + } +#endif +#if (ZSTD_LEGACY_SUPPORT <= 7) + case 7 : + { + ZBUFFv07_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv07_createDCtx() : (ZBUFFv07_DCtx*)*legacyContext; + if (dctx==NULL) return ERROR(memory_allocation); + ZBUFFv07_decompressInitDictionary(dctx, dict, dictSize); + *legacyContext = dctx; + return 0; + } +#endif + } +} + + + +MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version, + ZSTD_outBuffer* output, ZSTD_inBuffer* input) +{ + switch(version) + { + default : + case 1 : + case 2 : + case 3 : + (void)legacyContext; (void)output; (void)input; + return ERROR(version_unsupported); +#if (ZSTD_LEGACY_SUPPORT <= 4) + case 4 : + { + ZBUFFv04_DCtx* dctx = (ZBUFFv04_DCtx*) legacyContext; + const void* src = (const char*)input->src + input->pos; + size_t readSize = input->size - input->pos; + void* dst = (char*)output->dst + output->pos; + size_t decodedSize = output->size - output->pos; + size_t const hintSize = ZBUFFv04_decompressContinue(dctx, dst, &decodedSize, src, &readSize); + output->pos += decodedSize; + input->pos += readSize; + return hintSize; + } +#endif +#if (ZSTD_LEGACY_SUPPORT <= 5) + case 5 : + { + ZBUFFv05_DCtx* dctx = (ZBUFFv05_DCtx*) legacyContext; + const void* src = (const char*)input->src + input->pos; + size_t readSize = input->size - input->pos; + void* dst = (char*)output->dst + output->pos; + size_t decodedSize = output->size - output->pos; + size_t const hintSize = ZBUFFv05_decompressContinue(dctx, dst, &decodedSize, src, &readSize); + output->pos += decodedSize; + input->pos += readSize; + return hintSize; + } +#endif +#if (ZSTD_LEGACY_SUPPORT <= 6) + case 6 : + { + ZBUFFv06_DCtx* dctx = (ZBUFFv06_DCtx*) legacyContext; + const void* src = (const char*)input->src + input->pos; + size_t readSize = input->size - input->pos; + void* dst = (char*)output->dst + output->pos; + size_t decodedSize = output->size - output->pos; + size_t const hintSize = ZBUFFv06_decompressContinue(dctx, dst, &decodedSize, src, &readSize); + output->pos += decodedSize; + input->pos += readSize; + return hintSize; + } +#endif +#if (ZSTD_LEGACY_SUPPORT <= 7) + case 7 : + { + ZBUFFv07_DCtx* dctx = (ZBUFFv07_DCtx*) legacyContext; + const void* src = (const char*)input->src + input->pos; + size_t readSize = input->size - input->pos; + void* dst = (char*)output->dst + output->pos; + size_t decodedSize = output->size - output->pos; + size_t const hintSize = ZBUFFv07_decompressContinue(dctx, dst, &decodedSize, src, &readSize); + output->pos += decodedSize; + input->pos += readSize; + return hintSize; + } +#endif + } +} + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_LEGACY_H */ diff --git a/src/borg/algorithms/zstd/lib/legacy/zstd_v01.c b/src/borg/algorithms/zstd/lib/legacy/zstd_v01.c new file mode 100644 index 0000000000..70003cbedf --- /dev/null +++ b/src/borg/algorithms/zstd/lib/legacy/zstd_v01.c @@ -0,0 +1,2127 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/****************************************** +* Includes +******************************************/ +#include /* size_t, ptrdiff_t */ +#include "zstd_v01.h" +#include "error_private.h" + + +/****************************************** +* Static allocation +******************************************/ +/* You can statically allocate FSE CTable/DTable as a table of unsigned using below macro */ +#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) +* Increasing memory usage improves compression ratio +* Reduced memory usage can improve speed, due to cache effect +* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ +#define FSE_MAX_MEMORY_USAGE 14 +#define FSE_DEFAULT_MEMORY_USAGE 13 + +/* FSE_MAX_SYMBOL_VALUE : +* Maximum symbol value authorized. +* Required for proper stack allocation */ +#define FSE_MAX_SYMBOL_VALUE 255 + + +/**************************************************************** +* template functions type & suffix +****************************************************************/ +#define FSE_FUNCTION_TYPE BYTE +#define FSE_FUNCTION_EXTENSION + + +/**************************************************************** +* Byte symbol type +****************************************************************/ +typedef struct +{ + unsigned short newState; + unsigned char symbol; + unsigned char nbBits; +} FSE_decode_t; /* size == U32 */ + + + +/**************************************************************** +* Compiler specifics +****************************************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# include /* For Visual 2005 */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ +#else +# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +# else +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +#endif + + +/**************************************************************** +* Includes +****************************************************************/ +#include /* malloc, free, qsort */ +#include /* memcpy, memset */ +#include /* printf (debug) */ + + +#ifndef MEM_ACCESS_MODULE +#define MEM_ACCESS_MODULE +/**************************************************************** +* Basic Types +*****************************************************************/ +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# include +typedef uint8_t BYTE; +typedef uint16_t U16; +typedef int16_t S16; +typedef uint32_t U32; +typedef int32_t S32; +typedef uint64_t U64; +typedef int64_t S64; +#else +typedef unsigned char BYTE; +typedef unsigned short U16; +typedef signed short S16; +typedef unsigned int U32; +typedef signed int S32; +typedef unsigned long long U64; +typedef signed long long S64; +#endif + +#endif /* MEM_ACCESS_MODULE */ + +/**************************************************************** +* Memory I/O +*****************************************************************/ +/* FSE_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. + * Method 2 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets generating assembly depending on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1 > 2) + */ +#ifndef FSE_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +# define FSE_FORCE_MEMORY_ACCESS 2 +# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \ + (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) +# define FSE_FORCE_MEMORY_ACCESS 1 +# endif +#endif + + +static unsigned FSE_32bits(void) +{ + return sizeof(void*)==4; +} + +static unsigned FSE_isLittleEndian(void) +{ + const union { U32 i; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +} + +#if defined(FSE_FORCE_MEMORY_ACCESS) && (FSE_FORCE_MEMORY_ACCESS==2) + +static U16 FSE_read16(const void* memPtr) { return *(const U16*) memPtr; } +static U32 FSE_read32(const void* memPtr) { return *(const U32*) memPtr; } +static U64 FSE_read64(const void* memPtr) { return *(const U64*) memPtr; } + +#elif defined(FSE_FORCE_MEMORY_ACCESS) && (FSE_FORCE_MEMORY_ACCESS==1) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +typedef union { U16 u16; U32 u32; U64 u64; } __attribute__((packed)) unalign; + +static U16 FSE_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } +static U32 FSE_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } +static U64 FSE_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } + +#else + +static U16 FSE_read16(const void* memPtr) +{ + U16 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +static U32 FSE_read32(const void* memPtr) +{ + U32 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +static U64 FSE_read64(const void* memPtr) +{ + U64 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +#endif // FSE_FORCE_MEMORY_ACCESS + +static U16 FSE_readLE16(const void* memPtr) +{ + if (FSE_isLittleEndian()) + return FSE_read16(memPtr); + else + { + const BYTE* p = (const BYTE*)memPtr; + return (U16)(p[0] + (p[1]<<8)); + } +} + +static U32 FSE_readLE32(const void* memPtr) +{ + if (FSE_isLittleEndian()) + return FSE_read32(memPtr); + else + { + const BYTE* p = (const BYTE*)memPtr; + return (U32)((U32)p[0] + ((U32)p[1]<<8) + ((U32)p[2]<<16) + ((U32)p[3]<<24)); + } +} + + +static U64 FSE_readLE64(const void* memPtr) +{ + if (FSE_isLittleEndian()) + return FSE_read64(memPtr); + else + { + const BYTE* p = (const BYTE*)memPtr; + return (U64)((U64)p[0] + ((U64)p[1]<<8) + ((U64)p[2]<<16) + ((U64)p[3]<<24) + + ((U64)p[4]<<32) + ((U64)p[5]<<40) + ((U64)p[6]<<48) + ((U64)p[7]<<56)); + } +} + +static size_t FSE_readLEST(const void* memPtr) +{ + if (FSE_32bits()) + return (size_t)FSE_readLE32(memPtr); + else + return (size_t)FSE_readLE64(memPtr); +} + + + +/**************************************************************** +* Constants +*****************************************************************/ +#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2) +#define FSE_MAX_TABLESIZE (1U< FSE_TABLELOG_ABSOLUTE_MAX +#error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported" +#endif + + +/**************************************************************** +* Error Management +****************************************************************/ +#define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + + +/**************************************************************** +* Complex types +****************************************************************/ +typedef struct +{ + int deltaFindState; + U32 deltaNbBits; +} FSE_symbolCompressionTransform; /* total 8 bytes */ + +typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; + +/**************************************************************** +* Internal functions +****************************************************************/ +FORCE_INLINE unsigned FSE_highbit32 (register U32 val) +{ +# if defined(_MSC_VER) /* Visual */ + unsigned long r; + _BitScanReverse ( &r, val ); + return (unsigned) r; +# elif defined(__GNUC__) && (GCC_VERSION >= 304) /* GCC Intrinsic */ + return 31 - __builtin_clz (val); +# else /* Software version */ + static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; + U32 v = val; + unsigned r; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + r = DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27]; + return r; +# endif +} + + +/**************************************************************** +* Templates +****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ + +/* safety checks */ +#ifndef FSE_FUNCTION_EXTENSION +# error "FSE_FUNCTION_EXTENSION must be defined" +#endif +#ifndef FSE_FUNCTION_TYPE +# error "FSE_FUNCTION_TYPE must be defined" +#endif + +/* Function names */ +#define FSE_CAT(X,Y) X##Y +#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) +#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) + + + +static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3) + 3; } + +#define FSE_DECODE_TYPE FSE_decode_t + + +typedef struct { + U16 tableLog; + U16 fastMode; +} FSE_DTableHeader; /* sizeof U32 */ + +static size_t FSE_buildDTable +(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +{ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*)(ptr) + 1; /* because dt is unsigned, 32-bits aligned on 32-bits */ + const U32 tableSize = 1 << tableLog; + const U32 tableMask = tableSize-1; + const U32 step = FSE_tableStep(tableSize); + U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1]; + U32 position = 0; + U32 highThreshold = tableSize-1; + const S16 largeLimit= (S16)(1 << (tableLog-1)); + U32 noLarge = 1; + U32 s; + + /* Sanity Checks */ + if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return (size_t)-FSE_ERROR_maxSymbolValue_tooLarge; + if (tableLog > FSE_MAX_TABLELOG) return (size_t)-FSE_ERROR_tableLog_tooLarge; + + /* Init, lay down lowprob symbols */ + DTableH[0].tableLog = (U16)tableLog; + for (s=0; s<=maxSymbolValue; s++) + { + if (normalizedCounter[s]==-1) + { + tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s; + symbolNext[s] = 1; + } + else + { + if (normalizedCounter[s] >= largeLimit) noLarge=0; + symbolNext[s] = normalizedCounter[s]; + } + } + + /* Spread symbols */ + for (s=0; s<=maxSymbolValue; s++) + { + int i; + for (i=0; i highThreshold) position = (position + step) & tableMask; /* lowprob area */ + } + } + + if (position!=0) return (size_t)-FSE_ERROR_GENERIC; /* position must reach all cells once, otherwise normalizedCounter is incorrect */ + + /* Build Decoding table */ + { + U32 i; + for (i=0; ifastMode = (U16)noLarge; + return 0; +} + + +/****************************************** +* FSE byte symbol +******************************************/ +#ifndef FSE_COMMONDEFS_ONLY + +static unsigned FSE_isError(size_t code) { return (code > (size_t)(-FSE_ERROR_maxCode)); } + +static short FSE_abs(short a) +{ + return a<0? -a : a; +} + + +/**************************************************************** +* Header bitstream management +****************************************************************/ +static size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + const BYTE* const istart = (const BYTE*) headerBuffer; + const BYTE* const iend = istart + hbSize; + const BYTE* ip = istart; + int nbBits; + int remaining; + int threshold; + U32 bitStream; + int bitCount; + unsigned charnum = 0; + int previous0 = 0; + + if (hbSize < 4) return (size_t)-FSE_ERROR_srcSize_wrong; + bitStream = FSE_readLE32(ip); + nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */ + if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return (size_t)-FSE_ERROR_tableLog_tooLarge; + bitStream >>= 4; + bitCount = 4; + *tableLogPtr = nbBits; + remaining = (1<1) && (charnum<=*maxSVPtr)) + { + if (previous0) + { + unsigned n0 = charnum; + while ((bitStream & 0xFFFF) == 0xFFFF) + { + n0+=24; + if (ip < iend-5) + { + ip+=2; + bitStream = FSE_readLE32(ip) >> bitCount; + } + else + { + bitStream >>= 16; + bitCount+=16; + } + } + while ((bitStream & 3) == 3) + { + n0+=3; + bitStream>>=2; + bitCount+=2; + } + n0 += bitStream & 3; + bitCount += 2; + if (n0 > *maxSVPtr) return (size_t)-FSE_ERROR_maxSymbolValue_tooSmall; + while (charnum < n0) normalizedCounter[charnum++] = 0; + if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) + { + ip += bitCount>>3; + bitCount &= 7; + bitStream = FSE_readLE32(ip) >> bitCount; + } + else + bitStream >>= 2; + } + { + const short max = (short)((2*threshold-1)-remaining); + short count; + + if ((bitStream & (threshold-1)) < (U32)max) + { + count = (short)(bitStream & (threshold-1)); + bitCount += nbBits-1; + } + else + { + count = (short)(bitStream & (2*threshold-1)); + if (count >= threshold) count -= max; + bitCount += nbBits; + } + + count--; /* extra accuracy */ + remaining -= FSE_abs(count); + normalizedCounter[charnum++] = count; + previous0 = !count; + while (remaining < threshold) + { + nbBits--; + threshold >>= 1; + } + + { + if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) + { + ip += bitCount>>3; + bitCount &= 7; + } + else + { + bitCount -= (int)(8 * (iend - 4 - ip)); + ip = iend - 4; + } + bitStream = FSE_readLE32(ip) >> (bitCount & 31); + } + } + } + if (remaining != 1) return (size_t)-FSE_ERROR_GENERIC; + *maxSVPtr = charnum-1; + + ip += (bitCount+7)>>3; + if ((size_t)(ip-istart) > hbSize) return (size_t)-FSE_ERROR_srcSize_wrong; + return ip-istart; +} + + +/********************************************************* +* Decompression (Byte symbols) +*********************************************************/ +static size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue) +{ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + FSE_decode_t* const cell = (FSE_decode_t*)(ptr) + 1; /* because dt is unsigned */ + + DTableH->tableLog = 0; + DTableH->fastMode = 0; + + cell->newState = 0; + cell->symbol = symbolValue; + cell->nbBits = 0; + + return 0; +} + + +static size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits) +{ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + FSE_decode_t* const dinfo = (FSE_decode_t*)(ptr) + 1; /* because dt is unsigned */ + const unsigned tableSize = 1 << nbBits; + const unsigned tableMask = tableSize - 1; + const unsigned maxSymbolValue = tableMask; + unsigned s; + + /* Sanity checks */ + if (nbBits < 1) return (size_t)-FSE_ERROR_GENERIC; /* min size */ + + /* Build Decoding Table */ + DTableH->tableLog = (U16)nbBits; + DTableH->fastMode = 1; + for (s=0; s<=maxSymbolValue; s++) + { + dinfo[s].newState = 0; + dinfo[s].symbol = (BYTE)s; + dinfo[s].nbBits = (BYTE)nbBits; + } + + return 0; +} + + +/* FSE_initDStream + * Initialize a FSE_DStream_t. + * srcBuffer must point at the beginning of an FSE block. + * The function result is the size of the FSE_block (== srcSize). + * If srcSize is too small, the function will return an errorCode; + */ +static size_t FSE_initDStream(FSE_DStream_t* bitD, const void* srcBuffer, size_t srcSize) +{ + if (srcSize < 1) return (size_t)-FSE_ERROR_srcSize_wrong; + + if (srcSize >= sizeof(size_t)) + { + U32 contain32; + bitD->start = (const char*)srcBuffer; + bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(size_t); + bitD->bitContainer = FSE_readLEST(bitD->ptr); + contain32 = ((const BYTE*)srcBuffer)[srcSize-1]; + if (contain32 == 0) return (size_t)-FSE_ERROR_GENERIC; /* stop bit not present */ + bitD->bitsConsumed = 8 - FSE_highbit32(contain32); + } + else + { + U32 contain32; + bitD->start = (const char*)srcBuffer; + bitD->ptr = bitD->start; + bitD->bitContainer = *(const BYTE*)(bitD->start); + switch(srcSize) + { + case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16); + case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24); + case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32); + case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24; + case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16; + case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8; + default:; + } + contain32 = ((const BYTE*)srcBuffer)[srcSize-1]; + if (contain32 == 0) return (size_t)-FSE_ERROR_GENERIC; /* stop bit not present */ + bitD->bitsConsumed = 8 - FSE_highbit32(contain32); + bitD->bitsConsumed += (U32)(sizeof(size_t) - srcSize)*8; + } + + return srcSize; +} + + +/*!FSE_lookBits + * Provides next n bits from the bitContainer. + * bitContainer is not modified (bits are still present for next read/look) + * On 32-bits, maxNbBits==25 + * On 64-bits, maxNbBits==57 + * return : value extracted. + */ +static size_t FSE_lookBits(FSE_DStream_t* bitD, U32 nbBits) +{ + const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1; + return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask); +} + +static size_t FSE_lookBitsFast(FSE_DStream_t* bitD, U32 nbBits) /* only if nbBits >= 1 !! */ +{ + const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1; + return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask); +} + +static void FSE_skipBits(FSE_DStream_t* bitD, U32 nbBits) +{ + bitD->bitsConsumed += nbBits; +} + + +/*!FSE_readBits + * Read next n bits from the bitContainer. + * On 32-bits, don't read more than maxNbBits==25 + * On 64-bits, don't read more than maxNbBits==57 + * Use the fast variant *only* if n >= 1. + * return : value extracted. + */ +static size_t FSE_readBits(FSE_DStream_t* bitD, U32 nbBits) +{ + size_t value = FSE_lookBits(bitD, nbBits); + FSE_skipBits(bitD, nbBits); + return value; +} + +static size_t FSE_readBitsFast(FSE_DStream_t* bitD, U32 nbBits) /* only if nbBits >= 1 !! */ +{ + size_t value = FSE_lookBitsFast(bitD, nbBits); + FSE_skipBits(bitD, nbBits); + return value; +} + +static unsigned FSE_reloadDStream(FSE_DStream_t* bitD) +{ + if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */ + return FSE_DStream_tooFar; + + if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) + { + bitD->ptr -= bitD->bitsConsumed >> 3; + bitD->bitsConsumed &= 7; + bitD->bitContainer = FSE_readLEST(bitD->ptr); + return FSE_DStream_unfinished; + } + if (bitD->ptr == bitD->start) + { + if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return FSE_DStream_endOfBuffer; + return FSE_DStream_completed; + } + { + U32 nbBytes = bitD->bitsConsumed >> 3; + U32 result = FSE_DStream_unfinished; + if (bitD->ptr - nbBytes < bitD->start) + { + nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */ + result = FSE_DStream_endOfBuffer; + } + bitD->ptr -= nbBytes; + bitD->bitsConsumed -= nbBytes*8; + bitD->bitContainer = FSE_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */ + return result; + } +} + + +static void FSE_initDState(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD, const FSE_DTable* dt) +{ + const void* ptr = dt; + const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr; + DStatePtr->state = FSE_readBits(bitD, DTableH->tableLog); + FSE_reloadDStream(bitD); + DStatePtr->table = dt + 1; +} + +static BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD) +{ + const FSE_decode_t DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + const U32 nbBits = DInfo.nbBits; + BYTE symbol = DInfo.symbol; + size_t lowBits = FSE_readBits(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +static BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, FSE_DStream_t* bitD) +{ + const FSE_decode_t DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + const U32 nbBits = DInfo.nbBits; + BYTE symbol = DInfo.symbol; + size_t lowBits = FSE_readBitsFast(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +/* FSE_endOfDStream + Tells if bitD has reached end of bitStream or not */ + +static unsigned FSE_endOfDStream(const FSE_DStream_t* bitD) +{ + return ((bitD->ptr == bitD->start) && (bitD->bitsConsumed == sizeof(bitD->bitContainer)*8)); +} + +static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) +{ + return DStatePtr->state == 0; +} + + +FORCE_INLINE size_t FSE_decompress_usingDTable_generic( + void* dst, size_t maxDstSize, + const void* cSrc, size_t cSrcSize, + const FSE_DTable* dt, const unsigned fast) +{ + BYTE* const ostart = (BYTE*) dst; + BYTE* op = ostart; + BYTE* const omax = op + maxDstSize; + BYTE* const olimit = omax-3; + + FSE_DStream_t bitD; + FSE_DState_t state1; + FSE_DState_t state2; + size_t errorCode; + + /* Init */ + errorCode = FSE_initDStream(&bitD, cSrc, cSrcSize); /* replaced last arg by maxCompressed Size */ + if (FSE_isError(errorCode)) return errorCode; + + FSE_initDState(&state1, &bitD, dt); + FSE_initDState(&state2, &bitD, dt); + +#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD) + + /* 4 symbols per loop */ + for ( ; (FSE_reloadDStream(&bitD)==FSE_DStream_unfinished) && (op sizeof(bitD.bitContainer)*8) /* This test must be static */ + FSE_reloadDStream(&bitD); + + op[1] = FSE_GETSYMBOL(&state2); + + if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + { if (FSE_reloadDStream(&bitD) > FSE_DStream_unfinished) { op+=2; break; } } + + op[2] = FSE_GETSYMBOL(&state1); + + if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + FSE_reloadDStream(&bitD); + + op[3] = FSE_GETSYMBOL(&state2); + } + + /* tail */ + /* note : FSE_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly FSE_DStream_completed */ + while (1) + { + if ( (FSE_reloadDStream(&bitD)>FSE_DStream_completed) || (op==omax) || (FSE_endOfDStream(&bitD) && (fast || FSE_endOfDState(&state1))) ) + break; + + *op++ = FSE_GETSYMBOL(&state1); + + if ( (FSE_reloadDStream(&bitD)>FSE_DStream_completed) || (op==omax) || (FSE_endOfDStream(&bitD) && (fast || FSE_endOfDState(&state2))) ) + break; + + *op++ = FSE_GETSYMBOL(&state2); + } + + /* end ? */ + if (FSE_endOfDStream(&bitD) && FSE_endOfDState(&state1) && FSE_endOfDState(&state2)) + return op-ostart; + + if (op==omax) return (size_t)-FSE_ERROR_dstSize_tooSmall; /* dst buffer is full, but cSrc unfinished */ + + return (size_t)-FSE_ERROR_corruptionDetected; +} + + +static size_t FSE_decompress_usingDTable(void* dst, size_t originalSize, + const void* cSrc, size_t cSrcSize, + const FSE_DTable* dt) +{ + FSE_DTableHeader DTableH; + memcpy(&DTableH, dt, sizeof(DTableH)); /* memcpy() into local variable, to avoid strict aliasing warning */ + + /* select fast mode (static) */ + if (DTableH.fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); + return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); +} + + +static size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize) +{ + const BYTE* const istart = (const BYTE*)cSrc; + const BYTE* ip = istart; + short counting[FSE_MAX_SYMBOL_VALUE+1]; + DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */ + unsigned tableLog; + unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; + size_t errorCode; + + if (cSrcSize<2) return (size_t)-FSE_ERROR_srcSize_wrong; /* too small input size */ + + /* normal FSE decoding mode */ + errorCode = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize); + if (FSE_isError(errorCode)) return errorCode; + if (errorCode >= cSrcSize) return (size_t)-FSE_ERROR_srcSize_wrong; /* too small input size */ + ip += errorCode; + cSrcSize -= errorCode; + + errorCode = FSE_buildDTable (dt, counting, maxSymbolValue, tableLog); + if (FSE_isError(errorCode)) return errorCode; + + /* always return, even if it is an error code */ + return FSE_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt); +} + + + +/* ******************************************************* +* Huff0 : Huffman block compression +*********************************************************/ +#define HUF_MAX_SYMBOL_VALUE 255 +#define HUF_DEFAULT_TABLELOG 12 /* used by default, when not specified */ +#define HUF_MAX_TABLELOG 12 /* max possible tableLog; for allocation purpose; can be modified */ +#define HUF_ABSOLUTEMAX_TABLELOG 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ +#if (HUF_MAX_TABLELOG > HUF_ABSOLUTEMAX_TABLELOG) +# error "HUF_MAX_TABLELOG is too large !" +#endif + +typedef struct HUF_CElt_s { + U16 val; + BYTE nbBits; +} HUF_CElt ; + +typedef struct nodeElt_s { + U32 count; + U16 parent; + BYTE byte; + BYTE nbBits; +} nodeElt; + + +/* ******************************************************* +* Huff0 : Huffman block decompression +*********************************************************/ +typedef struct { + BYTE byte; + BYTE nbBits; +} HUF_DElt; + +static size_t HUF_readDTable (U16* DTable, const void* src, size_t srcSize) +{ + BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1]; + U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; /* large enough for values from 0 to 16 */ + U32 weightTotal; + U32 maxBits; + const BYTE* ip = (const BYTE*) src; + size_t iSize; + size_t oSize; + U32 n; + U32 nextRankStart; + void* ptr = DTable+1; + HUF_DElt* const dt = (HUF_DElt*)ptr; + + if (!srcSize) return (size_t)-FSE_ERROR_srcSize_wrong; + iSize = ip[0]; + + FSE_STATIC_ASSERT(sizeof(HUF_DElt) == sizeof(U16)); /* if compilation fails here, assertion is false */ + //memset(huffWeight, 0, sizeof(huffWeight)); /* should not be necessary, but some analyzer complain ... */ + if (iSize >= 128) /* special header */ + { + if (iSize >= (242)) /* RLE */ + { + static int l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 }; + oSize = l[iSize-242]; + memset(huffWeight, 1, sizeof(huffWeight)); + iSize = 0; + } + else /* Incompressible */ + { + oSize = iSize - 127; + iSize = ((oSize+1)/2); + if (iSize+1 > srcSize) return (size_t)-FSE_ERROR_srcSize_wrong; + ip += 1; + for (n=0; n> 4; + huffWeight[n+1] = ip[n/2] & 15; + } + } + } + else /* header compressed with FSE (normal case) */ + { + if (iSize+1 > srcSize) return (size_t)-FSE_ERROR_srcSize_wrong; + oSize = FSE_decompress(huffWeight, HUF_MAX_SYMBOL_VALUE, ip+1, iSize); /* max 255 values decoded, last one is implied */ + if (FSE_isError(oSize)) return oSize; + } + + /* collect weight stats */ + memset(rankVal, 0, sizeof(rankVal)); + weightTotal = 0; + for (n=0; n= HUF_ABSOLUTEMAX_TABLELOG) return (size_t)-FSE_ERROR_corruptionDetected; + rankVal[huffWeight[n]]++; + weightTotal += (1 << huffWeight[n]) >> 1; + } + if (weightTotal == 0) return (size_t)-FSE_ERROR_corruptionDetected; + + /* get last non-null symbol weight (implied, total must be 2^n) */ + maxBits = FSE_highbit32(weightTotal) + 1; + if (maxBits > DTable[0]) return (size_t)-FSE_ERROR_tableLog_tooLarge; /* DTable is too small */ + DTable[0] = (U16)maxBits; + { + U32 total = 1 << maxBits; + U32 rest = total - weightTotal; + U32 verif = 1 << FSE_highbit32(rest); + U32 lastWeight = FSE_highbit32(rest) + 1; + if (verif != rest) return (size_t)-FSE_ERROR_corruptionDetected; /* last value must be a clean power of 2 */ + huffWeight[oSize] = (BYTE)lastWeight; + rankVal[lastWeight]++; + } + + /* check tree construction validity */ + if ((rankVal[1] < 2) || (rankVal[1] & 1)) return (size_t)-FSE_ERROR_corruptionDetected; /* by construction : at least 2 elts of rank 1, must be even */ + + /* Prepare ranks */ + nextRankStart = 0; + for (n=1; n<=maxBits; n++) + { + U32 current = nextRankStart; + nextRankStart += (rankVal[n] << (n-1)); + rankVal[n] = current; + } + + /* fill DTable */ + for (n=0; n<=oSize; n++) + { + const U32 w = huffWeight[n]; + const U32 length = (1 << w) >> 1; + U32 i; + HUF_DElt D; + D.byte = (BYTE)n; D.nbBits = (BYTE)(maxBits + 1 - w); + for (i = rankVal[w]; i < rankVal[w] + length; i++) + dt[i] = D; + rankVal[w] += length; + } + + return iSize+1; +} + + +static BYTE HUF_decodeSymbol(FSE_DStream_t* Dstream, const HUF_DElt* dt, const U32 dtLog) +{ + const size_t val = FSE_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ + const BYTE c = dt[val].byte; + FSE_skipBits(Dstream, dt[val].nbBits); + return c; +} + +static size_t HUF_decompress_usingDTable( /* -3% slower when non static */ + void* dst, size_t maxDstSize, + const void* cSrc, size_t cSrcSize, + const U16* DTable) +{ + BYTE* const ostart = (BYTE*) dst; + BYTE* op = ostart; + BYTE* const omax = op + maxDstSize; + BYTE* const olimit = omax-15; + + const void* ptr = DTable; + const HUF_DElt* const dt = (const HUF_DElt*)(ptr)+1; + const U32 dtLog = DTable[0]; + size_t errorCode; + U32 reloadStatus; + + /* Init */ + + const U16* jumpTable = (const U16*)cSrc; + const size_t length1 = FSE_readLE16(jumpTable); + const size_t length2 = FSE_readLE16(jumpTable+1); + const size_t length3 = FSE_readLE16(jumpTable+2); + const size_t length4 = cSrcSize - 6 - length1 - length2 - length3; // check coherency !! + const char* const start1 = (const char*)(cSrc) + 6; + const char* const start2 = start1 + length1; + const char* const start3 = start2 + length2; + const char* const start4 = start3 + length3; + FSE_DStream_t bitD1, bitD2, bitD3, bitD4; + + if (length1+length2+length3+6 >= cSrcSize) return (size_t)-FSE_ERROR_srcSize_wrong; + + errorCode = FSE_initDStream(&bitD1, start1, length1); + if (FSE_isError(errorCode)) return errorCode; + errorCode = FSE_initDStream(&bitD2, start2, length2); + if (FSE_isError(errorCode)) return errorCode; + errorCode = FSE_initDStream(&bitD3, start3, length3); + if (FSE_isError(errorCode)) return errorCode; + errorCode = FSE_initDStream(&bitD4, start4, length4); + if (FSE_isError(errorCode)) return errorCode; + + reloadStatus=FSE_reloadDStream(&bitD2); + + /* 16 symbols per loop */ + for ( ; (reloadStatus12)) FSE_reloadDStream(&Dstream) + +#define HUF_DECODE_SYMBOL_2(n, Dstream) \ + op[n] = HUF_decodeSymbol(&Dstream, dt, dtLog); \ + if (FSE_32bits()) FSE_reloadDStream(&Dstream) + + HUF_DECODE_SYMBOL_1( 0, bitD1); + HUF_DECODE_SYMBOL_1( 1, bitD2); + HUF_DECODE_SYMBOL_1( 2, bitD3); + HUF_DECODE_SYMBOL_1( 3, bitD4); + HUF_DECODE_SYMBOL_2( 4, bitD1); + HUF_DECODE_SYMBOL_2( 5, bitD2); + HUF_DECODE_SYMBOL_2( 6, bitD3); + HUF_DECODE_SYMBOL_2( 7, bitD4); + HUF_DECODE_SYMBOL_1( 8, bitD1); + HUF_DECODE_SYMBOL_1( 9, bitD2); + HUF_DECODE_SYMBOL_1(10, bitD3); + HUF_DECODE_SYMBOL_1(11, bitD4); + HUF_DECODE_SYMBOL_0(12, bitD1); + HUF_DECODE_SYMBOL_0(13, bitD2); + HUF_DECODE_SYMBOL_0(14, bitD3); + HUF_DECODE_SYMBOL_0(15, bitD4); + } + + if (reloadStatus!=FSE_DStream_completed) /* not complete : some bitStream might be FSE_DStream_unfinished */ + return (size_t)-FSE_ERROR_corruptionDetected; + + /* tail */ + { + // bitTail = bitD1; // *much* slower : -20% !??! + FSE_DStream_t bitTail; + bitTail.ptr = bitD1.ptr; + bitTail.bitsConsumed = bitD1.bitsConsumed; + bitTail.bitContainer = bitD1.bitContainer; // required in case of FSE_DStream_endOfBuffer + bitTail.start = start1; + for ( ; (FSE_reloadDStream(&bitTail) < FSE_DStream_completed) && (op= cSrcSize) return (size_t)-FSE_ERROR_srcSize_wrong; + ip += errorCode; + cSrcSize -= errorCode; + + return HUF_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, DTable); +} + + +#endif /* FSE_COMMONDEFS_ONLY */ + +/* + zstd - standard compression library + Copyright (C) 2014-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - zstd source repository : https://github.com/Cyan4973/zstd + - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + +/**************************************************************** +* Tuning parameters +*****************************************************************/ +/* MEMORY_USAGE : +* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) +* Increasing memory usage improves compression ratio +* Reduced memory usage can improve speed, due to cache effect */ +#define ZSTD_MEMORY_USAGE 17 + + +/************************************** + CPU Feature Detection +**************************************/ +/* + * Automated efficient unaligned memory access detection + * Based on known hardware architectures + * This list will be updated thanks to feedbacks + */ +#if defined(CPU_HAS_EFFICIENT_UNALIGNED_MEMORY_ACCESS) \ + || defined(__ARM_FEATURE_UNALIGNED) \ + || defined(__i386__) || defined(__x86_64__) \ + || defined(_M_IX86) || defined(_M_X64) \ + || defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_8__) \ + || (defined(_M_ARM) && (_M_ARM >= 7)) +# define ZSTD_UNALIGNED_ACCESS 1 +#else +# define ZSTD_UNALIGNED_ACCESS 0 +#endif + + +/******************************************************** +* Includes +*********************************************************/ +#include /* calloc */ +#include /* memcpy, memmove */ +#include /* debug : printf */ + + +/******************************************************** +* Compiler specifics +*********************************************************/ +#ifdef __AVX2__ +# include /* AVX2 intrinsics */ +#endif + +#ifdef _MSC_VER /* Visual Studio */ +# include /* For Visual 2005 */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4324) /* disable: C4324: padded structure */ +#endif + + +#ifndef MEM_ACCESS_MODULE +#define MEM_ACCESS_MODULE +/******************************************************** +* Basic Types +*********************************************************/ +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# include +typedef uint8_t BYTE; +typedef uint16_t U16; +typedef int16_t S16; +typedef uint32_t U32; +typedef int32_t S32; +typedef uint64_t U64; +#else +typedef unsigned char BYTE; +typedef unsigned short U16; +typedef signed short S16; +typedef unsigned int U32; +typedef signed int S32; +typedef unsigned long long U64; +#endif + +#endif /* MEM_ACCESS_MODULE */ + + +/******************************************************** +* Constants +*********************************************************/ +static const U32 ZSTD_magicNumber = 0xFD2FB51E; /* 3rd version : seqNb header */ + +#define HASH_LOG (ZSTD_MEMORY_USAGE - 2) +#define HASH_TABLESIZE (1 << HASH_LOG) +#define HASH_MASK (HASH_TABLESIZE - 1) + +#define KNUTH 2654435761 + +#define BIT7 128 +#define BIT6 64 +#define BIT5 32 +#define BIT4 16 + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define BLOCKSIZE (128 KB) /* define, for static allocation */ + +#define WORKPLACESIZE (BLOCKSIZE*3) +#define MINMATCH 4 +#define MLbits 7 +#define LLbits 6 +#define Offbits 5 +#define MaxML ((1<>3]; +#else + U32 hashTable[HASH_TABLESIZE]; +#endif + BYTE buffer[WORKPLACESIZE]; +} cctxi_t; + + + + +/************************************** +* Error Management +**************************************/ +/* published entry point */ +unsigned ZSTDv01_isError(size_t code) { return ERR_isError(code); } + + +/************************************** +* Tool functions +**************************************/ +#define ZSTD_VERSION_MAJOR 0 /* for breaking interface changes */ +#define ZSTD_VERSION_MINOR 1 /* for new (non-breaking) interface capabilities */ +#define ZSTD_VERSION_RELEASE 3 /* for tweaks, bug-fixes, or development */ +#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) + +/************************************************************** +* Decompression code +**************************************************************/ + +size_t ZSTDv01_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr) +{ + const BYTE* const in = (const BYTE* const)src; + BYTE headerFlags; + U32 cSize; + + if (srcSize < 3) return ERROR(srcSize_wrong); + + headerFlags = *in; + cSize = in[2] + (in[1]<<8) + ((in[0] & 7)<<16); + + bpPtr->blockType = (blockType_t)(headerFlags >> 6); + bpPtr->origSize = (bpPtr->blockType == bt_rle) ? cSize : 0; + + if (bpPtr->blockType == bt_end) return 0; + if (bpPtr->blockType == bt_rle) return 1; + return cSize; +} + + +static size_t ZSTD_copyUncompressedBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize) +{ + if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall); + memcpy(dst, src, srcSize); + return srcSize; +} + + +static size_t ZSTD_decompressLiterals(void* ctx, + void* dst, size_t maxDstSize, + const void* src, size_t srcSize) +{ + BYTE* op = (BYTE*)dst; + BYTE* const oend = op + maxDstSize; + const BYTE* ip = (const BYTE*)src; + size_t errorCode; + size_t litSize; + + /* check : minimum 2, for litSize, +1, for content */ + if (srcSize <= 3) return ERROR(corruption_detected); + + litSize = ip[1] + (ip[0]<<8); + litSize += ((ip[-3] >> 3) & 7) << 16; // mmmmh.... + op = oend - litSize; + + (void)ctx; + if (litSize > maxDstSize) return ERROR(dstSize_tooSmall); + errorCode = HUF_decompress(op, litSize, ip+2, srcSize-2); + if (FSE_isError(errorCode)) return ERROR(GENERIC); + return litSize; +} + + +size_t ZSTDv01_decodeLiteralsBlock(void* ctx, + void* dst, size_t maxDstSize, + const BYTE** litStart, size_t* litSize, + const void* src, size_t srcSize) +{ + const BYTE* const istart = (const BYTE* const)src; + const BYTE* ip = istart; + BYTE* const ostart = (BYTE* const)dst; + BYTE* const oend = ostart + maxDstSize; + blockProperties_t litbp; + + size_t litcSize = ZSTDv01_getcBlockSize(src, srcSize, &litbp); + if (ZSTDv01_isError(litcSize)) return litcSize; + if (litcSize > srcSize - ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); + ip += ZSTD_blockHeaderSize; + + switch(litbp.blockType) + { + case bt_raw: + *litStart = ip; + ip += litcSize; + *litSize = litcSize; + break; + case bt_rle: + { + size_t rleSize = litbp.origSize; + if (rleSize>maxDstSize) return ERROR(dstSize_tooSmall); + if (!srcSize) return ERROR(srcSize_wrong); + memset(oend - rleSize, *ip, rleSize); + *litStart = oend - rleSize; + *litSize = rleSize; + ip++; + break; + } + case bt_compressed: + { + size_t decodedLitSize = ZSTD_decompressLiterals(ctx, dst, maxDstSize, ip, litcSize); + if (ZSTDv01_isError(decodedLitSize)) return decodedLitSize; + *litStart = oend - decodedLitSize; + *litSize = decodedLitSize; + ip += litcSize; + break; + } + case bt_end: + default: + return ERROR(GENERIC); + } + + return ip-istart; +} + + +size_t ZSTDv01_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr, + FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, + const void* src, size_t srcSize) +{ + const BYTE* const istart = (const BYTE* const)src; + const BYTE* ip = istart; + const BYTE* const iend = istart + srcSize; + U32 LLtype, Offtype, MLtype; + U32 LLlog, Offlog, MLlog; + size_t dumpsLength; + + /* check */ + if (srcSize < 5) return ERROR(srcSize_wrong); + + /* SeqHead */ + *nbSeq = ZSTD_readLE16(ip); ip+=2; + LLtype = *ip >> 6; + Offtype = (*ip >> 4) & 3; + MLtype = (*ip >> 2) & 3; + if (*ip & 2) + { + dumpsLength = ip[2]; + dumpsLength += ip[1] << 8; + ip += 3; + } + else + { + dumpsLength = ip[1]; + dumpsLength += (ip[0] & 1) << 8; + ip += 2; + } + *dumpsPtr = ip; + ip += dumpsLength; + *dumpsLengthPtr = dumpsLength; + + /* check */ + if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */ + + /* sequences */ + { + S16 norm[MaxML+1]; /* assumption : MaxML >= MaxLL and MaxOff */ + size_t headerSize; + + /* Build DTables */ + switch(LLtype) + { + case bt_rle : + LLlog = 0; + FSE_buildDTable_rle(DTableLL, *ip++); break; + case bt_raw : + LLlog = LLbits; + FSE_buildDTable_raw(DTableLL, LLbits); break; + default : + { U32 max = MaxLL; + headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip); + if (FSE_isError(headerSize)) return ERROR(GENERIC); + if (LLlog > LLFSELog) return ERROR(corruption_detected); + ip += headerSize; + FSE_buildDTable(DTableLL, norm, max, LLlog); + } } + + switch(Offtype) + { + case bt_rle : + Offlog = 0; + if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */ + FSE_buildDTable_rle(DTableOffb, *ip++); break; + case bt_raw : + Offlog = Offbits; + FSE_buildDTable_raw(DTableOffb, Offbits); break; + default : + { U32 max = MaxOff; + headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip); + if (FSE_isError(headerSize)) return ERROR(GENERIC); + if (Offlog > OffFSELog) return ERROR(corruption_detected); + ip += headerSize; + FSE_buildDTable(DTableOffb, norm, max, Offlog); + } } + + switch(MLtype) + { + case bt_rle : + MLlog = 0; + if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */ + FSE_buildDTable_rle(DTableML, *ip++); break; + case bt_raw : + MLlog = MLbits; + FSE_buildDTable_raw(DTableML, MLbits); break; + default : + { U32 max = MaxML; + headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip); + if (FSE_isError(headerSize)) return ERROR(GENERIC); + if (MLlog > MLFSELog) return ERROR(corruption_detected); + ip += headerSize; + FSE_buildDTable(DTableML, norm, max, MLlog); + } } } + + return ip-istart; +} + + +typedef struct { + size_t litLength; + size_t offset; + size_t matchLength; +} seq_t; + +typedef struct { + FSE_DStream_t DStream; + FSE_DState_t stateLL; + FSE_DState_t stateOffb; + FSE_DState_t stateML; + size_t prevOffset; + const BYTE* dumps; + const BYTE* dumpsEnd; +} seqState_t; + + +static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState) +{ + size_t litLength; + size_t prevOffset; + size_t offset; + size_t matchLength; + const BYTE* dumps = seqState->dumps; + const BYTE* const de = seqState->dumpsEnd; + + /* Literal length */ + litLength = FSE_decodeSymbol(&(seqState->stateLL), &(seqState->DStream)); + prevOffset = litLength ? seq->offset : seqState->prevOffset; + seqState->prevOffset = seq->offset; + if (litLength == MaxLL) + { + U32 add = dumps 1 byte */ + dumps += 3; + } + } + } + + /* Offset */ + { + U32 offsetCode, nbBits; + offsetCode = FSE_decodeSymbol(&(seqState->stateOffb), &(seqState->DStream)); + if (ZSTD_32bits()) FSE_reloadDStream(&(seqState->DStream)); + nbBits = offsetCode - 1; + if (offsetCode==0) nbBits = 0; /* cmove */ + offset = ((size_t)1 << (nbBits & ((sizeof(offset)*8)-1))) + FSE_readBits(&(seqState->DStream), nbBits); + if (ZSTD_32bits()) FSE_reloadDStream(&(seqState->DStream)); + if (offsetCode==0) offset = prevOffset; + } + + /* MatchLength */ + matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream)); + if (matchLength == MaxML) + { + U32 add = dumps 1 byte */ + dumps += 3; + } + } + } + matchLength += MINMATCH; + + /* save result */ + seq->litLength = litLength; + seq->offset = offset; + seq->matchLength = matchLength; + seqState->dumps = dumps; +} + + +static size_t ZSTD_execSequence(BYTE* op, + seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + BYTE* const base, BYTE* const oend) +{ + static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */ + static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* substracted */ + const BYTE* const ostart = op; + const size_t litLength = sequence.litLength; + BYTE* const endMatch = op + litLength + sequence.matchLength; /* risk : address space overflow (32-bits) */ + const BYTE* const litEnd = *litPtr + litLength; + + /* check */ + if (endMatch > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */ + if (litEnd > litLimit) return ERROR(corruption_detected); + if (sequence.matchLength > (size_t)(*litPtr-op)) return ERROR(dstSize_tooSmall); /* overwrite literal segment */ + + /* copy Literals */ + if (((size_t)(*litPtr - op) < 8) || ((size_t)(oend-litEnd) < 8) || (op+litLength > oend-8)) + memmove(op, *litPtr, litLength); /* overwrite risk */ + else + ZSTD_wildcopy(op, *litPtr, litLength); + op += litLength; + *litPtr = litEnd; /* update for next sequence */ + + /* check : last match must be at a minimum distance of 8 from end of dest buffer */ + if (oend-op < 8) return ERROR(dstSize_tooSmall); + + /* copy Match */ + { + const U32 overlapRisk = (((size_t)(litEnd - endMatch)) < 12); + const BYTE* match = op - sequence.offset; /* possible underflow at op - offset ? */ + size_t qutt = 12; + U64 saved[2]; + + /* check */ + if (match < base) return ERROR(corruption_detected); + if (sequence.offset > (size_t)base) return ERROR(corruption_detected); + + /* save beginning of literal sequence, in case of write overlap */ + if (overlapRisk) + { + if ((endMatch + qutt) > oend) qutt = oend-endMatch; + memcpy(saved, endMatch, qutt); + } + + if (sequence.offset < 8) + { + const int dec64 = dec64table[sequence.offset]; + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += dec32table[sequence.offset]; + ZSTD_copy4(op+4, match); + match -= dec64; + } else { ZSTD_copy8(op, match); } + op += 8; match += 8; + + if (endMatch > oend-(16-MINMATCH)) + { + if (op < oend-8) + { + ZSTD_wildcopy(op, match, (oend-8) - op); + match += (oend-8) - op; + op = oend-8; + } + while (opLLTable; + U32* DTableML = dctx->MLTable; + U32* DTableOffb = dctx->OffTable; + BYTE* const base = (BYTE*) (dctx->base); + + /* Build Decoding Tables */ + errorCode = ZSTDv01_decodeSeqHeaders(&nbSeq, &dumps, &dumpsLength, + DTableLL, DTableML, DTableOffb, + ip, iend-ip); + if (ZSTDv01_isError(errorCode)) return errorCode; + ip += errorCode; + + /* Regen sequences */ + { + seq_t sequence; + seqState_t seqState; + + memset(&sequence, 0, sizeof(sequence)); + seqState.dumps = dumps; + seqState.dumpsEnd = dumps + dumpsLength; + seqState.prevOffset = 1; + errorCode = FSE_initDStream(&(seqState.DStream), ip, iend-ip); + if (FSE_isError(errorCode)) return ERROR(corruption_detected); + FSE_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL); + FSE_initDState(&(seqState.stateOffb), &(seqState.DStream), DTableOffb); + FSE_initDState(&(seqState.stateML), &(seqState.DStream), DTableML); + + for ( ; (FSE_reloadDStream(&(seqState.DStream)) <= FSE_DStream_completed) && (nbSeq>0) ; ) + { + size_t oneSeqSize; + nbSeq--; + ZSTD_decodeSequence(&sequence, &seqState); + oneSeqSize = ZSTD_execSequence(op, sequence, &litPtr, litEnd, base, oend); + if (ZSTDv01_isError(oneSeqSize)) return oneSeqSize; + op += oneSeqSize; + } + + /* check if reached exact end */ + if ( !FSE_endOfDStream(&(seqState.DStream)) ) return ERROR(corruption_detected); /* requested too much : data is corrupted */ + if (nbSeq<0) return ERROR(corruption_detected); /* requested too many sequences : data is corrupted */ + + /* last literal segment */ + { + size_t lastLLSize = litEnd - litPtr; + if (op+lastLLSize > oend) return ERROR(dstSize_tooSmall); + if (op != litPtr) memmove(op, litPtr, lastLLSize); + op += lastLLSize; + } + } + + return op-ostart; +} + + +static size_t ZSTD_decompressBlock( + void* ctx, + void* dst, size_t maxDstSize, + const void* src, size_t srcSize) +{ + /* blockType == blockCompressed, srcSize is trusted */ + const BYTE* ip = (const BYTE*)src; + const BYTE* litPtr = NULL; + size_t litSize = 0; + size_t errorCode; + + /* Decode literals sub-block */ + errorCode = ZSTDv01_decodeLiteralsBlock(ctx, dst, maxDstSize, &litPtr, &litSize, src, srcSize); + if (ZSTDv01_isError(errorCode)) return errorCode; + ip += errorCode; + srcSize -= errorCode; + + return ZSTD_decompressSequences(ctx, dst, maxDstSize, ip, srcSize, litPtr, litSize); +} + + +size_t ZSTDv01_decompressDCtx(void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + const BYTE* iend = ip + srcSize; + BYTE* const ostart = (BYTE* const)dst; + BYTE* op = ostart; + BYTE* const oend = ostart + maxDstSize; + size_t remainingSize = srcSize; + U32 magicNumber; + size_t errorCode=0; + blockProperties_t blockProperties; + + /* Frame Header */ + if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); + magicNumber = ZSTD_readBE32(src); + if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown); + ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize; + + /* Loop on each block */ + while (1) + { + size_t blockSize = ZSTDv01_getcBlockSize(ip, iend-ip, &blockProperties); + if (ZSTDv01_isError(blockSize)) return blockSize; + + ip += ZSTD_blockHeaderSize; + remainingSize -= ZSTD_blockHeaderSize; + if (blockSize > remainingSize) return ERROR(srcSize_wrong); + + switch(blockProperties.blockType) + { + case bt_compressed: + errorCode = ZSTD_decompressBlock(ctx, op, oend-op, ip, blockSize); + break; + case bt_raw : + errorCode = ZSTD_copyUncompressedBlock(op, oend-op, ip, blockSize); + break; + case bt_rle : + return ERROR(GENERIC); /* not yet supported */ + break; + case bt_end : + /* end of frame */ + if (remainingSize) return ERROR(srcSize_wrong); + break; + default: + return ERROR(GENERIC); + } + if (blockSize == 0) break; /* bt_end */ + + if (ZSTDv01_isError(errorCode)) return errorCode; + op += errorCode; + ip += blockSize; + remainingSize -= blockSize; + } + + return op-ostart; +} + +size_t ZSTDv01_decompress(void* dst, size_t maxDstSize, const void* src, size_t srcSize) +{ + dctx_t ctx; + ctx.base = dst; + return ZSTDv01_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize); +} + +size_t ZSTDv01_findFrameCompressedSize(const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + size_t remainingSize = srcSize; + U32 magicNumber; + blockProperties_t blockProperties; + + /* Frame Header */ + if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); + magicNumber = ZSTD_readBE32(src); + if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown); + ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize; + + /* Loop on each block */ + while (1) + { + size_t blockSize = ZSTDv01_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTDv01_isError(blockSize)) return blockSize; + + ip += ZSTD_blockHeaderSize; + remainingSize -= ZSTD_blockHeaderSize; + if (blockSize > remainingSize) return ERROR(srcSize_wrong); + + if (blockSize == 0) break; /* bt_end */ + + ip += blockSize; + remainingSize -= blockSize; + } + + return ip - (const BYTE*)src; +} + +/******************************* +* Streaming Decompression API +*******************************/ + +size_t ZSTDv01_resetDCtx(ZSTDv01_Dctx* dctx) +{ + dctx->expected = ZSTD_frameHeaderSize; + dctx->phase = 0; + dctx->previousDstEnd = NULL; + dctx->base = NULL; + return 0; +} + +ZSTDv01_Dctx* ZSTDv01_createDCtx(void) +{ + ZSTDv01_Dctx* dctx = (ZSTDv01_Dctx*)malloc(sizeof(ZSTDv01_Dctx)); + if (dctx==NULL) return NULL; + ZSTDv01_resetDCtx(dctx); + return dctx; +} + +size_t ZSTDv01_freeDCtx(ZSTDv01_Dctx* dctx) +{ + free(dctx); + return 0; +} + +size_t ZSTDv01_nextSrcSizeToDecompress(ZSTDv01_Dctx* dctx) +{ + return ((dctx_t*)dctx)->expected; +} + +size_t ZSTDv01_decompressContinue(ZSTDv01_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize) +{ + dctx_t* ctx = (dctx_t*)dctx; + + /* Sanity check */ + if (srcSize != ctx->expected) return ERROR(srcSize_wrong); + if (dst != ctx->previousDstEnd) /* not contiguous */ + ctx->base = dst; + + /* Decompress : frame header */ + if (ctx->phase == 0) + { + /* Check frame magic header */ + U32 magicNumber = ZSTD_readBE32(src); + if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown); + ctx->phase = 1; + ctx->expected = ZSTD_blockHeaderSize; + return 0; + } + + /* Decompress : block header */ + if (ctx->phase == 1) + { + blockProperties_t bp; + size_t blockSize = ZSTDv01_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); + if (ZSTDv01_isError(blockSize)) return blockSize; + if (bp.blockType == bt_end) + { + ctx->expected = 0; + ctx->phase = 0; + } + else + { + ctx->expected = blockSize; + ctx->bType = bp.blockType; + ctx->phase = 2; + } + + return 0; + } + + /* Decompress : block content */ + { + size_t rSize; + switch(ctx->bType) + { + case bt_compressed: + rSize = ZSTD_decompressBlock(ctx, dst, maxDstSize, src, srcSize); + break; + case bt_raw : + rSize = ZSTD_copyUncompressedBlock(dst, maxDstSize, src, srcSize); + break; + case bt_rle : + return ERROR(GENERIC); /* not yet handled */ + break; + case bt_end : /* should never happen (filtered at phase 1) */ + rSize = 0; + break; + default: + return ERROR(GENERIC); + } + ctx->phase = 1; + ctx->expected = ZSTD_blockHeaderSize; + ctx->previousDstEnd = (void*)( ((char*)dst) + rSize); + return rSize; + } + +} diff --git a/src/borg/algorithms/zstd/lib/legacy/zstd_v01.h b/src/borg/algorithms/zstd/lib/legacy/zstd_v01.h new file mode 100644 index 0000000000..42f0897c7d --- /dev/null +++ b/src/borg/algorithms/zstd/lib/legacy/zstd_v01.h @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_V01_H_28739879432 +#define ZSTD_V01_H_28739879432 + +#if defined (__cplusplus) +extern "C" { +#endif + +/* ************************************* +* Includes +***************************************/ +#include /* size_t */ + + +/* ************************************* +* Simple one-step function +***************************************/ +/** +ZSTDv01_decompress() : decompress ZSTD frames compliant with v0.1.x format + compressedSize : is the exact source size + maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated. + It must be equal or larger than originalSize, otherwise decompression will fail. + return : the number of bytes decompressed into destination buffer (originalSize) + or an errorCode if it fails (which can be tested using ZSTDv01_isError()) +*/ +size_t ZSTDv01_decompress( void* dst, size_t maxOriginalSize, + const void* src, size_t compressedSize); + +/** +ZSTDv01_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.1.x format + compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' + return : the number of bytes that would be read to decompress this frame + or an errorCode if it fails (which can be tested using ZSTDv01_isError()) +*/ +size_t ZSTDv01_findFrameCompressedSize(const void* src, size_t compressedSize); + +/** +ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error +*/ +unsigned ZSTDv01_isError(size_t code); + + +/* ************************************* +* Advanced functions +***************************************/ +typedef struct ZSTDv01_Dctx_s ZSTDv01_Dctx; +ZSTDv01_Dctx* ZSTDv01_createDCtx(void); +size_t ZSTDv01_freeDCtx(ZSTDv01_Dctx* dctx); + +size_t ZSTDv01_decompressDCtx(void* ctx, + void* dst, size_t maxOriginalSize, + const void* src, size_t compressedSize); + +/* ************************************* +* Streaming functions +***************************************/ +size_t ZSTDv01_resetDCtx(ZSTDv01_Dctx* dctx); + +size_t ZSTDv01_nextSrcSizeToDecompress(ZSTDv01_Dctx* dctx); +size_t ZSTDv01_decompressContinue(ZSTDv01_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); +/** + Use above functions alternatively. + ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). + ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block. + Result is the number of bytes regenerated within 'dst'. + It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. +*/ + +/* ************************************* +* Prefix - version detection +***************************************/ +#define ZSTDv01_magicNumber 0xFD2FB51E /* Big Endian version */ +#define ZSTDv01_magicNumberLE 0x1EB52FFD /* Little Endian version */ + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_V01_H_28739879432 */ diff --git a/src/borg/algorithms/zstd/lib/legacy/zstd_v02.c b/src/borg/algorithms/zstd/lib/legacy/zstd_v02.c new file mode 100644 index 0000000000..b935a4d180 --- /dev/null +++ b/src/borg/algorithms/zstd/lib/legacy/zstd_v02.c @@ -0,0 +1,3556 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +#include /* size_t, ptrdiff_t */ +#include "zstd_v02.h" +#include "error_private.h" + + +/****************************************** +* Compiler-specific +******************************************/ +#if defined(_MSC_VER) /* Visual Studio */ +# include /* _byteswap_ulong */ +# include /* _byteswap_* */ +#endif + + +/* ****************************************************************** + mem.h + low-level memory access routines + Copyright (C) 2013-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ +#ifndef MEM_H_MODULE +#define MEM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + +/****************************************** +* Includes +******************************************/ +#include /* size_t, ptrdiff_t */ +#include /* memcpy */ + + +/****************************************** +* Compiler-specific +******************************************/ +#if defined(__GNUC__) +# define MEM_STATIC static __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define MEM_STATIC static inline +#elif defined(_MSC_VER) +# define MEM_STATIC static __inline +#else +# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + + +/**************************************************************** +* Basic Types +*****************************************************************/ +#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef int16_t S16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef int64_t S64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef signed short S16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; + typedef signed long long S64; +#endif + + +/**************************************************************** +* Memory I/O +*****************************************************************/ +/* MEM_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. + * Method 2 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets generating assembly depending on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1 > 2) + */ +#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +# define MEM_FORCE_MEMORY_ACCESS 2 +# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \ + (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) +# define MEM_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +MEM_STATIC unsigned MEM_32bits(void) { return sizeof(void*)==4; } +MEM_STATIC unsigned MEM_64bits(void) { return sizeof(void*)==8; } + +MEM_STATIC unsigned MEM_isLittleEndian(void) +{ + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +} + +#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) + +/* violates C standard on structure alignment. +Only use if no other choice to achieve best performance on target platform */ +MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; } +MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; } +MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } + +#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +typedef union { U16 u16; U32 u32; U64 u64; } __attribute__((packed)) unalign; + +MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } +MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } +MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } + +#else + +/* default method, safe and standard. + can sometimes prove slower */ + +MEM_STATIC U16 MEM_read16(const void* memPtr) +{ + U16 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U32 MEM_read32(const void* memPtr) +{ + U32 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U64 MEM_read64(const void* memPtr) +{ + U64 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +#endif // MEM_FORCE_MEMORY_ACCESS + + +MEM_STATIC U16 MEM_readLE16(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read16(memPtr); + else + { + const BYTE* p = (const BYTE*)memPtr; + return (U16)(p[0] + (p[1]<<8)); + } +} + +MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) +{ + if (MEM_isLittleEndian()) + { + MEM_write16(memPtr, val); + } + else + { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE)val; + p[1] = (BYTE)(val>>8); + } +} + +MEM_STATIC U32 MEM_readLE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read32(memPtr); + else + { + const BYTE* p = (const BYTE*)memPtr; + return (U32)((U32)p[0] + ((U32)p[1]<<8) + ((U32)p[2]<<16) + ((U32)p[3]<<24)); + } +} + + +MEM_STATIC U64 MEM_readLE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read64(memPtr); + else + { + const BYTE* p = (const BYTE*)memPtr; + return (U64)((U64)p[0] + ((U64)p[1]<<8) + ((U64)p[2]<<16) + ((U64)p[3]<<24) + + ((U64)p[4]<<32) + ((U64)p[5]<<40) + ((U64)p[6]<<48) + ((U64)p[7]<<56)); + } +} + + +MEM_STATIC size_t MEM_readLEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readLE32(memPtr); + else + return (size_t)MEM_readLE64(memPtr); +} + +#if defined (__cplusplus) +} +#endif + +#endif /* MEM_H_MODULE */ + + +/* ****************************************************************** + bitstream + Part of NewGen Entropy library + header file (to include) + Copyright (C) 2013-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ +#ifndef BITSTREAM_H_MODULE +#define BITSTREAM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* +* This API consists of small unitary functions, which highly benefit from being inlined. +* Since link-time-optimization is not available for all compilers, +* these functions are defined into a .h to be included. +*/ + + +/********************************************** +* bitStream decompression API (read backward) +**********************************************/ +typedef struct +{ + size_t bitContainer; + unsigned bitsConsumed; + const char* ptr; + const char* start; +} BIT_DStream_t; + +typedef enum { BIT_DStream_unfinished = 0, + BIT_DStream_endOfBuffer = 1, + BIT_DStream_completed = 2, + BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */ + /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */ + +MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize); +MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits); +MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD); +MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); + + +/* +* Start by invoking BIT_initDStream(). +* A chunk of the bitStream is then stored into a local register. +* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). +* You can then retrieve bitFields stored into the local register, **in reverse order**. +* Local register is manually filled from memory by the BIT_reloadDStream() method. +* A reload guarantee a minimum of ((8*sizeof(size_t))-7) bits when its result is BIT_DStream_unfinished. +* Otherwise, it can be less than that, so proceed accordingly. +* Checking if DStream has reached its end can be performed with BIT_endOfDStream() +*/ + + +/****************************************** +* unsafe API +******************************************/ +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits); +/* faster, but works only if nbBits >= 1 */ + + + +/**************************************************************** +* Helper functions +****************************************************************/ +MEM_STATIC unsigned BIT_highbit32 (register U32 val) +{ +# if defined(_MSC_VER) /* Visual */ + unsigned long r=0; + _BitScanReverse ( &r, val ); + return (unsigned) r; +# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ + return 31 - __builtin_clz (val); +# else /* Software version */ + static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; + U32 v = val; + unsigned r; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + r = DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27]; + return r; +# endif +} + + + +/********************************************************** +* bitStream decoding +**********************************************************/ + +/*!BIT_initDStream +* Initialize a BIT_DStream_t. +* @bitD : a pointer to an already allocated BIT_DStream_t structure +* @srcBuffer must point at the beginning of a bitStream +* @srcSize must be the exact size of the bitStream +* @result : size of stream (== srcSize) or an errorCode if a problem is detected +*/ +MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize) +{ + if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); } + + if (srcSize >= sizeof(size_t)) /* normal case */ + { + U32 contain32; + bitD->start = (const char*)srcBuffer; + bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(size_t); + bitD->bitContainer = MEM_readLEST(bitD->ptr); + contain32 = ((const BYTE*)srcBuffer)[srcSize-1]; + if (contain32 == 0) return ERROR(GENERIC); /* endMark not present */ + bitD->bitsConsumed = 8 - BIT_highbit32(contain32); + } + else + { + U32 contain32; + bitD->start = (const char*)srcBuffer; + bitD->ptr = bitD->start; + bitD->bitContainer = *(const BYTE*)(bitD->start); + switch(srcSize) + { + case 7: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[6]) << (sizeof(size_t)*8 - 16); + case 6: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[5]) << (sizeof(size_t)*8 - 24); + case 5: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[4]) << (sizeof(size_t)*8 - 32); + case 4: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[3]) << 24; + case 3: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[2]) << 16; + case 2: bitD->bitContainer += (size_t)(((const BYTE*)(bitD->start))[1]) << 8; + default:; + } + contain32 = ((const BYTE*)srcBuffer)[srcSize-1]; + if (contain32 == 0) return ERROR(GENERIC); /* endMark not present */ + bitD->bitsConsumed = 8 - BIT_highbit32(contain32); + bitD->bitsConsumed += (U32)(sizeof(size_t) - srcSize)*8; + } + + return srcSize; +} + +/*!BIT_lookBits + * Provides next n bits from local register + * local register is not modified (bits are still present for next read/look) + * On 32-bits, maxNbBits==25 + * On 64-bits, maxNbBits==57 + * @return : value extracted + */ +MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits) +{ + const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1; + return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask); +} + +/*! BIT_lookBitsFast : +* unsafe version; only works only if nbBits >= 1 */ +MEM_STATIC size_t BIT_lookBitsFast(BIT_DStream_t* bitD, U32 nbBits) +{ + const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1; + return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask); +} + +MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) +{ + bitD->bitsConsumed += nbBits; +} + +/*!BIT_readBits + * Read next n bits from local register. + * pay attention to not read more than nbBits contained into local register. + * @return : extracted value. + */ +MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits) +{ + size_t value = BIT_lookBits(bitD, nbBits); + BIT_skipBits(bitD, nbBits); + return value; +} + +/*!BIT_readBitsFast : +* unsafe version; only works only if nbBits >= 1 */ +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits) +{ + size_t value = BIT_lookBitsFast(bitD, nbBits); + BIT_skipBits(bitD, nbBits); + return value; +} + +MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) +{ + if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */ + return BIT_DStream_overflow; + + if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) + { + bitD->ptr -= bitD->bitsConsumed >> 3; + bitD->bitsConsumed &= 7; + bitD->bitContainer = MEM_readLEST(bitD->ptr); + return BIT_DStream_unfinished; + } + if (bitD->ptr == bitD->start) + { + if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer; + return BIT_DStream_completed; + } + { + U32 nbBytes = bitD->bitsConsumed >> 3; + BIT_DStream_status result = BIT_DStream_unfinished; + if (bitD->ptr - nbBytes < bitD->start) + { + nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */ + result = BIT_DStream_endOfBuffer; + } + bitD->ptr -= nbBytes; + bitD->bitsConsumed -= nbBytes*8; + bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */ + return result; + } +} + +/*! BIT_endOfDStream +* @return Tells if DStream has reached its exact end +*/ +MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) +{ + return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8)); +} + +#if defined (__cplusplus) +} +#endif + +#endif /* BITSTREAM_H_MODULE */ +/* ****************************************************************** + Error codes and messages + Copyright (C) 2013-2015, Yann Collet + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ +#ifndef ERROR_H_MODULE +#define ERROR_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + + +/****************************************** +* Compiler-specific +******************************************/ +#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define ERR_STATIC static inline +#elif defined(_MSC_VER) +# define ERR_STATIC static __inline +#elif defined(__GNUC__) +# define ERR_STATIC static __attribute__((unused)) +#else +# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + + +/****************************************** +* Error Management +******************************************/ +#define PREFIX(name) ZSTD_error_##name + +#define ERROR(name) (size_t)-PREFIX(name) + +#define ERROR_LIST(ITEM) \ + ITEM(PREFIX(No_Error)) ITEM(PREFIX(GENERIC)) \ + ITEM(PREFIX(dstSize_tooSmall)) ITEM(PREFIX(srcSize_wrong)) \ + ITEM(PREFIX(prefix_unknown)) ITEM(PREFIX(corruption_detected)) \ + ITEM(PREFIX(tableLog_tooLarge)) ITEM(PREFIX(maxSymbolValue_tooLarge)) ITEM(PREFIX(maxSymbolValue_tooSmall)) \ + ITEM(PREFIX(maxCode)) + +#define ERROR_GENERATE_ENUM(ENUM) ENUM, +typedef enum { ERROR_LIST(ERROR_GENERATE_ENUM) } ERR_codes; /* enum is exposed, to detect & handle specific errors; compare function result to -enum value */ + +#define ERROR_CONVERTTOSTRING(STRING) #STRING, +#define ERROR_GENERATE_STRING(EXPR) ERROR_CONVERTTOSTRING(EXPR) +static const char* ERR_strings[] = { ERROR_LIST(ERROR_GENERATE_STRING) }; + +ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } + +ERR_STATIC const char* ERR_getErrorName(size_t code) +{ + static const char* codeError = "Unspecified error code"; + if (ERR_isError(code)) return ERR_strings[-(int)(code)]; + return codeError; +} + + +#if defined (__cplusplus) +} +#endif + +#endif /* ERROR_H_MODULE */ +/* +Constructor and Destructor of type FSE_CTable + Note that its size depends on 'tableLog' and 'maxSymbolValue' */ +typedef unsigned FSE_CTable; /* don't allocate that. It's just a way to be more restrictive than void* */ +typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ + + +/* ****************************************************************** + FSE : Finite State Entropy coder + header file for static linking (only) + Copyright (C) 2013-2015, Yann Collet + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ +#if defined (__cplusplus) +extern "C" { +#endif + + +/****************************************** +* Static allocation +******************************************/ +/* FSE buffer bounds */ +#define FSE_NCOUNTBOUND 512 +#define FSE_BLOCKBOUND(size) (size + (size>>7)) +#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* You can statically allocate FSE CTable/DTable as a table of unsigned using below macro */ +#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2)) +#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<= BIT_DStream_completed + +When it's done, verify decompression is fully completed, by checking both DStream and the relevant states. +Checking if DStream has reached its end is performed by : + BIT_endOfDStream(&DStream); +Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible. + FSE_endOfDState(&DState); +*/ + + +/****************************************** +* FSE unsafe API +******************************************/ +static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD); +/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ + + +/****************************************** +* Implementation of inline functions +******************************************/ + +/* decompression */ + +typedef struct { + U16 tableLog; + U16 fastMode; +} FSE_DTableHeader; /* sizeof U32 */ + +typedef struct +{ + unsigned short newState; + unsigned char symbol; + unsigned char nbBits; +} FSE_decode_t; /* size == U32 */ + +MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt) +{ + FSE_DTableHeader DTableH; + memcpy(&DTableH, dt, sizeof(DTableH)); + DStatePtr->state = BIT_readBits(bitD, DTableH.tableLog); + BIT_reloadDStream(bitD); + DStatePtr->table = dt + 1; +} + +MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + const FSE_decode_t DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + const U32 nbBits = DInfo.nbBits; + BYTE symbol = DInfo.symbol; + size_t lowBits = BIT_readBits(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + const FSE_decode_t DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + const U32 nbBits = DInfo.nbBits; + BYTE symbol = DInfo.symbol; + size_t lowBits = BIT_readBitsFast(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) +{ + return DStatePtr->state == 0; +} + + +#if defined (__cplusplus) +} +#endif +/* ****************************************************************** + Huff0 : Huffman coder, part of New Generation Entropy library + header file for static linking (only) + Copyright (C) 2013-2015, Yann Collet + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + +#if defined (__cplusplus) +extern "C" { +#endif + +/****************************************** +* Static allocation macros +******************************************/ +/* Huff0 buffer bounds */ +#define HUF_CTABLEBOUND 129 +#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */ +#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* static allocation of Huff0's DTable */ +#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1< /* size_t */ + + +/* ************************************* +* Version +***************************************/ +#define ZSTD_VERSION_MAJOR 0 /* for breaking interface changes */ +#define ZSTD_VERSION_MINOR 2 /* for new (non-breaking) interface capabilities */ +#define ZSTD_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */ +#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) + + +/* ************************************* +* Advanced functions +***************************************/ +typedef struct ZSTD_CCtx_s ZSTD_CCtx; /* incomplete type */ + +#if defined (__cplusplus) +} +#endif +/* + zstd - standard compression library + Header File for static linking only + Copyright (C) 2014-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - zstd source repository : https://github.com/Cyan4973/zstd + - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + +/* The objects defined into this file should be considered experimental. + * They are not labelled stable, as their prototype may change in the future. + * You can use them for tests, provide feedback, or if you can endure risk of future changes. + */ + +#if defined (__cplusplus) +extern "C" { +#endif + +/* ************************************* +* Streaming functions +***************************************/ + +typedef struct ZSTD_DCtx_s ZSTD_DCtx; + +/* + Use above functions alternatively. + ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). + ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block. + Result is the number of bytes regenerated within 'dst'. + It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. +*/ + +/* ************************************* +* Prefix - version detection +***************************************/ +#define ZSTD_magicNumber 0xFD2FB522 /* v0.2 (current)*/ + + +#if defined (__cplusplus) +} +#endif +/* ****************************************************************** + FSE : Finite State Entropy coder + Copyright (C) 2013-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + +#ifndef FSE_COMMONDEFS_ONLY + +/**************************************************************** +* Tuning parameters +****************************************************************/ +/* MEMORY_USAGE : +* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) +* Increasing memory usage improves compression ratio +* Reduced memory usage can improve speed, due to cache effect +* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ +#define FSE_MAX_MEMORY_USAGE 14 +#define FSE_DEFAULT_MEMORY_USAGE 13 + +/* FSE_MAX_SYMBOL_VALUE : +* Maximum symbol value authorized. +* Required for proper stack allocation */ +#define FSE_MAX_SYMBOL_VALUE 255 + + +/**************************************************************** +* template functions type & suffix +****************************************************************/ +#define FSE_FUNCTION_TYPE BYTE +#define FSE_FUNCTION_EXTENSION + + +/**************************************************************** +* Byte symbol type +****************************************************************/ +#endif /* !FSE_COMMONDEFS_ONLY */ + + +/**************************************************************** +* Compiler specifics +****************************************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# include /* For Visual 2005 */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ +#else +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +# else +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +#endif + + +/**************************************************************** +* Includes +****************************************************************/ +#include /* malloc, free, qsort */ +#include /* memcpy, memset */ +#include /* printf (debug) */ + +/**************************************************************** +* Constants +*****************************************************************/ +#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2) +#define FSE_MAX_TABLESIZE (1U< FSE_TABLELOG_ABSOLUTE_MAX +#error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported" +#endif + + +/**************************************************************** +* Error Management +****************************************************************/ +#define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + + +/**************************************************************** +* Complex types +****************************************************************/ +typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; + + +/**************************************************************** +* Templates +****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ + +/* safety checks */ +#ifndef FSE_FUNCTION_EXTENSION +# error "FSE_FUNCTION_EXTENSION must be defined" +#endif +#ifndef FSE_FUNCTION_TYPE +# error "FSE_FUNCTION_TYPE must be defined" +#endif + +/* Function names */ +#define FSE_CAT(X,Y) X##Y +#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) +#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) + + +/* Function templates */ + +#define FSE_DECODE_TYPE FSE_decode_t + +static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3) + 3; } + +static size_t FSE_buildDTable +(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +{ + void* ptr = dt+1; + FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*)ptr; + FSE_DTableHeader DTableH; + const U32 tableSize = 1 << tableLog; + const U32 tableMask = tableSize-1; + const U32 step = FSE_tableStep(tableSize); + U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1]; + U32 position = 0; + U32 highThreshold = tableSize-1; + const S16 largeLimit= (S16)(1 << (tableLog-1)); + U32 noLarge = 1; + U32 s; + + /* Sanity Checks */ + if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); + + /* Init, lay down lowprob symbols */ + DTableH.tableLog = (U16)tableLog; + for (s=0; s<=maxSymbolValue; s++) + { + if (normalizedCounter[s]==-1) + { + tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s; + symbolNext[s] = 1; + } + else + { + if (normalizedCounter[s] >= largeLimit) noLarge=0; + symbolNext[s] = normalizedCounter[s]; + } + } + + /* Spread symbols */ + for (s=0; s<=maxSymbolValue; s++) + { + int i; + for (i=0; i highThreshold) position = (position + step) & tableMask; /* lowprob area */ + } + } + + if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ + + /* Build Decoding table */ + { + U32 i; + for (i=0; i FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); + bitStream >>= 4; + bitCount = 4; + *tableLogPtr = nbBits; + remaining = (1<1) && (charnum<=*maxSVPtr)) + { + if (previous0) + { + unsigned n0 = charnum; + while ((bitStream & 0xFFFF) == 0xFFFF) + { + n0+=24; + if (ip < iend-5) + { + ip+=2; + bitStream = MEM_readLE32(ip) >> bitCount; + } + else + { + bitStream >>= 16; + bitCount+=16; + } + } + while ((bitStream & 3) == 3) + { + n0+=3; + bitStream>>=2; + bitCount+=2; + } + n0 += bitStream & 3; + bitCount += 2; + if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall); + while (charnum < n0) normalizedCounter[charnum++] = 0; + if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) + { + ip += bitCount>>3; + bitCount &= 7; + bitStream = MEM_readLE32(ip) >> bitCount; + } + else + bitStream >>= 2; + } + { + const short max = (short)((2*threshold-1)-remaining); + short count; + + if ((bitStream & (threshold-1)) < (U32)max) + { + count = (short)(bitStream & (threshold-1)); + bitCount += nbBits-1; + } + else + { + count = (short)(bitStream & (2*threshold-1)); + if (count >= threshold) count -= max; + bitCount += nbBits; + } + + count--; /* extra accuracy */ + remaining -= FSE_abs(count); + normalizedCounter[charnum++] = count; + previous0 = !count; + while (remaining < threshold) + { + nbBits--; + threshold >>= 1; + } + + { + if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) + { + ip += bitCount>>3; + bitCount &= 7; + } + else + { + bitCount -= (int)(8 * (iend - 4 - ip)); + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> (bitCount & 31); + } + } + } + if (remaining != 1) return ERROR(GENERIC); + *maxSVPtr = charnum-1; + + ip += (bitCount+7)>>3; + if ((size_t)(ip-istart) > hbSize) return ERROR(srcSize_wrong); + return ip-istart; +} + + +/********************************************************* +* Decompression (Byte symbols) +*********************************************************/ +static size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue) +{ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + FSE_decode_t* const cell = (FSE_decode_t*)(ptr) + 1; /* because dt is unsigned */ + + DTableH->tableLog = 0; + DTableH->fastMode = 0; + + cell->newState = 0; + cell->symbol = symbolValue; + cell->nbBits = 0; + + return 0; +} + + +static size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits) +{ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + FSE_decode_t* const dinfo = (FSE_decode_t*)(ptr) + 1; /* because dt is unsigned */ + const unsigned tableSize = 1 << nbBits; + const unsigned tableMask = tableSize - 1; + const unsigned maxSymbolValue = tableMask; + unsigned s; + + /* Sanity checks */ + if (nbBits < 1) return ERROR(GENERIC); /* min size */ + + /* Build Decoding Table */ + DTableH->tableLog = (U16)nbBits; + DTableH->fastMode = 1; + for (s=0; s<=maxSymbolValue; s++) + { + dinfo[s].newState = 0; + dinfo[s].symbol = (BYTE)s; + dinfo[s].nbBits = (BYTE)nbBits; + } + + return 0; +} + +FORCE_INLINE size_t FSE_decompress_usingDTable_generic( + void* dst, size_t maxDstSize, + const void* cSrc, size_t cSrcSize, + const FSE_DTable* dt, const unsigned fast) +{ + BYTE* const ostart = (BYTE*) dst; + BYTE* op = ostart; + BYTE* const omax = op + maxDstSize; + BYTE* const olimit = omax-3; + + BIT_DStream_t bitD; + FSE_DState_t state1; + FSE_DState_t state2; + size_t errorCode; + + /* Init */ + errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); /* replaced last arg by maxCompressed Size */ + if (FSE_isError(errorCode)) return errorCode; + + FSE_initDState(&state1, &bitD, dt); + FSE_initDState(&state2, &bitD, dt); + +#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD) + + /* 4 symbols per loop */ + for ( ; (BIT_reloadDStream(&bitD)==BIT_DStream_unfinished) && (op sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[1] = FSE_GETSYMBOL(&state2); + + if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + { if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } } + + op[2] = FSE_GETSYMBOL(&state1); + + if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[3] = FSE_GETSYMBOL(&state2); + } + + /* tail */ + /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ + while (1) + { + if ( (BIT_reloadDStream(&bitD)>BIT_DStream_completed) || (op==omax) || (BIT_endOfDStream(&bitD) && (fast || FSE_endOfDState(&state1))) ) + break; + + *op++ = FSE_GETSYMBOL(&state1); + + if ( (BIT_reloadDStream(&bitD)>BIT_DStream_completed) || (op==omax) || (BIT_endOfDStream(&bitD) && (fast || FSE_endOfDState(&state2))) ) + break; + + *op++ = FSE_GETSYMBOL(&state2); + } + + /* end ? */ + if (BIT_endOfDStream(&bitD) && FSE_endOfDState(&state1) && FSE_endOfDState(&state2)) + return op-ostart; + + if (op==omax) return ERROR(dstSize_tooSmall); /* dst buffer is full, but cSrc unfinished */ + + return ERROR(corruption_detected); +} + + +static size_t FSE_decompress_usingDTable(void* dst, size_t originalSize, + const void* cSrc, size_t cSrcSize, + const FSE_DTable* dt) +{ + FSE_DTableHeader DTableH; + memcpy(&DTableH, dt, sizeof(DTableH)); + + /* select fast mode (static) */ + if (DTableH.fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); + return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); +} + + +static size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize) +{ + const BYTE* const istart = (const BYTE*)cSrc; + const BYTE* ip = istart; + short counting[FSE_MAX_SYMBOL_VALUE+1]; + DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */ + unsigned tableLog; + unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; + size_t errorCode; + + if (cSrcSize<2) return ERROR(srcSize_wrong); /* too small input size */ + + /* normal FSE decoding mode */ + errorCode = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize); + if (FSE_isError(errorCode)) return errorCode; + if (errorCode >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size */ + ip += errorCode; + cSrcSize -= errorCode; + + errorCode = FSE_buildDTable (dt, counting, maxSymbolValue, tableLog); + if (FSE_isError(errorCode)) return errorCode; + + /* always return, even if it is an error code */ + return FSE_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt); +} + + + +#endif /* FSE_COMMONDEFS_ONLY */ +/* ****************************************************************** + Huff0 : Huffman coder, part of New Generation Entropy library + Copyright (C) 2013-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE+Huff0 source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + +/**************************************************************** +* Compiler specifics +****************************************************************/ +#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +/* inline is defined */ +#elif defined(_MSC_VER) +# define inline __inline +#else +# define inline /* disable inline */ +#endif + + +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + + +/**************************************************************** +* Includes +****************************************************************/ +#include /* malloc, free, qsort */ +#include /* memcpy, memset */ +#include /* printf (debug) */ + +/**************************************************************** +* Error Management +****************************************************************/ +#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + + +/****************************************** +* Helper functions +******************************************/ +static unsigned HUF_isError(size_t code) { return ERR_isError(code); } + +#define HUF_ABSOLUTEMAX_TABLELOG 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ +#define HUF_MAX_TABLELOG 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */ +#define HUF_DEFAULT_TABLELOG HUF_MAX_TABLELOG /* tableLog by default, when not specified */ +#define HUF_MAX_SYMBOL_VALUE 255 +#if (HUF_MAX_TABLELOG > HUF_ABSOLUTEMAX_TABLELOG) +# error "HUF_MAX_TABLELOG is too large !" +#endif + + + +/********************************************************* +* Huff0 : Huffman block decompression +*********************************************************/ +typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX2; /* single-symbol decoding */ + +typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX4; /* double-symbols decoding */ + +typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t; + +/*! HUF_readStats + Read compact Huffman tree, saved by HUF_writeCTable + @huffWeight : destination buffer + @return : size read from `src` +*/ +static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize) +{ + U32 weightTotal; + U32 tableLog; + const BYTE* ip = (const BYTE*) src; + size_t iSize; + size_t oSize; + U32 n; + + if (!srcSize) return ERROR(srcSize_wrong); + iSize = ip[0]; + //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */ + + if (iSize >= 128) /* special header */ + { + if (iSize >= (242)) /* RLE */ + { + static int l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 }; + oSize = l[iSize-242]; + memset(huffWeight, 1, hwSize); + iSize = 0; + } + else /* Incompressible */ + { + oSize = iSize - 127; + iSize = ((oSize+1)/2); + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + if (oSize >= hwSize) return ERROR(corruption_detected); + ip += 1; + for (n=0; n> 4; + huffWeight[n+1] = ip[n/2] & 15; + } + } + } + else /* header compressed with FSE (normal case) */ + { + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + oSize = FSE_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */ + if (FSE_isError(oSize)) return oSize; + } + + /* collect weight stats */ + memset(rankStats, 0, (HUF_ABSOLUTEMAX_TABLELOG + 1) * sizeof(U32)); + weightTotal = 0; + for (n=0; n= HUF_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected); + rankStats[huffWeight[n]]++; + weightTotal += (1 << huffWeight[n]) >> 1; + } + if (weightTotal == 0) return ERROR(corruption_detected); + + /* get last non-null symbol weight (implied, total must be 2^n) */ + tableLog = BIT_highbit32(weightTotal) + 1; + if (tableLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected); + { + U32 total = 1 << tableLog; + U32 rest = total - weightTotal; + U32 verif = 1 << BIT_highbit32(rest); + U32 lastWeight = BIT_highbit32(rest) + 1; + if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ + huffWeight[oSize] = (BYTE)lastWeight; + rankStats[lastWeight]++; + } + + /* check tree construction validity */ + if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ + + /* results */ + *nbSymbolsPtr = (U32)(oSize+1); + *tableLogPtr = tableLog; + return iSize+1; +} + + +/**************************/ +/* single-symbol decoding */ +/**************************/ + +static size_t HUF_readDTableX2 (U16* DTable, const void* src, size_t srcSize) +{ + BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1]; + U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; /* large enough for values from 0 to 16 */ + U32 tableLog = 0; + const BYTE* ip = (const BYTE*) src; + size_t iSize = ip[0]; + U32 nbSymbols = 0; + U32 n; + U32 nextRankStart; + void* ptr = DTable+1; + HUF_DEltX2* const dt = (HUF_DEltX2*)ptr; + + HUF_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U16)); /* if compilation fails here, assertion is false */ + //memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats(huffWeight, HUF_MAX_SYMBOL_VALUE + 1, rankVal, &nbSymbols, &tableLog, src, srcSize); + if (HUF_isError(iSize)) return iSize; + + /* check result */ + if (tableLog > DTable[0]) return ERROR(tableLog_tooLarge); /* DTable is too small */ + DTable[0] = (U16)tableLog; /* maybe should separate sizeof DTable, as allocated, from used size of DTable, in case of DTable re-use */ + + /* Prepare ranks */ + nextRankStart = 0; + for (n=1; n<=tableLog; n++) + { + U32 current = nextRankStart; + nextRankStart += (rankVal[n] << (n-1)); + rankVal[n] = current; + } + + /* fill DTable */ + for (n=0; n> 1; + U32 i; + HUF_DEltX2 D; + D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w); + for (i = rankVal[w]; i < rankVal[w] + length; i++) + dt[i] = D; + rankVal[w] += length; + } + + return iSize; +} + +static BYTE HUF_decodeSymbolX2(BIT_DStream_t* Dstream, const HUF_DEltX2* dt, const U32 dtLog) +{ + const size_t val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ + const BYTE c = dt[val].byte; + BIT_skipBits(Dstream, dt[val].nbBits); + return c; +} + +#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ + *ptr++ = HUF_decodeSymbolX2(DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ + if (MEM_64bits() || (HUF_MAX_TABLELOG<=12)) \ + HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) + +#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ + if (MEM_64bits()) \ + HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) + +static inline size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX2* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 4 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-4)) + { + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_1(p, bitDPtr); + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + } + + /* closer to the end */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd)) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + + /* no more data to retrieve from bitstream, hence no need to reload */ + while (p < pEnd) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + + return pEnd-pStart; +} + + +static size_t HUF_decompress4X2_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const U16* DTable) +{ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + + { + const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + + const void* ptr = DTable; + const HUF_DEltX2* const dt = ((const HUF_DEltX2*)ptr) +1; + const U32 dtLog = DTable[0]; + size_t errorCode; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + const size_t length1 = MEM_readLE16(istart); + const size_t length2 = MEM_readLE16(istart+2); + const size_t length3 = MEM_readLE16(istart+4); + size_t length4; + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + const size_t segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + U32 endSignal; + + length4 = cSrcSize - (length1 + length2 + length3 + 6); + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + errorCode = BIT_initDStream(&bitD1, istart1, length1); + if (HUF_isError(errorCode)) return errorCode; + errorCode = BIT_initDStream(&bitD2, istart2, length2); + if (HUF_isError(errorCode)) return errorCode; + errorCode = BIT_initDStream(&bitD3, istart3, length3); + if (HUF_isError(errorCode)) return errorCode; + errorCode = BIT_initDStream(&bitD4, istart4, length4); + if (HUF_isError(errorCode)) return errorCode; + + /* 16-32 symbols per loop (4-8 symbols per stream) */ + endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + for ( ; (endSignal==BIT_DStream_unfinished) && (op4<(oend-7)) ; ) + { + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + + endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + } + + /* check corruption */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 supposed already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); + + /* check */ + endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endSignal) return ERROR(corruption_detected); + + /* decoded size */ + return dstSize; + } +} + + +static size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_MAX_TABLELOG); + const BYTE* ip = (const BYTE*) cSrc; + size_t errorCode; + + errorCode = HUF_readDTableX2 (DTable, cSrc, cSrcSize); + if (HUF_isError(errorCode)) return errorCode; + if (errorCode >= cSrcSize) return ERROR(srcSize_wrong); + ip += errorCode; + cSrcSize -= errorCode; + + return HUF_decompress4X2_usingDTable (dst, dstSize, ip, cSrcSize, DTable); +} + + +/***************************/ +/* double-symbols decoding */ +/***************************/ + +static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 consumed, + const U32* rankValOrigin, const int minWeight, + const sortedSymbol_t* sortedSymbols, const U32 sortedListSize, + U32 nbBitsBaseline, U16 baseSeq) +{ + HUF_DEltX4 DElt; + U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; + U32 s; + + /* get pre-calculated rankVal */ + memcpy(rankVal, rankValOrigin, sizeof(rankVal)); + + /* fill skipped values */ + if (minWeight>1) + { + U32 i, skipSize = rankVal[minWeight]; + MEM_writeLE16(&(DElt.sequence), baseSeq); + DElt.nbBits = (BYTE)(consumed); + DElt.length = 1; + for (i = 0; i < skipSize; i++) + DTable[i] = DElt; + } + + /* fill DTable */ + for (s=0; s= 1 */ + + rankVal[weight] += length; + } +} + +typedef U32 rankVal_t[HUF_ABSOLUTEMAX_TABLELOG][HUF_ABSOLUTEMAX_TABLELOG + 1]; + +static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog, + const sortedSymbol_t* sortedList, const U32 sortedListSize, + const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight, + const U32 nbBitsBaseline) +{ + U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; + const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ + const U32 minBits = nbBitsBaseline - maxWeight; + U32 s; + + memcpy(rankVal, rankValOrigin, sizeof(rankVal)); + + /* fill DTable */ + for (s=0; s= minBits) /* enough room for a second symbol */ + { + U32 sortedRank; + int minWeight = nbBits + scaleLog; + if (minWeight < 1) minWeight = 1; + sortedRank = rankStart[minWeight]; + HUF_fillDTableX4Level2(DTable+start, targetLog-nbBits, nbBits, + rankValOrigin[nbBits], minWeight, + sortedList+sortedRank, sortedListSize-sortedRank, + nbBitsBaseline, symbol); + } + else + { + U32 i; + const U32 end = start + length; + HUF_DEltX4 DElt; + + MEM_writeLE16(&(DElt.sequence), symbol); + DElt.nbBits = (BYTE)(nbBits); + DElt.length = 1; + for (i = start; i < end; i++) + DTable[i] = DElt; + } + rankVal[weight] += length; + } +} + +static size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize) +{ + BYTE weightList[HUF_MAX_SYMBOL_VALUE + 1]; + sortedSymbol_t sortedSymbol[HUF_MAX_SYMBOL_VALUE + 1]; + U32 rankStats[HUF_ABSOLUTEMAX_TABLELOG + 1] = { 0 }; + U32 rankStart0[HUF_ABSOLUTEMAX_TABLELOG + 2] = { 0 }; + U32* const rankStart = rankStart0+1; + rankVal_t rankVal; + U32 tableLog, maxW, sizeOfSort, nbSymbols; + const U32 memLog = DTable[0]; + const BYTE* ip = (const BYTE*) src; + size_t iSize = ip[0]; + void* ptr = DTable; + HUF_DEltX4* const dt = ((HUF_DEltX4*)ptr) + 1; + + HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */ + if (memLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge); + //memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats(weightList, HUF_MAX_SYMBOL_VALUE + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); + if (HUF_isError(iSize)) return iSize; + + /* check result */ + if (tableLog > memLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ + + /* find maxWeight */ + for (maxW = tableLog; rankStats[maxW]==0; maxW--) + {if (!maxW) return ERROR(GENERIC); } /* necessarily finds a solution before maxW==0 */ + + /* Get start index of each weight */ + { + U32 w, nextRankStart = 0; + for (w=1; w<=maxW; w++) + { + U32 current = nextRankStart; + nextRankStart += rankStats[w]; + rankStart[w] = current; + } + rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/ + sizeOfSort = nextRankStart; + } + + /* sort symbols by weight */ + { + U32 s; + for (s=0; s> consumed; + } + } + } + + HUF_fillDTableX4(dt, memLog, + sortedSymbol, sizeOfSort, + rankStart0, rankVal, maxW, + tableLog+1); + + return iSize; +} + + +static U32 HUF_decodeSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog) +{ + const size_t val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + memcpy(op, dt+val, 2); + BIT_skipBits(DStream, dt[val].nbBits); + return dt[val].length; +} + +static U32 HUF_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog) +{ + const size_t val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + memcpy(op, dt+val, 1); + if (dt[val].length==1) BIT_skipBits(DStream, dt[val].nbBits); + else + { + if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) + { + BIT_skipBits(DStream, dt[val].nbBits); + if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) + DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ + } + } + return 1; +} + + +#define HUF_DECODE_SYMBOLX4_0(ptr, DStreamPtr) \ + ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \ + if (MEM_64bits() || (HUF_MAX_TABLELOG<=12)) \ + ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX4_2(ptr, DStreamPtr) \ + if (MEM_64bits()) \ + ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) + +static inline size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const HUF_DEltX4* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 8 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd-7)) + { + HUF_DECODE_SYMBOLX4_2(p, bitDPtr); + HUF_DECODE_SYMBOLX4_1(p, bitDPtr); + HUF_DECODE_SYMBOLX4_2(p, bitDPtr); + HUF_DECODE_SYMBOLX4_0(p, bitDPtr); + } + + /* closer to the end */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-2)) + HUF_DECODE_SYMBOLX4_0(p, bitDPtr); + + while (p <= pEnd-2) + HUF_DECODE_SYMBOLX4_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ + + if (p < pEnd) + p += HUF_decodeLastSymbolX4(p, bitDPtr, dt, dtLog); + + return p-pStart; +} + + + +static size_t HUF_decompress4X4_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const U32* DTable) +{ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + + { + const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + + const void* ptr = DTable; + const HUF_DEltX4* const dt = ((const HUF_DEltX4*)ptr) +1; + const U32 dtLog = DTable[0]; + size_t errorCode; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + const size_t length1 = MEM_readLE16(istart); + const size_t length2 = MEM_readLE16(istart+2); + const size_t length3 = MEM_readLE16(istart+4); + size_t length4; + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + const size_t segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + U32 endSignal; + + length4 = cSrcSize - (length1 + length2 + length3 + 6); + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + errorCode = BIT_initDStream(&bitD1, istart1, length1); + if (HUF_isError(errorCode)) return errorCode; + errorCode = BIT_initDStream(&bitD2, istart2, length2); + if (HUF_isError(errorCode)) return errorCode; + errorCode = BIT_initDStream(&bitD3, istart3, length3); + if (HUF_isError(errorCode)) return errorCode; + errorCode = BIT_initDStream(&bitD4, istart4, length4); + if (HUF_isError(errorCode)) return errorCode; + + /* 16-32 symbols per loop (4-8 symbols per stream) */ + endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + for ( ; (endSignal==BIT_DStream_unfinished) && (op4<(oend-7)) ; ) + { + HUF_DECODE_SYMBOLX4_2(op1, &bitD1); + HUF_DECODE_SYMBOLX4_2(op2, &bitD2); + HUF_DECODE_SYMBOLX4_2(op3, &bitD3); + HUF_DECODE_SYMBOLX4_2(op4, &bitD4); + HUF_DECODE_SYMBOLX4_1(op1, &bitD1); + HUF_DECODE_SYMBOLX4_1(op2, &bitD2); + HUF_DECODE_SYMBOLX4_1(op3, &bitD3); + HUF_DECODE_SYMBOLX4_1(op4, &bitD4); + HUF_DECODE_SYMBOLX4_2(op1, &bitD1); + HUF_DECODE_SYMBOLX4_2(op2, &bitD2); + HUF_DECODE_SYMBOLX4_2(op3, &bitD3); + HUF_DECODE_SYMBOLX4_2(op4, &bitD4); + HUF_DECODE_SYMBOLX4_0(op1, &bitD1); + HUF_DECODE_SYMBOLX4_0(op2, &bitD2); + HUF_DECODE_SYMBOLX4_0(op3, &bitD3); + HUF_DECODE_SYMBOLX4_0(op4, &bitD4); + + endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + } + + /* check corruption */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 supposed already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX4(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX4(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX4(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX4(op4, &bitD4, oend, dt, dtLog); + + /* check */ + endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endSignal) return ERROR(corruption_detected); + + /* decoded size */ + return dstSize; + } +} + + +static size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_MAX_TABLELOG); + const BYTE* ip = (const BYTE*) cSrc; + + size_t hSize = HUF_readDTableX4 (DTable, cSrc, cSrcSize); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; + cSrcSize -= hSize; + + return HUF_decompress4X4_usingDTable (dst, dstSize, ip, cSrcSize, DTable); +} + + +/**********************************/ +/* quad-symbol decoding */ +/**********************************/ +typedef struct { BYTE nbBits; BYTE nbBytes; } HUF_DDescX6; +typedef union { BYTE byte[4]; U32 sequence; } HUF_DSeqX6; + +/* recursive, up to level 3; may benefit from