diff --git a/setup.py b/setup.py index 6032366c85..36e7d49df8 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,7 @@ import setup_lz4 import setup_zstd import setup_b2 +import setup_xxhash # True: use the shared liblz4 (>= 1.7.0 / r129) from the system, False: use the bundled lz4 code prefer_system_liblz4 = True @@ -21,6 +22,9 @@ # True: use the shared libb2 from the system, False: use the bundled blake2 code prefer_system_libb2 = True +# True: use the shared libxxhash (>= 0.6.5 [>= 0.7.2 on ARM]) from the system, False: use the bundled xxhash code +prefer_system_libxxhash = True + # prefer_system_msgpack is another option, but you need to set it in src/borg/helpers.py. min_python = (3, 5) @@ -213,6 +217,18 @@ def detect_openssl(prefixes): else: libzstd_system = False +possible_libxxhash_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libxxhash', '/usr/local/libxxhash', + '/usr/local/borg', '/opt/local', '/opt/pkg', ] +if os.environ.get('BORG_LIBXXHASH_PREFIX'): + possible_libxxhash_prefixes.insert(0, os.environ.get('BORG_LIBXXHASH_PREFIX')) +libxxhash_prefix = setup_xxhash.xxhash_system_prefix(possible_libxxhash_prefixes) +if prefer_system_libxxhash and libxxhash_prefix: + print('Detected and preferring libxxhash over bundled XXHASH') + define_macros.append(('BORG_USE_LIBXXHASH', 'YES')) + libxxhash_system = True +else: + libxxhash_system = False + with open('README.rst', 'r') as fd: long_description = fd.read() @@ -792,6 +808,10 @@ def run(self): system_prefix=libb2_prefix, system=libb2_system, **crypto_ext_kwargs) + crypto_ext_kwargs = setup_xxhash.xxhash_ext_kwargs(bundled_path='src/borg/algorithms/xxh64', + system_prefix=libxxhash_prefix, system=libxxhash_system, + **crypto_ext_kwargs) + msgpack_endian = '__BIG_ENDIAN__' if (sys.byteorder == 'big') else '__LITTLE_ENDIAN__' msgpack_macros = [(msgpack_endian, '1')] msgpack_packer_ext_kwargs = dict( diff --git a/setup_xxhash.py b/setup_xxhash.py new file mode 100644 index 0000000000..244e2323b1 --- /dev/null +++ b/setup_xxhash.py @@ -0,0 +1,73 @@ +# Support code for building a C extension with xxhash files +# +# Copyright (c) 2016-present, Gregory Szorc (original code for zstd) +# 2017-present, Thomas Waldmann (mods to make it more generic, code for blake2) +# 2020-present, Gianfranco Costamagna (code for xxhash) +# All rights reserved. +# +# This software may be modified and distributed under the terms +# of the BSD license. See the LICENSE file for details. + +import os + +# xxhash files, structure as seen in XXHASH (reference implementation) project repository: + +xxhash_sources = [ + 'xxhash.c', +] + +xxhash_includes = [ + '.', +] + + +def xxhash_system_prefix(prefixes): + for prefix in prefixes: + filename = os.path.join(prefix, 'include', 'xxhash.h') + if os.path.exists(filename): + with open(filename, 'rb') as fd: + if b'XXH64_digest' in fd.read(): + return prefix + + +def xxhash_ext_kwargs(bundled_path, system_prefix=None, system=False, **kwargs): + """amend kwargs with xxhash stuff for a distutils.extension.Extension initialization. + + bundled_path: relative (to this file) path to the bundled library source code files + system_prefix: where the system-installed library can be found + system: True: use the system-installed shared library, False: use the bundled library code + kwargs: distutils.extension.Extension kwargs that should be amended + returns: amended kwargs + """ + def multi_join(paths, *path_segments): + """apply os.path.join on a list of paths""" + return [os.path.join(*(path_segments + (path, ))) for path in paths] + + use_system = system and system_prefix is not None + + sources = kwargs.get('sources', []) + if not use_system: + sources += multi_join(xxhash_sources, bundled_path) + + include_dirs = kwargs.get('include_dirs', []) + if use_system: + include_dirs += multi_join(['include'], system_prefix) + else: + include_dirs += multi_join(xxhash_includes, bundled_path) + + library_dirs = kwargs.get('library_dirs', []) + if use_system: + library_dirs += multi_join(['lib'], system_prefix) + + libraries = kwargs.get('libraries', []) + if use_system: + libraries += ['xxhash', ] + + extra_compile_args = kwargs.get('extra_compile_args', []) + if not use_system: + extra_compile_args += [] # not used yet + + ret = dict(**kwargs) + ret.update(dict(sources=sources, extra_compile_args=extra_compile_args, + include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries)) + return ret diff --git a/src/borg/algorithms/checksums.pyx b/src/borg/algorithms/checksums.pyx index 75c67280b3..6d35c2209b 100644 --- a/src/borg/algorithms/checksums.pyx +++ b/src/borg/algorithms/checksums.pyx @@ -14,7 +14,7 @@ cdef extern from "crc32_dispatch.c": int _have_clmul "have_clmul"() -cdef extern from "xxh64/xxhash.c": +cdef extern from "../algorithms/xxhash-libselect.h": ctypedef struct XXH64_canonical_t: char digest[8] diff --git a/src/borg/algorithms/xxhash-libselect.h b/src/borg/algorithms/xxhash-libselect.h new file mode 100644 index 0000000000..365e677ffd --- /dev/null +++ b/src/borg/algorithms/xxhash-libselect.h @@ -0,0 +1,5 @@ +#ifdef BORG_USE_LIBXXHASH +#include +#else +#include "xxh64/xxhash.c" +#endif