From a05a80f3a993a726c53efe513c11480c594572be Mon Sep 17 00:00:00 2001 From: Jack Lloyd Date: Sat, 6 Sep 2025 10:57:18 -0400 Subject: [PATCH 1/2] Format Python code using ruff --- configure.py | 3462 ++++++++++------- src/configs/sphinx/conf.py | 121 +- src/ct_selftest/ct_selftest.py | 51 +- src/editors/vscode/scripts/bogo.py | 25 +- src/editors/vscode/scripts/common.py | 18 +- src/editors/vscode/scripts/test.py | 20 +- src/python/botan3.py | 1698 +++++--- src/scripts/bench.py | 469 ++- src/scripts/build_docs.py | 135 +- src/scripts/check.py | 39 +- src/scripts/ci/ci_tlsanvil_check.py | 62 +- src/scripts/ci/ci_tlsanvil_test.py | 108 +- src/scripts/ci/gh_clang_tidy_fixes_in_pr.py | 55 +- src/scripts/ci/gha_linux_packages.py | 206 +- src/scripts/ci_build.py | 1559 +++++--- src/scripts/ci_check_headers.py | 43 +- src/scripts/ci_check_install.py | 106 +- src/scripts/ci_report_sizes.py | 18 +- src/scripts/cleanup.py | 76 +- src/scripts/compare_perf.py | 78 +- src/scripts/config_for_oss_fuzz.py | 6 +- src/scripts/create_corpus_zip.py | 12 +- src/scripts/dev_tools/addchain.py | 49 +- .../dev_tools/analyze_timing_results.py | 51 +- src/scripts/dev_tools/file_size_check.py | 17 +- .../dev_tools/format_wycheproof_ecdsa.py | 51 +- src/scripts/dev_tools/gen_dilithium_kat.py | 93 +- src/scripts/dev_tools/gen_ec_groups.py | 140 +- src/scripts/dev_tools/gen_ffi_decls.py | 119 +- src/scripts/dev_tools/gen_frodo_kat.py | 47 +- src/scripts/dev_tools/gen_kyber_kat.py | 100 +- src/scripts/dev_tools/gen_mlkem_acvp_kat.py | 118 +- src/scripts/dev_tools/gen_mp_comba.py | 48 +- src/scripts/dev_tools/gen_mp_monty.py | 35 +- src/scripts/dev_tools/gen_oids.py | 61 +- src/scripts/dev_tools/gen_os_features.py | 41 +- src/scripts/dev_tools/gen_pqc_dsa_kats.py | 144 +- src/scripts/dev_tools/gen_sphincsplus_kat.py | 29 +- src/scripts/dev_tools/gen_tls_suite_info.py | 386 +- src/scripts/dev_tools/macro_checks.py | 25 +- src/scripts/dev_tools/run_clang_format.py | 114 +- src/scripts/dev_tools/run_clang_tidy.py | 334 +- src/scripts/dev_tools/show_dependencies.py | 113 +- .../xmss_test_vector_from_reference.py | 58 +- src/scripts/dist.py | 386 +- src/scripts/gdb/strubtest.py | 36 +- src/scripts/install.py | 283 +- src/scripts/python_unittests.py | 202 +- src/scripts/python_unittests_unix.py | 8 +- src/scripts/repo_config.py | 52 +- src/scripts/rewrite_lcov.py | 12 +- src/scripts/run_limbo_tests.py | 283 +- src/scripts/run_tests_under_valgrind.py | 107 +- src/scripts/run_tls_attacker.py | 170 +- src/scripts/run_tls_fuzzer.py | 59 +- src/scripts/test_all_configs.py | 95 +- src/scripts/test_cli.py | 1782 ++++++--- src/scripts/test_cli_crypt.py | 96 +- src/scripts/test_fuzzers.py | 130 +- src/scripts/test_python.py | 614 +-- src/scripts/test_strubbed_symbols.py | 107 +- src/scripts/tls_scanner/tls_scanner.py | 21 +- src/scripts/website.py | 124 +- 63 files changed, 9646 insertions(+), 5461 deletions(-) diff --git a/configure.py b/configure.py index 799e36dbca7..8e6325f16d5 100755 --- a/configure.py +++ b/configure.py @@ -28,7 +28,8 @@ import logging import time import errno -import optparse # pylint: disable=deprecated-module +import optparse # pylint: disable=deprecated-module + # An error caused by and to be fixed by the user, e.g. invalid command line argument class UserError(Exception): @@ -44,53 +45,60 @@ class InternalError(Exception): def flatten(lst): return sum(lst, []) + def normalize_source_path(source): """ cmake and some versions of make need this, and nothing else minds """ - return os.path.normpath(source).replace('\\', '/') + return os.path.normpath(source).replace("\\", "/") + def normalize_source_paths(sources): return [normalize_source_path(p) for p in sources] + def parse_version_file(version_path): - version_file = open(version_path, encoding='utf8') + version_file = open(version_path, encoding="utf8") key_and_val = re.compile(r"([a-z_]+) = ([a-zA-Z0-9:\-\']+)") results = {} for line in version_file.readlines(): - if not line or line[0] == '#': + if not line or line[0] == "#": continue match = key_and_val.match(line) if match: key = match.group(1) val = match.group(2) - if val == 'None': + if val == "None": val = None elif val.startswith("'") and val.endswith("'"): - val = val[1:len(val)-1] + val = val[1 : len(val) - 1] else: val = int(val) results[key] = val return results + class Version: """ Version information are all static members """ + data = {} @staticmethod def get_data(): if not Version.data: root_dir = os.path.dirname(os.path.realpath(__file__)) - Version.data = parse_version_file(os.path.join(root_dir, 'src/build-data/version.txt')) + Version.data = parse_version_file( + os.path.join(root_dir, "src/build-data/version.txt") + ) suffix = Version.data["release_suffix"] if suffix != "": - suffix_re = re.compile('-(alpha|beta|rc)[0-9]+') + suffix_re = re.compile("-(alpha|beta|rc)[0-9]+") if not suffix_re.match(suffix): raise Exception("Unexpected version suffix '%s'" % (suffix)) @@ -114,7 +122,7 @@ def suffix(): @staticmethod def packed(): - # Used on macOS for dylib versioning + # Used on macOS for dylib versioning return Version.major() * 1000 + Version.minor() @staticmethod @@ -131,7 +139,12 @@ def datestamp(): @staticmethod def short_version_string(): - return "%d.%d.%d%s" % (Version.major(), Version.minor(), Version.patch(), Version.suffix()) + return "%d.%d.%d%s" % ( + Version.major(), + Version.minor(), + Version.patch(), + Version.suffix(), + ) @staticmethod def full_version_string(options): @@ -161,10 +174,14 @@ def full_version_string(options): return version - @staticmethod def as_string(): - return '%d.%d.%d%s' % (Version.major(), Version.minor(), Version.patch(), Version.suffix()) + return "%d.%d.%d%s" % ( + Version.major(), + Version.minor(), + Version.patch(), + Version.suffix(), + ) @staticmethod def vc_rev(): @@ -175,7 +192,7 @@ def vc_rev(): @staticmethod def _local_repo_vc_revision(): - vc_command = ['git', 'rev-parse', 'HEAD'] + vc_command = ["git", "rev-parse", "HEAD"] cmdname = vc_command[0] try: @@ -183,22 +200,26 @@ def _local_repo_vc_revision(): vc_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True) + universal_newlines=True, + ) (stdout, stderr) = vc.communicate() if vc.returncode != 0: - logging.debug('Error getting rev from %s - %d (%s)', - cmdname, vc.returncode, stderr) - return 'unknown' + logging.debug( + "Error getting rev from %s - %d (%s)", + cmdname, + vc.returncode, + stderr, + ) + return "unknown" rev = str(stdout).strip() - logging.debug('%s reported revision %s', cmdname, rev) + logging.debug("%s reported revision %s", cmdname, rev) - return '%s:%s' % (cmdname, rev) + return "%s:%s" % (cmdname, rev) except OSError as ex: - logging.debug('Error getting rev from %s - %s', cmdname, ex.strerror) - return 'unknown' - + logging.debug("Error getting rev from %s - %s", cmdname, ex.strerror) + return "unknown" class SourcePaths: @@ -210,86 +231,107 @@ class SourcePaths: def __init__(self, base_dir): self.base_dir = base_dir - self.doc_dir = os.path.join(self.base_dir, 'doc') - self.src_dir = os.path.join(self.base_dir, 'src') + self.doc_dir = os.path.join(self.base_dir, "doc") + self.src_dir = os.path.join(self.base_dir, "src") # dirs in src/ - self.build_data_dir = os.path.join(self.src_dir, 'build-data') - self.configs_dir = os.path.join(self.src_dir, 'configs') - self.lib_dir = os.path.join(self.src_dir, 'lib') - self.python_dir = os.path.join(self.src_dir, 'python') - self.scripts_dir = os.path.join(self.src_dir, 'scripts') + self.build_data_dir = os.path.join(self.src_dir, "build-data") + self.configs_dir = os.path.join(self.src_dir, "configs") + self.lib_dir = os.path.join(self.src_dir, "lib") + self.python_dir = os.path.join(self.src_dir, "python") + self.scripts_dir = os.path.join(self.src_dir, "scripts") # subdirs of src/ - self.test_data_dir = os.path.join(self.src_dir, 'tests/data') - self.sphinx_config_dir = os.path.join(self.configs_dir, 'sphinx') + self.test_data_dir = os.path.join(self.src_dir, "tests/data") + self.sphinx_config_dir = os.path.join(self.configs_dir, "sphinx") class BuildPaths: """ Constructor """ - def __init__(self, source_paths, options, modules): - self.build_dir = os.path.join(options.with_build_dir, 'build') - self.libobj_dir = os.path.join(self.build_dir, 'obj', 'lib') - self.cliobj_dir = os.path.join(self.build_dir, 'obj', 'cli') - self.testobj_dir = os.path.join(self.build_dir, 'obj', 'test') - - self.doc_output_dir = os.path.join(self.build_dir, 'docs') - self.handbook_output_dir = os.path.join(self.doc_output_dir, 'handbook') - self.doc_output_dir_doxygen = os.path.join(self.doc_output_dir, 'doxygen') if options.with_doxygen else None - self.doc_module_info = os.path.join(self.build_dir, 'module_info') if options.with_doxygen else None - self.response_file_dir = os.path.join(self.build_dir, 'response_files') + def __init__(self, source_paths, options, modules): + self.build_dir = os.path.join(options.with_build_dir, "build") + + self.libobj_dir = os.path.join(self.build_dir, "obj", "lib") + self.cliobj_dir = os.path.join(self.build_dir, "obj", "cli") + self.testobj_dir = os.path.join(self.build_dir, "obj", "test") + + self.doc_output_dir = os.path.join(self.build_dir, "docs") + self.handbook_output_dir = os.path.join(self.doc_output_dir, "handbook") + self.doc_output_dir_doxygen = ( + os.path.join(self.doc_output_dir, "doxygen") + if options.with_doxygen + else None + ) + self.doc_module_info = ( + os.path.join(self.build_dir, "module_info") + if options.with_doxygen + else None + ) + self.response_file_dir = os.path.join(self.build_dir, "response_files") # We split the header include paths into 'public', 'internal' and 'external' # to allow for better control over what is exposed to each compilation unit. # For instance, the examples should only see the public headers, while the # test suite should see both public and internal headers. - self.include_dir = os.path.join(self.build_dir, 'include') - self.public_include_basedir = os.path.join(self.include_dir, 'public') - self.internal_include_basedir = os.path.join(self.include_dir, 'internal') - self.external_include_dir = os.path.join(self.include_dir, 'external') - self.public_include_dir = os.path.join(self.public_include_basedir, 'botan') - self.internal_include_dir = os.path.join(self.internal_include_basedir, 'botan', 'internal') + self.include_dir = os.path.join(self.build_dir, "include") + self.public_include_basedir = os.path.join(self.include_dir, "public") + self.internal_include_basedir = os.path.join(self.include_dir, "internal") + self.external_include_dir = os.path.join(self.include_dir, "external") + self.public_include_dir = os.path.join(self.public_include_basedir, "botan") + self.internal_include_dir = os.path.join( + self.internal_include_basedir, "botan", "internal" + ) self.internal_headers = sorted(flatten([m.internal_headers() for m in modules])) self.external_headers = sorted(flatten([m.external_headers() for m in modules])) # this is overwritten if amalgamation is used - self.lib_sources = normalize_source_paths(sorted(flatten([mod.sources() for mod in modules]))) + self.lib_sources = normalize_source_paths( + sorted(flatten([mod.sources() for mod in modules])) + ) self.public_headers = sorted(flatten([m.public_headers() for m in modules])) def find_sources_in(basedir, srcdir): - for (dirpath, _, filenames) in os.walk(os.path.join(basedir, srcdir)): + for dirpath, _, filenames in os.walk(os.path.join(basedir, srcdir)): for filename in filenames: - if filename.endswith('.cpp') and not filename.startswith('.'): + if filename.endswith(".cpp") and not filename.startswith("."): yield os.path.join(dirpath, filename) def find_headers_in(basedir, srcdir): - for (dirpath, _, filenames) in os.walk(os.path.join(basedir, srcdir)): + for dirpath, _, filenames in os.walk(os.path.join(basedir, srcdir)): for filename in filenames: - if filename.endswith('.h') and not filename.startswith('.'): + if filename.endswith(".h") and not filename.startswith("."): yield os.path.join(dirpath, filename) - self.cli_sources = normalize_source_paths(find_sources_in(source_paths.src_dir, 'cli')) - self.cli_headers = normalize_source_paths(find_headers_in(source_paths.src_dir, 'cli')) - self.test_sources = normalize_source_paths(find_sources_in(source_paths.src_dir, 'tests')) - - if 'examples' in options.build_targets: - self.example_sources = normalize_source_paths(find_sources_in(source_paths.src_dir, 'examples')) - self.example_output_dir = os.path.join(self.build_dir, 'examples') - self.example_obj_dir = os.path.join(self.build_dir, 'obj', 'examples') + self.cli_sources = normalize_source_paths( + find_sources_in(source_paths.src_dir, "cli") + ) + self.cli_headers = normalize_source_paths( + find_headers_in(source_paths.src_dir, "cli") + ) + self.test_sources = normalize_source_paths( + find_sources_in(source_paths.src_dir, "tests") + ) + + if "examples" in options.build_targets: + self.example_sources = normalize_source_paths( + find_sources_in(source_paths.src_dir, "examples") + ) + self.example_output_dir = os.path.join(self.build_dir, "examples") + self.example_obj_dir = os.path.join(self.build_dir, "obj", "examples") else: self.example_sources = None self.example_output_dir = None self.example_obj_dir = None if options.build_fuzzers: - self.fuzzer_sources = list(find_sources_in(source_paths.src_dir, 'fuzzer')) - self.fuzzer_output_dir = os.path.join(self.build_dir, 'fuzzer') - self.fuzzobj_dir = os.path.join(self.build_dir, 'obj', 'fuzzer') + self.fuzzer_sources = list(find_sources_in(source_paths.src_dir, "fuzzer")) + self.fuzzer_output_dir = os.path.join(self.build_dir, "fuzzer") + self.fuzzobj_dir = os.path.join(self.build_dir, "obj", "fuzzer") else: self.fuzzer_sources = None self.fuzzer_output_dir = None @@ -304,7 +346,7 @@ def build_dirs(self): self.internal_include_dir, self.external_include_dir, self.handbook_output_dir, - self.response_file_dir + self.response_file_dir, ] if self.doc_output_dir_doxygen: out += [self.doc_output_dir_doxygen, self.doc_module_info] @@ -315,34 +357,57 @@ def build_dirs(self): return out def format_public_include_flags(self, cc): - return cc.add_include_dir_option + ' ' + normalize_source_path(self.public_include_basedir) + return ( + cc.add_include_dir_option + + " " + + normalize_source_path(self.public_include_basedir) + ) def format_internal_include_flags(self, cc): - return cc.add_include_dir_option + ' ' + normalize_source_path(self.internal_include_basedir) + return ( + cc.add_include_dir_option + + " " + + normalize_source_path(self.internal_include_basedir) + ) def format_external_include_flags(self, cc, external_includes): dash_isystem = cc.add_system_include_dir_option - output = '' + output = "" if self.external_headers: - output += ' ' + dash_isystem + ' ' + normalize_source_path(self.external_include_dir) + output += ( + " " + + dash_isystem + + " " + + normalize_source_path(self.external_include_dir) + ) for external_include in external_includes: - output += ' ' + dash_isystem + ' ' + normalize_source_path(external_include) + output += " " + dash_isystem + " " + normalize_source_path(external_include) return output def src_info(self, typ): - if typ == 'lib': + if typ == "lib": return (self.lib_sources, self.libobj_dir) - if typ == 'cli': + if typ == "cli": return (self.cli_sources, self.cliobj_dir) - if typ == 'test': + if typ == "test": return (self.test_sources, self.testobj_dir) - if typ == 'fuzzer': + if typ == "fuzzer": return (self.fuzzer_sources, self.fuzzobj_dir) - if typ == 'examples': + if typ == "examples": return (self.example_sources, self.example_obj_dir) raise InternalError("Unknown src info type '%s'" % (typ)) -ACCEPTABLE_BUILD_TARGETS = ["static", "shared", "cli", "tests", "bogo_shim", "examples", "ct_selftest"] + +ACCEPTABLE_BUILD_TARGETS = [ + "static", + "shared", + "cli", + "tests", + "bogo_shim", + "examples", + "ct_selftest", +] + def process_command_line(args): """ @@ -351,310 +416,555 @@ def process_command_line(args): available before logging is setup. """ - def define_option_pair(group, verb, nverb, what, default, msg=optparse.SUPPRESS_HELP): - dest = '%s_%s' % (verb, what.replace('-', '_')) + def define_option_pair( + group, verb, nverb, what, default, msg=optparse.SUPPRESS_HELP + ): + dest = "%s_%s" % (verb, what.replace("-", "_")) # always show the help for the option that switches away from the default with_help = msg if not default else optparse.SUPPRESS_HELP without_help = msg if default else optparse.SUPPRESS_HELP - group.add_option('--%s-%s' % (verb, what), - dest=dest, - action='store_true', - default=default, - help=with_help) - - group.add_option('--%s-%s' % (nverb, what), - dest=dest, - action='store_false', - help=without_help) + group.add_option( + "--%s-%s" % (verb, what), + dest=dest, + action="store_true", + default=default, + help=with_help, + ) + + group.add_option( + "--%s-%s" % (nverb, what), + dest=dest, + action="store_false", + help=without_help, + ) def add_with_without_pair(group, what, default, msg=optparse.SUPPRESS_HELP): - define_option_pair(group, 'with', 'without', what, default, msg) + define_option_pair(group, "with", "without", what, default, msg) def add_enable_disable_pair(group, what, default, msg=optparse.SUPPRESS_HELP): - define_option_pair(group, 'enable', 'disable', what, default, msg) + define_option_pair(group, "enable", "disable", what, default, msg) parser = optparse.OptionParser( formatter=optparse.IndentedHelpFormatter(max_help_position=50), - version=Version.as_string()) - - parser.add_option('--verbose', action='store_true', default=False, - help='Show debug messages') - parser.add_option('--quiet', action='store_true', default=False, - help='Show only warnings and errors') - - target_group = optparse.OptionGroup(parser, 'Target options') - - target_group.add_option('--cpu', help='set the target CPU architecture') - - target_group.add_option('--os', help='set the target operating system') - - target_group.add_option('--cc', dest='compiler', help='set the desired build compiler') - - target_group.add_option('--cc-min-version', dest='cc_min_version', default=None, - metavar='MAJOR.MINOR', - help='Set the minimal version of the target compiler. ' \ - 'Use --cc-min-version=0.0 to support all compiler versions. ' \ - 'Default is auto detection.') - - target_group.add_option('--cc-bin', dest='compiler_binary', metavar='BINARY', - help='set path to compiler binary') - - target_group.add_option('--cc-abi-flags', metavar='FLAGS', default='', - help='set compiler ABI flags') - - target_group.add_option('--cxxflags', metavar='FLAGS', default=None, - help='override all compiler flags') - - target_group.add_option('--extra-cxxflags', metavar='FLAGS', default=[], action='append', - help='set extra compiler flags') - - target_group.add_option('--lto-cxxflags-to-ldflags', default=False, action='store_true', - help='set all compilation flags also during linking (for LTO)') - - target_group.add_option('--ldflags', metavar='FLAGS', - help='set linker flags', default=None) - - target_group.add_option('--extra-libs', metavar='LIBS', - help='specify extra libraries to link against', default='') - - target_group.add_option('--ar-command', dest='ar_command', metavar='AR', default=None, - help='set path to static archive creator') - - target_group.add_option('--ar-options', dest='ar_options', metavar='AR_OPTIONS', default=None, - help='set options for ar') - - target_group.add_option('--msvc-runtime', metavar='RT', default=None, - help='specify MSVC runtime (MT, MD, MTd, MDd)') - - target_group.add_option('--compiler-cache', - help='specify a compiler cache to use') - - target_group.add_option('--ct-value-barrier-type', metavar='TYPE', default=None, - help=optparse.SUPPRESS_HELP) - - target_group.add_option('--with-os-features', action='append', metavar='FEAT', - help='specify OS features to use') - target_group.add_option('--without-os-features', action='append', metavar='FEAT', - help='specify OS features to disable') - - add_with_without_pair(target_group, 'compilation-database', True, 'disable compile_commands.json') - - isa_extensions_that_can_be_disabled = [('NEON', 'arm32')] - - for (isa_extn_name,arch) in isa_extensions_that_can_be_disabled: - isa_extn = isa_extn_name.lower().replace(' ', '') - - nm = isa_extn.replace('-', '').replace('.', '').replace(' ', '') - - target_group.add_option('--disable-%s' % (isa_extn), - help='disable %s intrinsics' % (isa_extn_name), - action='append_const', - const=(nm,arch), - dest='disable_intrinsics') - - build_group = optparse.OptionGroup(parser, 'Build options') - - build_group.add_option('--system-cert-bundle', metavar='PATH', default=None, - help='set path to trusted CA bundle') - - add_with_without_pair(build_group, 'debug-info', False, 'include debug symbols') - - add_with_without_pair(build_group, 'sanitizers', False, 'enable ASan/UBSan checks') - - add_enable_disable_pair(build_group, 'asm', True, 'disable inline assembly') - - add_enable_disable_pair(build_group, 'stack-scrubbing', False, 'enable compiler-assisted stack scrubbing') - - build_group.add_option('--enable-sanitizers', metavar='SAN', default='', - help='enable specific sanitizers') - - add_with_without_pair(build_group, 'stack-protector', None, 'disable stack smashing protections') - - add_with_without_pair(build_group, 'coverage-info', False, 'add coverage info') - - build_group.add_option('--enable-shared-library', dest='build_shared_lib', - action='store_true', default=None, - help=optparse.SUPPRESS_HELP) - build_group.add_option('--disable-shared-library', dest='build_shared_lib', - action='store_false', - help='disable building shared library') - - build_group.add_option('--enable-static-library', dest='build_static_lib', - action='store_true', default=None, - help=optparse.SUPPRESS_HELP) - build_group.add_option('--disable-static-library', dest='build_static_lib', - action='store_false', - help='disable building static library') - - build_group.add_option('--optimize-for-size', dest='optimize_for_size', - action='store_true', default=False, - help='optimize for code size') - - build_group.add_option('--no-optimizations', dest='no_optimizations', - action='store_true', default=False, - help='disable all optimizations (for debugging)') - - build_group.add_option('--debug-mode', action='store_true', default=False, dest='debug_mode', - help='enable debug info, disable optimizations') - - build_group.add_option('--amalgamation', dest='amalgamation', - default=False, action='store_true', - help='use amalgamation to build') - - build_group.add_option('--name-amalgamation', metavar='NAME', default='botan_all', - help='specify alternate name for amalgamation files') - - build_group.add_option('--with-build-dir', metavar='DIR', default='', - help='setup the build in DIR') - - build_group.add_option('--with-external-includedir', metavar='DIR', default=[], - help='use DIR for external includes', action='append') - - build_group.add_option('--with-external-libdir', metavar='DIR', default=[], - help='use DIR for external libs', action='append') - - build_group.add_option('--define-build-macro', metavar='DEFINE', default=[], - help='set compile-time pre-processor definition like KEY[=VALUE]', action='append') - - build_group.add_option('--with-sysroot-dir', metavar='DIR', default='', - help='use DIR for system root while cross-compiling') - - link_methods = ['symlink', 'hardlink', 'copy'] - build_group.add_option('--link-method', default=None, metavar='METHOD', - choices=link_methods, - help='choose how links to include headers are created (%s)' % ', '.join(link_methods)) - - build_group.add_option('--distribution-info', metavar='STRING', - help='distribution specific version', default=None) - - build_group.add_option('--maintainer-mode', dest='maintainer_mode', - action='store_true', default=False, - help=optparse.SUPPRESS_HELP) - - build_group.add_option('--werror-mode', dest='werror_mode', - action='store_true', default=False, - help="Prohibit compiler warnings") - - build_group.add_option('--no-install-python-module', action='store_true', default=False, - help='skip installing Python module') - - build_group.add_option('--with-python-versions', dest='python_version', - metavar='N.M', - default='%d.%d' % (sys.version_info[0], sys.version_info[1]), - help='where to install botan2.py (def %default)') - - build_group.add_option('--disable-cc-tests', dest='enable_cc_tests', - default=True, action='store_false', - help=optparse.SUPPRESS_HELP) - - add_with_without_pair(build_group, 'valgrind', False, 'use valgrind API') - - build_group.add_option('--unsafe-fuzzer-mode', action='store_true', default=False, - help=optparse.SUPPRESS_HELP) - - build_group.add_option('--build-fuzzers', dest='build_fuzzers', - metavar='TYPE', default=None, - help='Build fuzzers (afl, libfuzzer, klee, test)') - - build_group.add_option('--with-fuzzer-lib', metavar='LIB', default=None, dest='fuzzer_lib', - help='additionally link in LIB') - - add_with_without_pair(build_group, 'debug-asserts', default=False) - - build_group.add_option('--unsafe-terminate-on-asserts', action='store_true', default=False, - help=optparse.SUPPRESS_HELP) - - build_group.add_option('--build-targets', default=None, dest="build_targets", action='append', - help="build specific targets and tools (%s)" % ', '.join(ACCEPTABLE_BUILD_TARGETS)) - - build_group.add_option('--build-tool', default='make', - help="specify the build tool (make, ninja)") - - add_with_without_pair(build_group, 'pkg-config', default=None) - add_with_without_pair(build_group, 'cmake-config', default=True) - - docs_group = optparse.OptionGroup(parser, 'Documentation Options') - - add_with_without_pair(docs_group, 'documentation', True, 'skip building/installing documentation') - - add_with_without_pair(docs_group, 'sphinx', None, 'run Sphinx to generate docs') - - add_with_without_pair(docs_group, 'pdf', False, 'run Sphinx to generate PDF doc') - - add_with_without_pair(docs_group, 'rst2man', None, 'run rst2man to generate man page') - - add_with_without_pair(docs_group, 'doxygen', False, 'run Doxygen') - - mods_group = optparse.OptionGroup(parser, 'Module selection') - - mods_group.add_option('--module-policy', dest='module_policy', - help="module policy file (see src/build-data/policy)", - metavar='POL', default=None) - - mods_group.add_option('--enable-modules', dest='enabled_modules', - metavar='MODS', action='append', - help='enable specific modules') - mods_group.add_option('--disable-modules', dest='disabled_modules', - metavar='MODS', action='append', - help='disable specific modules') - mods_group.add_option('--no-autoload', action='store_true', default=False, - help=optparse.SUPPRESS_HELP) - mods_group.add_option('--minimized-build', action='store_true', dest='no_autoload', - help='minimize build') - - add_enable_disable_pair(mods_group, 'experimental-features', False, - 'enable building of experimental features and modules') - - add_enable_disable_pair(mods_group, 'deprecated-features', True, - 'disable building of deprecated features and modules') + version=Version.as_string(), + ) + + parser.add_option( + "--verbose", action="store_true", default=False, help="Show debug messages" + ) + parser.add_option( + "--quiet", + action="store_true", + default=False, + help="Show only warnings and errors", + ) + + target_group = optparse.OptionGroup(parser, "Target options") + + target_group.add_option("--cpu", help="set the target CPU architecture") + + target_group.add_option("--os", help="set the target operating system") + + target_group.add_option( + "--cc", dest="compiler", help="set the desired build compiler" + ) + + target_group.add_option( + "--cc-min-version", + dest="cc_min_version", + default=None, + metavar="MAJOR.MINOR", + help="Set the minimal version of the target compiler. " + "Use --cc-min-version=0.0 to support all compiler versions. " + "Default is auto detection.", + ) + + target_group.add_option( + "--cc-bin", + dest="compiler_binary", + metavar="BINARY", + help="set path to compiler binary", + ) + + target_group.add_option( + "--cc-abi-flags", metavar="FLAGS", default="", help="set compiler ABI flags" + ) + + target_group.add_option( + "--cxxflags", metavar="FLAGS", default=None, help="override all compiler flags" + ) + + target_group.add_option( + "--extra-cxxflags", + metavar="FLAGS", + default=[], + action="append", + help="set extra compiler flags", + ) + + target_group.add_option( + "--lto-cxxflags-to-ldflags", + default=False, + action="store_true", + help="set all compilation flags also during linking (for LTO)", + ) + + target_group.add_option( + "--ldflags", metavar="FLAGS", help="set linker flags", default=None + ) + + target_group.add_option( + "--extra-libs", + metavar="LIBS", + help="specify extra libraries to link against", + default="", + ) + + target_group.add_option( + "--ar-command", + dest="ar_command", + metavar="AR", + default=None, + help="set path to static archive creator", + ) + + target_group.add_option( + "--ar-options", + dest="ar_options", + metavar="AR_OPTIONS", + default=None, + help="set options for ar", + ) + + target_group.add_option( + "--msvc-runtime", + metavar="RT", + default=None, + help="specify MSVC runtime (MT, MD, MTd, MDd)", + ) + + target_group.add_option("--compiler-cache", help="specify a compiler cache to use") + + target_group.add_option( + "--ct-value-barrier-type", + metavar="TYPE", + default=None, + help=optparse.SUPPRESS_HELP, + ) + + target_group.add_option( + "--with-os-features", + action="append", + metavar="FEAT", + help="specify OS features to use", + ) + target_group.add_option( + "--without-os-features", + action="append", + metavar="FEAT", + help="specify OS features to disable", + ) + + add_with_without_pair( + target_group, "compilation-database", True, "disable compile_commands.json" + ) + + isa_extensions_that_can_be_disabled = [("NEON", "arm32")] + + for isa_extn_name, arch in isa_extensions_that_can_be_disabled: + isa_extn = isa_extn_name.lower().replace(" ", "") + + nm = isa_extn.replace("-", "").replace(".", "").replace(" ", "") + + target_group.add_option( + "--disable-%s" % (isa_extn), + help="disable %s intrinsics" % (isa_extn_name), + action="append_const", + const=(nm, arch), + dest="disable_intrinsics", + ) + + build_group = optparse.OptionGroup(parser, "Build options") + + build_group.add_option( + "--system-cert-bundle", + metavar="PATH", + default=None, + help="set path to trusted CA bundle", + ) + + add_with_without_pair(build_group, "debug-info", False, "include debug symbols") + + add_with_without_pair(build_group, "sanitizers", False, "enable ASan/UBSan checks") + + add_enable_disable_pair(build_group, "asm", True, "disable inline assembly") + + add_enable_disable_pair( + build_group, + "stack-scrubbing", + False, + "enable compiler-assisted stack scrubbing", + ) + + build_group.add_option( + "--enable-sanitizers", + metavar="SAN", + default="", + help="enable specific sanitizers", + ) + + add_with_without_pair( + build_group, "stack-protector", None, "disable stack smashing protections" + ) + + add_with_without_pair(build_group, "coverage-info", False, "add coverage info") + + build_group.add_option( + "--enable-shared-library", + dest="build_shared_lib", + action="store_true", + default=None, + help=optparse.SUPPRESS_HELP, + ) + build_group.add_option( + "--disable-shared-library", + dest="build_shared_lib", + action="store_false", + help="disable building shared library", + ) + + build_group.add_option( + "--enable-static-library", + dest="build_static_lib", + action="store_true", + default=None, + help=optparse.SUPPRESS_HELP, + ) + build_group.add_option( + "--disable-static-library", + dest="build_static_lib", + action="store_false", + help="disable building static library", + ) + + build_group.add_option( + "--optimize-for-size", + dest="optimize_for_size", + action="store_true", + default=False, + help="optimize for code size", + ) + + build_group.add_option( + "--no-optimizations", + dest="no_optimizations", + action="store_true", + default=False, + help="disable all optimizations (for debugging)", + ) + + build_group.add_option( + "--debug-mode", + action="store_true", + default=False, + dest="debug_mode", + help="enable debug info, disable optimizations", + ) + + build_group.add_option( + "--amalgamation", + dest="amalgamation", + default=False, + action="store_true", + help="use amalgamation to build", + ) + + build_group.add_option( + "--name-amalgamation", + metavar="NAME", + default="botan_all", + help="specify alternate name for amalgamation files", + ) + + build_group.add_option( + "--with-build-dir", metavar="DIR", default="", help="setup the build in DIR" + ) + + build_group.add_option( + "--with-external-includedir", + metavar="DIR", + default=[], + help="use DIR for external includes", + action="append", + ) + + build_group.add_option( + "--with-external-libdir", + metavar="DIR", + default=[], + help="use DIR for external libs", + action="append", + ) + + build_group.add_option( + "--define-build-macro", + metavar="DEFINE", + default=[], + help="set compile-time pre-processor definition like KEY[=VALUE]", + action="append", + ) + + build_group.add_option( + "--with-sysroot-dir", + metavar="DIR", + default="", + help="use DIR for system root while cross-compiling", + ) + + link_methods = ["symlink", "hardlink", "copy"] + build_group.add_option( + "--link-method", + default=None, + metavar="METHOD", + choices=link_methods, + help="choose how links to include headers are created (%s)" + % ", ".join(link_methods), + ) + + build_group.add_option( + "--distribution-info", + metavar="STRING", + help="distribution specific version", + default=None, + ) + + build_group.add_option( + "--maintainer-mode", + dest="maintainer_mode", + action="store_true", + default=False, + help=optparse.SUPPRESS_HELP, + ) + + build_group.add_option( + "--werror-mode", + dest="werror_mode", + action="store_true", + default=False, + help="Prohibit compiler warnings", + ) + + build_group.add_option( + "--no-install-python-module", + action="store_true", + default=False, + help="skip installing Python module", + ) + + build_group.add_option( + "--with-python-versions", + dest="python_version", + metavar="N.M", + default="%d.%d" % (sys.version_info[0], sys.version_info[1]), + help="where to install botan2.py (def %default)", + ) + + build_group.add_option( + "--disable-cc-tests", + dest="enable_cc_tests", + default=True, + action="store_false", + help=optparse.SUPPRESS_HELP, + ) + + add_with_without_pair(build_group, "valgrind", False, "use valgrind API") + + build_group.add_option( + "--unsafe-fuzzer-mode", + action="store_true", + default=False, + help=optparse.SUPPRESS_HELP, + ) + + build_group.add_option( + "--build-fuzzers", + dest="build_fuzzers", + metavar="TYPE", + default=None, + help="Build fuzzers (afl, libfuzzer, klee, test)", + ) + + build_group.add_option( + "--with-fuzzer-lib", + metavar="LIB", + default=None, + dest="fuzzer_lib", + help="additionally link in LIB", + ) + + add_with_without_pair(build_group, "debug-asserts", default=False) + + build_group.add_option( + "--unsafe-terminate-on-asserts", + action="store_true", + default=False, + help=optparse.SUPPRESS_HELP, + ) + + build_group.add_option( + "--build-targets", + default=None, + dest="build_targets", + action="append", + help="build specific targets and tools (%s)" + % ", ".join(ACCEPTABLE_BUILD_TARGETS), + ) + + build_group.add_option( + "--build-tool", default="make", help="specify the build tool (make, ninja)" + ) + + add_with_without_pair(build_group, "pkg-config", default=None) + add_with_without_pair(build_group, "cmake-config", default=True) + + docs_group = optparse.OptionGroup(parser, "Documentation Options") + + add_with_without_pair( + docs_group, "documentation", True, "skip building/installing documentation" + ) + + add_with_without_pair(docs_group, "sphinx", None, "run Sphinx to generate docs") + + add_with_without_pair(docs_group, "pdf", False, "run Sphinx to generate PDF doc") + + add_with_without_pair( + docs_group, "rst2man", None, "run rst2man to generate man page" + ) + + add_with_without_pair(docs_group, "doxygen", False, "run Doxygen") + + mods_group = optparse.OptionGroup(parser, "Module selection") + + mods_group.add_option( + "--module-policy", + dest="module_policy", + help="module policy file (see src/build-data/policy)", + metavar="POL", + default=None, + ) + + mods_group.add_option( + "--enable-modules", + dest="enabled_modules", + metavar="MODS", + action="append", + help="enable specific modules", + ) + mods_group.add_option( + "--disable-modules", + dest="disabled_modules", + metavar="MODS", + action="append", + help="disable specific modules", + ) + mods_group.add_option( + "--no-autoload", action="store_true", default=False, help=optparse.SUPPRESS_HELP + ) + mods_group.add_option( + "--minimized-build", + action="store_true", + dest="no_autoload", + help="minimize build", + ) + + add_enable_disable_pair( + mods_group, + "experimental-features", + False, + "enable building of experimental features and modules", + ) + + add_enable_disable_pair( + mods_group, + "deprecated-features", + True, + "disable building of deprecated features and modules", + ) # Should be derived from info.txt but this runs too early - third_party = ['boost', 'bzip2', 'esdm_rng', 'lzma', 'commoncrypto', 'sqlite3', 'zlib', 'tpm', 'tpm2'] + third_party = [ + "boost", + "bzip2", + "esdm_rng", + "lzma", + "commoncrypto", + "sqlite3", + "zlib", + "tpm", + "tpm2", + ] for mod in third_party: - mods_group.add_option('--with-%s' % (mod), - help=('use %s' % (mod)) if mod in third_party else optparse.SUPPRESS_HELP, - action='append_const', - const=mod, - dest='enabled_modules') - - mods_group.add_option('--without-%s' % (mod), - help=optparse.SUPPRESS_HELP, - action='append_const', - const=mod, - dest='disabled_modules') - - install_group = optparse.OptionGroup(parser, 'Installation options') - - install_group.add_option('--program-suffix', metavar='SUFFIX', - help='append string to program names') - install_group.add_option('--library-suffix', metavar='SUFFIX', default='', - help='append string to library names') - - install_group.add_option('--prefix', metavar='DIR', - help='set the install prefix') - install_group.add_option('--docdir', metavar='DIR', - help='set the doc install dir') - install_group.add_option('--bindir', metavar='DIR', - help='set the binary install dir') - install_group.add_option('--libdir', metavar='DIR', - help='set the library install dir') - install_group.add_option('--mandir', metavar='DIR', - help='set the install dir for man pages') - install_group.add_option('--includedir', metavar='DIR', - help='set the include file install dir') - - info_group = optparse.OptionGroup(parser, 'Informational') - - info_group.add_option('--list-modules', dest='list_modules', - action='store_true', - help='list available modules and exit') - - info_group.add_option('--list-os-features', dest='list_os_features', - action='store_true', - help='list available OS features and exit') + mods_group.add_option( + "--with-%s" % (mod), + help=("use %s" % (mod)) if mod in third_party else optparse.SUPPRESS_HELP, + action="append_const", + const=mod, + dest="enabled_modules", + ) + + mods_group.add_option( + "--without-%s" % (mod), + help=optparse.SUPPRESS_HELP, + action="append_const", + const=mod, + dest="disabled_modules", + ) + + install_group = optparse.OptionGroup(parser, "Installation options") + + install_group.add_option( + "--program-suffix", metavar="SUFFIX", help="append string to program names" + ) + install_group.add_option( + "--library-suffix", + metavar="SUFFIX", + default="", + help="append string to library names", + ) + + install_group.add_option("--prefix", metavar="DIR", help="set the install prefix") + install_group.add_option("--docdir", metavar="DIR", help="set the doc install dir") + install_group.add_option( + "--bindir", metavar="DIR", help="set the binary install dir" + ) + install_group.add_option( + "--libdir", metavar="DIR", help="set the library install dir" + ) + install_group.add_option( + "--mandir", metavar="DIR", help="set the install dir for man pages" + ) + install_group.add_option( + "--includedir", metavar="DIR", help="set the include file install dir" + ) + + info_group = optparse.OptionGroup(parser, "Informational") + + info_group.add_option( + "--list-modules", + dest="list_modules", + action="store_true", + help="list available modules and exit", + ) + + info_group.add_option( + "--list-os-features", + dest="list_os_features", + action="store_true", + help="list available OS features and exit", + ) parser.add_option_group(target_group) parser.add_option_group(build_group) @@ -665,30 +975,30 @@ def add_enable_disable_pair(group, what, default, msg=optparse.SUPPRESS_HELP): # These exist only for autoconf compatibility (requested by zw for mtn) compat_with_autoconf_options = [ - 'datadir', - 'datarootdir', - 'dvidir', - 'exec-prefix', - 'htmldir', - 'infodir', - 'libexecdir', - 'localedir', - 'localstatedir', - 'oldincludedir', - 'pdfdir', - 'psdir', - 'sbindir', - 'sharedstatedir', - 'sysconfdir' - ] + "datadir", + "datarootdir", + "dvidir", + "exec-prefix", + "htmldir", + "infodir", + "libexecdir", + "localedir", + "localstatedir", + "oldincludedir", + "pdfdir", + "psdir", + "sbindir", + "sharedstatedir", + "sysconfdir", + ] for opt in compat_with_autoconf_options: - parser.add_option('--' + opt, help=optparse.SUPPRESS_HELP) + parser.add_option("--" + opt, help=optparse.SUPPRESS_HELP) (options, args) = parser.parse_args(args) if args != []: - raise UserError('Unhandled option(s): ' + ' '.join(args)) + raise UserError("Unhandled option(s): " + " ".join(args)) if options.debug_mode: options.no_optimizations = True @@ -698,7 +1008,7 @@ def parse_multiple_enable(modules): if modules is None: return [] - return sorted({m for m in flatten([s.split(',') for s in modules]) if m != ''}) + return sorted({m for m in flatten([s.split(",") for s in modules]) if m != ""}) options.enabled_modules = parse_multiple_enable(options.enabled_modules) options.disabled_modules = parse_multiple_enable(options.disabled_modules) @@ -706,10 +1016,13 @@ def parse_multiple_enable(modules): options.with_os_features = parse_multiple_enable(options.with_os_features) options.without_os_features = parse_multiple_enable(options.without_os_features) - options.disable_intrinsics = [] if options.disable_intrinsics is None else options.disable_intrinsics + options.disable_intrinsics = ( + [] if options.disable_intrinsics is None else options.disable_intrinsics + ) return options + def take_options_from_env(options): # Take some values from environment, if not set on command line @@ -717,18 +1030,27 @@ def update_from_env(val, var, name): if val is None: val = os.getenv(var) if val is not None: - logging.info('Implicit --%s=%s due to environment variable %s', name, val, var) + logging.info( + "Implicit --%s=%s due to environment variable %s", name, val, var + ) return val - if os.getenv('CXX') and options.compiler_binary is None and options.compiler is not None: - logging.info('CXX environment variable is set which will override compiler path') + if ( + os.getenv("CXX") + and options.compiler_binary is None + and options.compiler is not None + ): + logging.info( + "CXX environment variable is set which will override compiler path" + ) + + options.ar_command = update_from_env(options.ar_command, "AR", "ar-command") + options.ar_options = update_from_env(options.ar_options, "AR_OPTIONS", "ar-options") + options.compiler_binary = update_from_env(options.compiler_binary, "CXX", "cc-bin") + options.cxxflags = update_from_env(options.cxxflags, "CXXFLAGS", "cxxflags") + options.ldflags = update_from_env(options.ldflags, "LDFLAGS", "ldflags") - options.ar_command = update_from_env(options.ar_command, 'AR', 'ar-command') - options.ar_options = update_from_env(options.ar_options, 'AR_OPTIONS', 'ar-options') - options.compiler_binary = update_from_env(options.compiler_binary, 'CXX', 'cc-bin') - options.cxxflags = update_from_env(options.cxxflags, 'CXXFLAGS', 'cxxflags') - options.ldflags = update_from_env(options.ldflags, 'LDFLAGS', 'ldflags') class LexResult: pass @@ -742,21 +1064,31 @@ def __init__(self, msg, lexfile, line): self.line = line def __str__(self): - return '%s at %s:%d' % (self.msg, self.lexfile, self.line) + return "%s at %s:%d" % (self.msg, self.lexfile, self.line) + def parse_lex_dict(as_list, map_name, infofile): if len(as_list) % 3 != 0: - raise InternalError("Lex dictionary has invalid format (input not divisible by 3): %s" % as_list) + raise InternalError( + "Lex dictionary has invalid format (input not divisible by 3): %s" % as_list + ) result = {} - for key, sep, value in [as_list[3*i:3*i+3] for i in range(0, len(as_list)//3)]: - if sep != '->': - raise InternalError("Map %s in %s has invalid format" % (map_name, infofile)) + for key, sep, value in [ + as_list[3 * i : 3 * i + 3] for i in range(0, len(as_list) // 3) + ]: + if sep != "->": + raise InternalError( + "Map %s in %s has invalid format" % (map_name, infofile) + ) if key in result: - raise InternalError("Duplicate map entry %s in map %s file %s" % (key, map_name, infofile)) + raise InternalError( + "Duplicate map entry %s in map %s file %s" % (key, map_name, infofile) + ) result[key] = value return result + def lex_me_harder(infofile, allowed_groups, allowed_maps, name_val_pairs): """ Generic lexer function for info.txt and src/build-data files @@ -765,18 +1097,18 @@ def lex_me_harder(infofile, allowed_groups, allowed_maps, name_val_pairs): # Format as a nameable Python variable def py_var(group): - return group.replace(':', '_') + return group.replace(":", "_") - lexer = shlex.shlex(open(infofile, encoding='utf8'), infofile, posix=True) - lexer.wordchars += '=:.<>/,-!?+*' # handle various funky chars in info.txt + lexer = shlex.shlex(open(infofile, encoding="utf8"), infofile, posix=True) + lexer.wordchars += "=:.<>/,-!?+*" # handle various funky chars in info.txt groups = allowed_groups + allowed_maps for group in groups: out.__dict__[py_var(group)] = [] - for (key, val) in name_val_pairs.items(): + for key, val in name_val_pairs.items(): out.__dict__[key] = val - def lexed_tokens(): # Convert to an iterator + def lexed_tokens(): # Convert to an iterator while True: token = lexer.get_token() if token != lexer.eof: @@ -785,25 +1117,25 @@ def lexed_tokens(): # Convert to an iterator return for token in lexed_tokens(): - match = re.match('<(.*)>', token) + match = re.match("<(.*)>", token) # Check for a grouping if match is not None: group = match.group(1) if group not in groups: - raise LexerError('Unknown group "%s"' % (group), - infofile, lexer.lineno) + raise LexerError('Unknown group "%s"' % (group), infofile, lexer.lineno) - end_marker = '' + end_marker = "" token = lexer.get_token() while token != end_marker: out.__dict__[py_var(group)].append(token) token = lexer.get_token() if token is None: - raise LexerError('Group "%s" not terminated' % (group), - infofile, lexer.lineno) + raise LexerError( + 'Group "%s" not terminated' % (group), infofile, lexer.lineno + ) elif token in name_val_pairs.keys(): if isinstance(out.__dict__[token], list): @@ -811,7 +1143,7 @@ def lexed_tokens(): # Convert to an iterator else: out.__dict__[token] = lexer.get_token() - else: # No match -> error + else: # No match -> error raise LexerError('Bad token "%s"' % (token), infofile, lexer.lineno) for group in allowed_maps: @@ -819,6 +1151,7 @@ def lexed_tokens(): # Convert to an iterator return out + class InfoObject: def __init__(self, infofile): """ @@ -828,21 +1161,21 @@ def __init__(self, infofile): self.infofile = infofile (dirname, basename) = os.path.split(infofile) self.lives_in = dirname - if basename == 'info.txt': + if basename == "info.txt": (next_dir, self.basename) = os.path.split(dirname) self.parent_module = None - obj_dir = '' + obj_dir = "" while next_dir != obj_dir: obj_dir = next_dir - if os.access(os.path.join(obj_dir, 'info.txt'), os.R_OK): + if os.access(os.path.join(obj_dir, "info.txt"), os.R_OK): self.parent_module = os.path.basename(obj_dir) break (next_dir, _) = os.path.split(obj_dir) - if os.path.basename(next_dir) == 'src': + if os.path.basename(next_dir) == "src": break else: - self.basename = basename.replace('.txt', '') + self.basename = basename.replace(".txt", "") class ModuleInfo(InfoObject): @@ -854,19 +1187,32 @@ def __init__(self, infofile): super().__init__(infofile) lex = lex_me_harder( infofile, - ['header:internal', 'header:public', 'header:external', 'requires', - 'os_features', 'arch', 'isa', 'cc', 'comment', 'warning'], - ['defines', 'internal_defines', 'libs', 'frameworks', 'module_info'], + [ + "header:internal", + "header:public", + "header:external", + "requires", + "os_features", + "arch", + "isa", + "cc", + "comment", + "warning", + ], + ["defines", "internal_defines", "libs", "frameworks", "module_info"], { - 'load_on': 'auto', - }) + "load_on": "auto", + }, + ) def check_header_duplicates(header_list_public, header_list_internal): pub_header = set(header_list_public) int_header = set(header_list_internal) if not pub_header.isdisjoint(int_header): - logging.error("Module %s has same header in public and internal sections", - self.infofile) + logging.error( + "Module %s has same header in public and internal sections", + self.infofile, + ) check_header_duplicates(lex.header_public, lex.header_internal) @@ -874,9 +1220,9 @@ def check_header_duplicates(header_list_public, header_list_internal): all_header_files = [] for fspath in os.listdir(self.lives_in): - if fspath.endswith('.cpp'): + if fspath.endswith(".cpp"): all_source_files.append(fspath) - elif fspath.endswith('.h'): + elif fspath.endswith(".h"): all_header_files.append(fspath) self.source = all_source_files @@ -892,12 +1238,12 @@ def check_header_duplicates(header_list_public, header_list_internal): def convert_lib_list(libs): out = {} - for (os_name, lib_list) in libs.items(): - out[os_name] = lib_list.split(',') + for os_name, lib_list in libs.items(): + out[os_name] = lib_list.split(",") return out def combine_lines(c): - return ' '.join(c) if c else None + return " ".join(c) if c else None # Convert remaining lex result to members self.arch = lex.arch @@ -917,13 +1263,26 @@ def combine_lines(c): self._parse_module_info(lex) # Modify members - self.source = [normalize_source_path(os.path.join(self.lives_in, s)) for s in self.source] - self.header_internal = [os.path.join(self.lives_in, s) for s in self.header_internal] - self.header_public = [os.path.join(self.lives_in, s) for s in self.header_public] - self.header_external = [os.path.join(self.lives_in, s) for s in self.header_external] + self.source = [ + normalize_source_path(os.path.join(self.lives_in, s)) for s in self.source + ] + self.header_internal = [ + os.path.join(self.lives_in, s) for s in self.header_internal + ] + self.header_public = [ + os.path.join(self.lives_in, s) for s in self.header_public + ] + self.header_external = [ + os.path.join(self.lives_in, s) for s in self.header_external + ] # Filesystem read access check - for src in self.source + self.header_internal + self.header_public + self.header_external: + for src in ( + self.source + + self.header_internal + + self.header_public + + self.header_external + ): if not os.access(src, os.R_OK): logging.error("Missing file %s in %s", src, infofile) @@ -931,82 +1290,121 @@ def combine_lines(c): def intersect_check(type_a, list_a, type_b, list_b): intersection = set.intersection(set(list_a), set(list_b)) if intersection: - logging.error('Headers %s marked both %s and %s', ' '.join(intersection), type_a, type_b) - - intersect_check('public', self.header_public, 'internal', self.header_internal) - intersect_check('public', self.header_public, 'external', self.header_external) - intersect_check('external', self.header_external, 'internal', self.header_internal) + logging.error( + "Headers %s marked both %s and %s", + " ".join(intersection), + type_a, + type_b, + ) + + intersect_check("public", self.header_public, "internal", self.header_internal) + intersect_check("public", self.header_public, "external", self.header_external) + intersect_check( + "external", self.header_external, "internal", self.header_internal + ) # Check module type constraints source_file_count = len(all_source_files) + len(all_header_files) if self.is_virtual() and source_file_count > 0: - logging.error("Module '%s' is virtual but contains %d source code files", self.basename, source_file_count) + logging.error( + "Module '%s' is virtual but contains %d source code files", + self.basename, + source_file_count, + ) def _parse_module_info(self, lex): info = lex.module_info if "name" not in info: - raise InternalError("Module '%s' does not contain a section with at least a documentation-friendly 'name' definition" % self.basename) + raise InternalError( + "Module '%s' does not contain a section with at least a documentation-friendly 'name' definition" + % self.basename + ) self.name = info["name"] - self.brief = info.get("brief") # possibly None + self.brief = info.get("brief") # possibly None self.type = info.get("type") or "Public" self.lifecycle = info.get("lifecycle") or "Stable" if self.type not in ["Public", "Internal", "Virtual"]: - raise InternalError("Module '%s' has an unknown type: %s" % (self.basename, self.type)) + raise InternalError( + "Module '%s' has an unknown type: %s" % (self.basename, self.type) + ) if self.lifecycle not in ["Stable", "Experimental", "Deprecated"]: - raise InternalError("Module '%s' has an unknown lifecycle status: %s" % (self.basename, self.lifecycle)) + raise InternalError( + "Module '%s' has an unknown lifecycle status: %s" + % (self.basename, self.lifecycle) + ) @staticmethod def _validate_defines_content(defines): for key, value in defines.items(): - if not re.match('^[0-9A-Za-z_]{3,30}$', key): + if not re.match("^[0-9A-Za-z_]{3,30}$", key): raise InternalError('Module defines key has invalid format: "%s"' % key) - if not re.match('^20[0-9]{6}$', value): - raise InternalError('Module defines value has invalid format: "%s" (should be YYYYMMDD)' % value) + if not re.match("^20[0-9]{6}$", value): + raise InternalError( + 'Module defines value has invalid format: "%s" (should be YYYYMMDD)' + % value + ) year = int(value[0:4]) month = int(value[4:6]) day = int(value[6:]) if year < 2013 or month == 0 or month > 12 or day == 0 or day > 31: - raise InternalError('Module defines value has invalid format: "%s" (should be YYYYMMDD)' % value) + raise InternalError( + 'Module defines value has invalid format: "%s" (should be YYYYMMDD)' + % value + ) def cross_check(self, arch_info, cc_info, all_os_features, all_isa_extn): - - for feat in set(flatten([o.split(',') for o in self.os_features])): + for feat in set(flatten([o.split(",") for o in self.os_features])): if feat not in all_os_features: - logging.error("Module %s uses an OS feature (%s) which no OS supports", self.infofile, feat) + logging.error( + "Module %s uses an OS feature (%s) which no OS supports", + self.infofile, + feat, + ) for supp_cc in self.cc: if supp_cc not in cc_info: - if supp_cc.startswith('!') and supp_cc[1:] in cc_info: + if supp_cc.startswith("!") and supp_cc[1:] in cc_info: continue - colon_idx = supp_cc.find(':') + colon_idx = supp_cc.find(":") # a versioned compiler dependency if colon_idx > 0 and supp_cc[0:colon_idx] in cc_info: pass else: - raise InternalError('Module %s mentions unknown compiler %s' % (self.infofile, supp_cc)) + raise InternalError( + "Module %s mentions unknown compiler %s" + % (self.infofile, supp_cc) + ) for supp_arch in self.arch: if supp_arch not in arch_info: - raise InternalError('Module %s mentions unknown arch %s' % (self.infofile, supp_arch)) + raise InternalError( + "Module %s mentions unknown arch %s" % (self.infofile, supp_arch) + ) def known_isa(isa): if isa in all_isa_extn: return True - compound_isa = isa.split(':') - if len(compound_isa) == 2 and compound_isa[0] in arch_info and compound_isa[1] in all_isa_extn: + compound_isa = isa.split(":") + if ( + len(compound_isa) == 2 + and compound_isa[0] in arch_info + and compound_isa[1] in all_isa_extn + ): return True return False for isa in self.isa: if not known_isa(isa): - raise InternalError('Module %s uses unknown ISA extension %s' % (self.infofile, isa)) + raise InternalError( + "Module %s uses unknown ISA extension %s" % (self.infofile, isa) + ) def sources(self): return self.source @@ -1024,32 +1422,32 @@ def isas_needed(self, arch): isas = [] for isa in self.isa: - if isa.find(':') == -1: + if isa.find(":") == -1: isas.append(isa) - elif isa.startswith(arch + ':'): - isas.append(isa[len(arch)+1:]) + elif isa.startswith(arch + ":"): + isas.append(isa[len(arch) + 1 :]) return isas def defines(self): - return [(key + ' ' + value) for key, value in self._defines.items()] + return [(key + " " + value) for key, value in self._defines.items()] def internal_defines(self): - return [(key + ' ' + value) for key, value in self._internal_defines.items()] + return [(key + " " + value) for key, value in self._internal_defines.items()] def compatible_cpu(self, archinfo, options): arch_name = archinfo.basename cpu_name = options.arch for isa in self.isa: - if isa.find(':') > 0: - (arch, isa) = isa.split(':') + if isa.find(":") > 0: + (arch, isa) = isa.split(":") if arch != arch_name: continue if (isa, arch_name) in options.disable_intrinsics: - return False # explicitly disabled + return False # explicitly disabled if isa not in archinfo.isa_extensions: return False @@ -1073,7 +1471,7 @@ def has_all(needed, provided): provided_features = os_data.enabled_features(options) for feature_set in self.os_features: - if has_all(feature_set.split(','), provided_features): + if has_all(feature_set.split(","), provided_features): return True return False @@ -1096,12 +1494,12 @@ def supported_compiler(ccinfo, cc_min_version): # compiler is supported, independent of version return True - if '!%s' % (ccinfo.basename) in self.cc: + if "!%s" % (ccinfo.basename) in self.cc: # an explicit exclusion of this compiler return False # If just exclusions are given, treat as accept if we do not match - if all(cc.startswith('!') for cc in self.cc): + if all(cc.startswith("!") for cc in self.cc): return True # Maybe a versioned compiler dep @@ -1109,27 +1507,29 @@ def supported_compiler(ccinfo, cc_min_version): try: name, version = cc.split(":") if name == ccinfo.basename: - min_cc_version = [int(v) for v in version.split('.')] - cur_cc_version = [int(v) for v in cc_min_version.split('.')] + min_cc_version = [int(v) for v in version.split(".")] + cur_cc_version = [int(v) for v in cc_min_version.split(".")] # With lists of ints, this does what we want return cur_cc_version >= min_cc_version except ValueError: # No version part specified pass - return False # compiler not listed + return False # compiler not listed - return supported_isa_flags(ccinfo, arch) and supported_compiler(ccinfo, cc_min_version) + return supported_isa_flags(ccinfo, arch) and supported_compiler( + ccinfo, cc_min_version + ) def dependencies(self, osinfo, archinfo): # base is an implicit dep for all submodules - deps = ['base'] + deps = ["base"] if self.parent_module is not None: deps.append(self.parent_module) for req in self.requires: - if req.find('?') != -1: - (cond, dep) = req.split('?') + if req.find("?") != -1: + (cond, dep) = req.split("?") if osinfo is None and archinfo is None: deps.append(dep) else: @@ -1155,16 +1555,24 @@ def is_dependency_on_virtual(this_module, dependency): return True - missing = [s for s in self.dependencies(None, None) if s not in modules or is_dependency_on_virtual(self, modules[s])] + missing = [ + s + for s in self.dependencies(None, None) + if s not in modules or is_dependency_on_virtual(self, modules[s]) + ] for modname in missing: if modname not in modules: - logging.error("Module '%s', dep of '%s', does not exist", - missing, self.basename) + logging.error( + "Module '%s', dep of '%s', does not exist", missing, self.basename + ) else: assert modules[modname].is_virtual() - logging.error("Module '%s' is virtual and cannot be depended on by '%s'", - modname, self.basename) + logging.error( + "Module '%s' is virtual and cannot be depended on by '%s'", + modname, + self.basename, + ) def is_public(self): return self.type == "Public" @@ -1184,14 +1592,13 @@ def is_experimental(self): def is_deprecated(self): return self.lifecycle == "Deprecated" + class ModulePolicyInfo(InfoObject): def __init__(self, infofile): super().__init__(infofile) lex = lex_me_harder( - infofile, - ['required', 'if_available', 'prohibited'], - [], - {}) + infofile, ["required", "if_available", "prohibited"], [], {} + ) self.if_available = lex.if_available self.required = lex.required @@ -1208,9 +1615,9 @@ def check(tp, lst, required): else: logging.warning(msg, self.infofile, mod, tp) - check('required', self.required, True) - check('if_available', self.if_available, False) - check('prohibited', self.prohibited, False) + check("required", self.required, True) + check("if_available", self.if_available, False) + check("prohibited", self.prohibited, False) class ArchInfo(InfoObject): @@ -1218,17 +1625,18 @@ def __init__(self, infofile): super().__init__(infofile) lex = lex_me_harder( infofile, - ['aliases', 'isa_extensions'], + ["aliases", "isa_extensions"], [], { - 'family': None, - }) + "family": None, + }, + ) self.aliases = lex.aliases self.family = lex.family self.isa_extensions = lex.isa_extensions - alphanumeric = re.compile('^[a-z0-9]+$') + alphanumeric = re.compile("^[a-z0-9]+$") for isa in self.isa_extensions: if alphanumeric.match(isa) is None: logging.error('Invalid name for ISA extension "%s"', isa) @@ -1250,46 +1658,56 @@ def __init__(self, infofile): lex = lex_me_harder( infofile, [], - ['cpu_flags', 'cpu_flags_no_debug', 'so_link_commands', 'binary_link_commands', - 'mach_abi_linking', 'isa_flags', 'sanitizers', 'lib_flags', 'ct_value_barrier'], + [ + "cpu_flags", + "cpu_flags_no_debug", + "so_link_commands", + "binary_link_commands", + "mach_abi_linking", + "isa_flags", + "sanitizers", + "lib_flags", + "ct_value_barrier", + ], { - 'binary_name': None, - 'linker_name': None, - 'macro_name': None, - 'minimum_supported_version': None, - 'output_to_object': '-o ', - 'output_to_exe': '-o ', - 'add_include_dir_option': '-I', - 'add_system_include_dir_option': '-I', - 'add_lib_dir_option': '-L', - 'add_compile_definition_option': '-D', - 'add_sysroot_option': '', - 'add_lib_option': '-l%s', - 'add_framework_option': '-framework ', - 'preproc_flags': '-E', - 'compile_flags': '-c', - 'debug_info_flags': '-g', - 'optimization_flags': '', - 'size_optimization_flags': '', - 'sanitizer_optimization_flags': '', - 'coverage_flags': '', - 'stack_protector_flags': '', - 'shared_flags': '', - 'lang_flags': '', - 'lang_binary_linker_flags': '', - 'warning_flags': '', - 'maintainer_warning_flags': '', - 'visibility_build_flags': '', - 'visibility_attribute': '', - 'ar_command': '', - 'ar_options': '', - 'ar_output_to': '', - 'werror_flags': '', - 'supports_gcc_inline_asm': 'no', - 'ninja_header_deps_style': '', - 'header_deps_flag': '', - 'header_deps_out': '', - }) + "binary_name": None, + "linker_name": None, + "macro_name": None, + "minimum_supported_version": None, + "output_to_object": "-o ", + "output_to_exe": "-o ", + "add_include_dir_option": "-I", + "add_system_include_dir_option": "-I", + "add_lib_dir_option": "-L", + "add_compile_definition_option": "-D", + "add_sysroot_option": "", + "add_lib_option": "-l%s", + "add_framework_option": "-framework ", + "preproc_flags": "-E", + "compile_flags": "-c", + "debug_info_flags": "-g", + "optimization_flags": "", + "size_optimization_flags": "", + "sanitizer_optimization_flags": "", + "coverage_flags": "", + "stack_protector_flags": "", + "shared_flags": "", + "lang_flags": "", + "lang_binary_linker_flags": "", + "warning_flags": "", + "maintainer_warning_flags": "", + "visibility_build_flags": "", + "visibility_attribute": "", + "ar_command": "", + "ar_options": "", + "ar_output_to": "", + "werror_flags": "", + "supports_gcc_inline_asm": "no", + "ninja_header_deps_style": "", + "header_deps_flag": "", + "header_deps_out": "", + }, + ) self.add_framework_option = lex.add_framework_option self.add_include_dir_option = lex.add_include_dir_option @@ -1327,7 +1745,7 @@ def __init__(self, infofile): self.size_optimization_flags = lex.size_optimization_flags self.so_link_commands = lex.so_link_commands self.stack_protector_flags = lex.stack_protector_flags - self.supports_gcc_inline_asm = lex.supports_gcc_inline_asm == 'yes' + self.supports_gcc_inline_asm = lex.supports_gcc_inline_asm == "yes" self.visibility_attribute = lex.visibility_attribute self.visibility_build_flags = lex.visibility_build_flags self.warning_flags = lex.warning_flags @@ -1339,38 +1757,49 @@ def __init__(self, infofile): self.ct_value_barrier = lex.ct_value_barrier def cross_check(self, os_info, arch_info, all_isas): - for isa in self.isa_flags: if ":" in isa: (arch, isa) = isa.split(":") if isa not in all_isas: - raise InternalError('Compiler %s has flags for unknown ISA %s' % (self.infofile, isa)) + raise InternalError( + "Compiler %s has flags for unknown ISA %s" + % (self.infofile, isa) + ) if arch not in arch_info: - raise InternalError('Compiler %s has flags for unknown arch/ISA %s:%s' % (self.infofile, arch, isa)) + raise InternalError( + "Compiler %s has flags for unknown arch/ISA %s:%s" + % (self.infofile, arch, isa) + ) for os_name in self.binary_link_commands: if os_name in ["default", "default-debug"]: continue if os_name not in os_info: - raise InternalError("Compiler %s has binary_link_command for unknown OS %s" % (self.infofile, os_name)) + raise InternalError( + "Compiler %s has binary_link_command for unknown OS %s" + % (self.infofile, os_name) + ) for os_name in self.so_link_commands: if os_name in ["default", "default-debug"]: continue if os_name not in os_info: - raise InternalError("Compiler %s has so_link_command for unknown OS %s" % (self.infofile, os_name)) + raise InternalError( + "Compiler %s has so_link_command for unknown OS %s" + % (self.infofile, os_name) + ) def isa_flags_for(self, isa, arch): - if isa.find(':') > 0: - (isa_arch, isa) = isa.split(':') + if isa.find(":") > 0: + (isa_arch, isa) = isa.split(":") if isa_arch != arch: - return '' + return "" if isa in self.isa_flags: return self.isa_flags[isa] if isa in self.isa_flags: return self.isa_flags[isa] - arch_isa = '%s:%s' % (arch, isa) + arch_isa = "%s:%s" % (arch, isa) if arch_isa in self.isa_flags: return self.isa_flags[arch_isa] @@ -1380,16 +1809,17 @@ def get_isa_specific_flags(self, isas, arch, options): flags = set() def simd32_impl(): - for simd_isa in ['ssse3', 'altivec', 'neon']: - if simd_isa in arch.isa_extensions and \ - (simd_isa, arch.basename) not in options.disable_intrinsics and \ - self.isa_flags_for(simd_isa, arch.basename): + for simd_isa in ["ssse3", "altivec", "neon"]: + if ( + simd_isa in arch.isa_extensions + and (simd_isa, arch.basename) not in options.disable_intrinsics + and self.isa_flags_for(simd_isa, arch.basename) + ): return simd_isa return None for isa in isas: - - if isa == 'simd': + if isa == "simd": isa = simd32_impl() if isa is None: @@ -1397,7 +1827,9 @@ def simd32_impl(): flagset = self.isa_flags_for(isa, arch.basename) if flagset is None: - raise UserError('Compiler %s does not support %s' % (self.basename, isa)) + raise UserError( + "Compiler %s does not support %s" % (self.basename, isa) + ) flags.add(flagset) return " ".join(sorted(flags)) @@ -1416,40 +1848,45 @@ def flag_builder(): if options.build_shared_lib: yield self.visibility_build_flags - if 'debug' in self.lib_flags and options.with_debug_info: - yield process_template_string(self.lib_flags['debug'], variables, self.infofile) - + if "debug" in self.lib_flags and options.with_debug_info: + yield process_template_string( + self.lib_flags["debug"], variables, self.infofile + ) - return ' '.join(list(flag_builder())) + return " ".join(list(flag_builder())) def gen_visibility_attribute(self, options): if options.build_shared_lib: return self.visibility_attribute - return '' + return "" def ct_value_barrier_type(self, options): if options.ct_value_barrier_type: - if options.ct_value_barrier_type == 'asm' and not self.supports_gcc_inline_asm: - raise UserError('Invalid setting for --ct-value-barrier-type: the requested compiler does not support GCC inline asm') + if ( + options.ct_value_barrier_type == "asm" + and not self.supports_gcc_inline_asm + ): + raise UserError( + "Invalid setting for --ct-value-barrier-type: the requested compiler does not support GCC inline asm" + ) return options.ct_value_barrier_type - if 'memory' in self.sanitizer_types: + if "memory" in self.sanitizer_types: return None if self.ct_value_barrier: - for pref in [options.arch, 'default']: + for pref in [options.arch, "default"]: if pref in self.ct_value_barrier: x = self.ct_value_barrier[pref] - if x == 'asm' and not options.enable_asm: + if x == "asm" and not options.enable_asm: return None - if x == 'none': + if x == "none": return None return x return None def mach_abi_link_flags(self, options, debug_mode=None): - """ Return the machine specific ABI flags """ @@ -1458,17 +1895,18 @@ def mach_abi_link_flags(self, options, debug_mode=None): debug_mode = options.debug_mode def mach_abi_groups(): - - yield 'all' + yield "all" if options.msvc_runtime is None: if debug_mode: - yield 'rt-debug' + yield "rt-debug" else: - yield 'rt' + yield "rt" - for all_except in [s for s in self.mach_abi_linking.keys() if s.startswith('all!')]: - exceptions = all_except[4:].split(',') + for all_except in [ + s for s in self.mach_abi_linking.keys() if s.startswith("all!") + ]: + exceptions = all_except[4:].split(",") if options.os not in exceptions and options.arch not in exceptions: yield all_except @@ -1479,46 +1917,48 @@ def mach_abi_groups(): for what in mach_abi_groups(): if what in self.mach_abi_linking: flag = self.mach_abi_linking.get(what) - if flag is not None and flag != '' and flag not in abi_link: + if flag is not None and flag != "" and flag not in abi_link: abi_link.add(flag) if options.msvc_runtime: abi_link.add("/" + options.msvc_runtime) - if options.with_stack_protector and self.stack_protector_flags != '': + if options.with_stack_protector and self.stack_protector_flags != "": abi_link.add(self.stack_protector_flags) if options.with_coverage_info: - if self.coverage_flags == '': - raise UserError('No coverage handling for %s' % (self.basename)) + if self.coverage_flags == "": + raise UserError("No coverage handling for %s" % (self.basename)) abi_link.add(self.coverage_flags) - if options.with_sanitizers or options.enable_sanitizers != '': + if options.with_sanitizers or options.enable_sanitizers != "": if not self.sanitizers: - raise UserError('No sanitizer handling for %s' % (self.basename)) + raise UserError("No sanitizer handling for %s" % (self.basename)) - default_san = self.sanitizers['default'].split(',') + default_san = self.sanitizers["default"].split(",") if options.enable_sanitizers: - san = options.enable_sanitizers.split(',') + san = options.enable_sanitizers.split(",") else: san = default_san for s in san: if s not in self.sanitizers: - raise UserError('No flags defined for sanitizer %s in %s' % (s, self.basename)) + raise UserError( + "No flags defined for sanitizer %s in %s" % (s, self.basename) + ) - if s == 'default': + if s == "default": abi_link.update([self.sanitizers[x] for x in default_san]) else: abi_link.add(self.sanitizers[s]) self.sanitizer_types = san - abi_flags = ' '.join(sorted(abi_link)) + abi_flags = " ".join(sorted(abi_link)) - if options.cc_abi_flags != '': - abi_flags += ' ' + options.cc_abi_flags + if options.cc_abi_flags != "": + abi_flags += " " + options.cc_abi_flags return abi_flags @@ -1530,7 +1970,7 @@ def gen_flags(): if options.maintainer_mode: yield self.maintainer_warning_flags - return (' '.join(gen_flags())).strip() + return (" ".join(gen_flags())).strip() def cc_lang_flags(self): return self.lang_flags @@ -1546,7 +1986,9 @@ def ldflags(self, options): yield from self.cc_compile_flags(options) def cc_compile_flags(self, options): - sanitizers_enabled = options.with_sanitizers or (len(options.enable_sanitizers) > 0) + sanitizers_enabled = options.with_sanitizers or ( + len(options.enable_sanitizers) > 0 + ) if options.cxxflags: # CXXFLAGS is assumed to be the entire set of desired compilation flags @@ -1559,12 +2001,14 @@ def cc_compile_flags(self, options): if not options.no_optimizations: if options.optimize_for_size: - if self.size_optimization_flags != '': + if self.size_optimization_flags != "": yield self.size_optimization_flags else: - logging.warning("No size optimization flags set for current compiler") + logging.warning( + "No size optimization flags set for current compiler" + ) yield self.optimization_flags - elif sanitizers_enabled and self.sanitizer_optimization_flags != '': + elif sanitizers_enabled and self.sanitizer_optimization_flags != "": yield self.sanitizer_optimization_flags else: yield self.optimization_flags @@ -1584,9 +2028,9 @@ def cc_compile_flags(self, options): @staticmethod def _so_link_search(osname, debug_info): - so_link_typ = [osname, 'default'] + so_link_typ = [osname, "default"] if debug_info: - so_link_typ = [link + '-debug' for link in so_link_typ] + so_link_typ + so_link_typ = [link + "-debug" for link in so_link_typ] + so_link_typ return so_link_typ def so_link_command_for(self, osname, options): @@ -1599,8 +2043,9 @@ def so_link_command_for(self, osname, options): return self.so_link_commands[s] raise InternalError( - "No shared library link command found for target '%s' in compiler settings '%s'" % - (osname, self.infofile)) + "No shared library link command found for target '%s' in compiler settings '%s'" + % (osname, self.infofile) + ) def binary_link_command_for(self, osname, options): """ @@ -1611,50 +2056,52 @@ def binary_link_command_for(self, osname, options): if s in self.binary_link_commands: return self.binary_link_commands[s] - return '{linker}' + return "{linker}" + class OsInfo(InfoObject): def __init__(self, infofile): super().__init__(infofile) lex = lex_me_harder( infofile, - ['aliases', 'target_features', 'feature_macros'], + ["aliases", "target_features", "feature_macros"], [], { - 'program_suffix': '', - 'obj_suffix': 'o', - 'soname_suffix': '', - 'soname_pattern_patch': '', - 'soname_pattern_abi': '', - 'soname_pattern_base': '', - 'static_suffix': 'a', - 'ar_command': 'ar', - 'ar_options': '', - 'ar_output_to': '', - 'install_root': '/usr/local', - 'header_dir': 'include', - 'bin_dir': 'bin', - 'lib_dir': 'lib', - 'doc_dir': 'share/doc', - 'man_dir': 'share/man', - 'use_stack_protector': 'true', - 'cli_exe_name': 'botan', - 'lib_prefix': 'lib', - 'library_name': 'botan-{major}{suffix}', - 'shared_lib_symlinks': 'yes', - 'default_compiler': 'gcc', - 'uses_pkg_config': 'yes', - }) - - if lex.ar_command == 'ar' and lex.ar_options == '': - lex.ar_options = 'crs' + "program_suffix": "", + "obj_suffix": "o", + "soname_suffix": "", + "soname_pattern_patch": "", + "soname_pattern_abi": "", + "soname_pattern_base": "", + "static_suffix": "a", + "ar_command": "ar", + "ar_options": "", + "ar_output_to": "", + "install_root": "/usr/local", + "header_dir": "include", + "bin_dir": "bin", + "lib_dir": "lib", + "doc_dir": "share/doc", + "man_dir": "share/man", + "use_stack_protector": "true", + "cli_exe_name": "botan", + "lib_prefix": "lib", + "library_name": "botan-{major}{suffix}", + "shared_lib_symlinks": "yes", + "default_compiler": "gcc", + "uses_pkg_config": "yes", + }, + ) + + if lex.ar_command == "ar" and lex.ar_options == "": + lex.ar_options = "crs" if lex.soname_pattern_base: self.soname_pattern_base = lex.soname_pattern_base - if lex.soname_pattern_patch == '' and lex.soname_pattern_abi == '': + if lex.soname_pattern_patch == "" and lex.soname_pattern_abi == "": self.soname_pattern_patch = lex.soname_pattern_base self.soname_pattern_abi = lex.soname_pattern_base - elif lex.soname_pattern_patch != '' and lex.soname_pattern_abi != '': + elif lex.soname_pattern_patch != "" and lex.soname_pattern_abi != "": self.soname_pattern_patch = lex.soname_pattern_patch self.soname_pattern_abi = lex.soname_pattern_abi else: @@ -1662,9 +2109,13 @@ def __init__(self, infofile): raise InternalError("Invalid soname_patterns in %s" % (self.infofile)) else: if lex.soname_suffix: - self.soname_pattern_base = "{lib_prefix}{libname}.%s" % (lex.soname_suffix) + self.soname_pattern_base = "{lib_prefix}{libname}.%s" % ( + lex.soname_suffix + ) self.soname_pattern_abi = self.soname_pattern_base + ".{abi_rev}" - self.soname_pattern_patch = self.soname_pattern_abi + ".{version_minor}.{version_patch}" + self.soname_pattern_patch = ( + self.soname_pattern_abi + ".{version_minor}.{version_patch}" + ) else: # Could not calculate soname_pattern_* # This happens for OSs without shared library support (e.g. nacl, mingw, includeos, cygwin) @@ -1689,16 +2140,16 @@ def __init__(self, infofile): self.static_suffix = lex.static_suffix self.target_features = lex.target_features self.use_stack_protector = lex.use_stack_protector == "true" - self.shared_lib_uses_symlinks = lex.shared_lib_symlinks == 'yes' + self.shared_lib_uses_symlinks = lex.shared_lib_symlinks == "yes" self.default_compiler = lex.default_compiler - self.uses_pkg_config = lex.uses_pkg_config == 'yes' + self.uses_pkg_config = lex.uses_pkg_config == "yes" self.feature_macros = lex.feature_macros self._validate_os_features(self.target_features, infofile) @staticmethod def _validate_os_features(features, infofile): - feature_re = re.compile('^[a-z][a-z0-9_]*[a-z0-9]$') + feature_re = re.compile("^[a-z][a-z0-9_]*[a-z0-9]$") for feature in features: if not feature_re.match(feature): logging.error("Invalid OS feature %s in %s", feature, infofile) @@ -1727,25 +2178,28 @@ def enabled_features(self, options): return sorted(feats) def enabled_features_public(self, options): - public_feat = set(['threads', 'filesystem']) + public_feat = set(["threads", "filesystem"]) return sorted(list(set(self.enabled_features(options)) & public_feat)) def enabled_features_internal(self, options): - public_feat = set(['threads', 'filesystem']) + public_feat = set(["threads", "filesystem"]) return sorted(list(set(self.enabled_features(options)) - public_feat)) def macros(self, cc): - value = [cc.add_compile_definition_option + define - for define in self.feature_macros] + value = [ + cc.add_compile_definition_option + define for define in self.feature_macros + ] + + return " ".join(value) - return ' '.join(value) def fixup_proc_name(proc): - proc = proc.lower().replace(' ', '') - for junk in ['(tm)', '(r)']: - proc = proc.replace(junk, '') + proc = proc.lower().replace(" ", "") + for junk in ["(tm)", "(r)"]: + proc = proc.replace(junk, "") return proc + def canon_processor(archinfo, proc): proc = fixup_proc_name(proc) @@ -1756,21 +2210,22 @@ def canon_processor(archinfo, proc): return None -def system_cpu_info(): +def system_cpu_info(): cpu_info = [] - if platform.machine() != '': + if platform.machine() != "": cpu_info.append(platform.machine()) - if platform.processor() != '': + if platform.processor() != "": cpu_info.append(platform.processor()) - if 'uname' in os.__dict__: + if "uname" in os.__dict__: cpu_info.append(os.uname()[4]) return cpu_info + def guess_processor(archinfo): for info_part in system_cpu_info(): if info_part: @@ -1781,7 +2236,7 @@ def guess_processor(archinfo): else: logging.debug("Failed to deduce CPU from '%s'", info_part) - raise UserError('Could not determine target CPU; set with --cpu') + raise UserError("Could not determine target CPU; set with --cpu") def read_textfile(filepath): @@ -1789,10 +2244,10 @@ def read_textfile(filepath): Read a whole file into memory as a string """ if filepath is None: - return '' + return "" - with open(filepath, encoding='utf8') as f: - return ''.join(f.readlines()) + with open(filepath, encoding="utf8") as f: + return "".join(f.readlines()) def process_template_string(template_text, variables, template_source): @@ -1801,15 +2256,15 @@ def process_template_string(template_text, variables, template_source): The template language supports (un-nested) conditionals. """ - class SimpleTemplate: + class SimpleTemplate: def __init__(self, vals): self.vals = vals - self.value_pattern = re.compile(r'%{([a-z][a-z_0-9\|]+)(?::([^}]+))?}') - self.cond_pattern = re.compile('%{(if|unless) ([a-z][a-z_0-9]+)}') - self.for_pattern = re.compile('(.*)%{for ([a-z][a-z_0-9]+)}') - self.omitlast_pattern = re.compile('(.*)%{omitlast ([^}]*)}(.*)', re.DOTALL) - self.join_pattern = re.compile('%{join ([a-z][a-z_0-9]+)}') + self.value_pattern = re.compile(r"%{([a-z][a-z_0-9\|]+)(?::([^}]+))?}") + self.cond_pattern = re.compile("%{(if|unless) ([a-z][a-z_0-9]+)}") + self.for_pattern = re.compile("(.*)%{for ([a-z][a-z_0-9]+)}") + self.omitlast_pattern = re.compile("(.*)%{omitlast ([^}]*)}(.*)", re.DOTALL) + self.join_pattern = re.compile("%{join ([a-z][a-z_0-9]+)}") def substitute(self, template): def get_replacement(k): @@ -1819,20 +2274,22 @@ def get_replacement(k): def insert_value(match): k = match.group(1) - if k.endswith('|upper'): - k = k.replace('|upper', '') + if k.endswith("|upper"): + k = k.replace("|upper", "") return get_replacement(k).upper() - elif k.endswith('|concat'): - k = k.replace('|concat', '') + elif k.endswith("|concat"): + k = k.replace("|concat", "") if not match.group(2): - raise InternalError("|concat must be of the form '%{val|concat:}'") + raise InternalError( + "|concat must be of the form '%{val|concat:}'" + ) v = get_replacement(k) if v: return f"{v}{match.group(2)}" else: return v - elif k.endswith('|as_bool'): - k = k.replace('|as_bool', '') + elif k.endswith("|as_bool"): + k = k.replace("|as_bool", "") if k not in self.vals: raise KeyError(k) @@ -1845,7 +2302,7 @@ def insert_value(match): def insert_join(match): var = match.group(1) if var in self.vals: - return ' '.join(self.vals.get(var)) + return " ".join(self.vals.get(var)) raise KeyError(var) lines = template.splitlines() @@ -1864,14 +2321,20 @@ def insert_join(match): include_cond = False - if cond_type == 'if' and cond_var in self.vals and self.vals.get(cond_var): + if ( + cond_type == "if" + and cond_var in self.vals + and self.vals.get(cond_var) + ): include_cond = True - elif cond_type == 'unless' and (cond_var not in self.vals or (not self.vals.get(cond_var))): + elif cond_type == "unless" and ( + cond_var not in self.vals or (not self.vals.get(cond_var)) + ): include_cond = True idx += 1 while idx < len(lines): - if lines[idx] == '%{endif}': + if lines[idx] == "%{endif}": break if include_cond: output += lines[idx] + "\n" @@ -1882,16 +2345,20 @@ def insert_join(match): for_var = for_match.group(2) if for_var not in self.vals: - raise InternalError("Unknown for loop iteration variable '%s'" % (for_var)) + raise InternalError( + "Unknown for loop iteration variable '%s'" % (for_var) + ) var = self.vals[for_var] if not isinstance(var, list): - raise InternalError("For loop iteration variable '%s' is not a list" % (for_var)) + raise InternalError( + "For loop iteration variable '%s' is not a list" % (for_var) + ) idx += 1 for_body = "" while idx < len(lines): - if lines[idx] == '%{endfor}': + if lines[idx] == "%{endfor}": break for_body += lines[idx] + "\n" idx += 1 @@ -1900,12 +2367,14 @@ def insert_join(match): if isinstance(v, dict): for_val = for_body for ik, iv in v.items(): - for_val = for_val.replace('%{' + ik + '}', iv) + for_val = for_val.replace("%{" + ik + "}", iv) output += for_val + "\n" else: - output += for_body.replace('%{i}', v).replace('%{i|upper}', v.upper()) + output += for_body.replace("%{i}", v).replace( + "%{i|upper}", v.upper() + ) - if output.find('%{omitlast') >= 0: + if output.find("%{omitlast") >= 0: omitlast_match = self.omitlast_pattern.match(output) if omitlast_match: output = omitlast_match.group(1) @@ -1918,30 +2387,38 @@ def insert_join(match): output += lines[idx] + "\n" idx += 1 - output = self.join_pattern.sub(insert_join, self.value_pattern.sub(insert_value, output)) + output = self.join_pattern.sub( + insert_join, self.value_pattern.sub(insert_value, output) + ) # Prevent newlines being added if the template was not a multiline string - if len(lines) == 1 and not template.endswith('\n'): - return output.rstrip('\n') + if len(lines) == 1 and not template.endswith("\n"): + return output.rstrip("\n") return output try: return SimpleTemplate(variables).substitute(template_text) except KeyError as ex: - logging.error('Unbound var %s in template %s', ex, template_source) + logging.error("Unbound var %s in template %s", ex, template_source) except Exception as ex: - logging.error('Exception %s during template processing file %s', ex, template_source) + logging.error( + "Exception %s during template processing file %s", ex, template_source + ) + def process_template(template_file, variables): - return process_template_string(read_textfile(template_file), variables, template_file) + return process_template_string( + read_textfile(template_file), variables, template_file + ) + def yield_objectfile_list(sources, obj_dir, obj_suffix, options): - obj_suffix = '.' + obj_suffix + obj_suffix = "." + obj_suffix for src in sources: (directory, filename) = os.path.split(os.path.normpath(src)) - parts_in_src = directory.split('src' + os.sep) + parts_in_src = directory.split("src" + os.sep) parts = [] if len(parts_in_src) > 1: @@ -1951,10 +2428,10 @@ def yield_objectfile_list(sources, obj_dir, obj_suffix, options): if parts != []: # Handle src/X/X.cpp -> X.o - if filename == parts[-1] + '.cpp': - name = '_'.join(parts) + '.cpp' + if filename == parts[-1] + ".cpp": + name = "_".join(parts) + ".cpp" else: - name = '_'.join(parts) + '_' + filename + name = "_".join(parts) + "_" + filename def fixup_obj_name(name): def remove_dups(parts): @@ -1964,15 +2441,16 @@ def remove_dups(parts): last = part yield part - return '_'.join(remove_dups(name.split('_'))) + return "_".join(remove_dups(name.split("_"))) name = fixup_obj_name(name) else: name = filename - name = name.replace('.cpp', obj_suffix) + name = name.replace(".cpp", obj_suffix) yield normalize_source_path(os.path.join(obj_dir, name)) + def generate_build_info(build_paths, modules, cc, arch, osinfo, options): # first create a map of src_file->owning module @@ -1983,36 +2461,38 @@ def generate_build_info(build_paths, modules, cc, arch, osinfo, options): module_that_owns[src] = mod def _isa_specific_flags(src): - if os.path.basename(src) == 'test_simd.cpp': - return cc.get_isa_specific_flags(['simd'], arch, options) + if os.path.basename(src) == "test_simd.cpp": + return cc.get_isa_specific_flags(["simd"], arch, options) if src in module_that_owns: module = module_that_owns[src] isas = module.isas_needed(arch.basename) - if 'simd_4x32' in module.dependencies(osinfo, arch): - isas.append('simd') + if "simd_4x32" in module.dependencies(osinfo, arch): + isas.append("simd") return cc.get_isa_specific_flags(isas, arch, options) - return '' + return "" def _build_info(sources, objects, target_type): output = [] - for (obj_file, src) in zip(objects, sources): - info = { - 'src': src, - 'obj': obj_file, - 'isa_flags': _isa_specific_flags(src) - } - - if target_type in ['fuzzer', 'examples']: - exe_basename = os.path.basename(obj_file).replace('.' + osinfo.obj_suffix, osinfo.program_suffix) - info['exe_basename'] = exe_basename - - if target_type == 'fuzzer': - info['exe'] = os.path.join(build_paths.fuzzer_output_dir, exe_basename) + for obj_file, src in zip(objects, sources): + info = {"src": src, "obj": obj_file, "isa_flags": _isa_specific_flags(src)} + + if target_type in ["fuzzer", "examples"]: + exe_basename = os.path.basename(obj_file).replace( + "." + osinfo.obj_suffix, osinfo.program_suffix + ) + info["exe_basename"] = exe_basename + + if target_type == "fuzzer": + info["exe"] = os.path.join( + build_paths.fuzzer_output_dir, exe_basename + ) else: - info['exe'] = os.path.join(build_paths.example_output_dir, exe_basename) + info["exe"] = os.path.join( + build_paths.example_output_dir, exe_basename + ) output.append(info) @@ -2020,9 +2500,9 @@ def _build_info(sources, objects, target_type): out = {} - targets = ['lib', 'cli', 'test', 'fuzzer', 'examples'] + targets = ["lib", "cli", "test", "fuzzer", "examples"] - out['isa_build_info'] = [] + out["isa_build_info"] = [] fuzzer_bin = [] example_bin = [] @@ -2030,61 +2510,68 @@ def _build_info(sources, objects, target_type): for t in targets: src_list, src_dir = build_paths.src_info(t) - src_key = '%s_srcs' % (t) - obj_key = '%s_objs' % (t) - build_key = '%s_build_info' % (t) + src_key = "%s_srcs" % (t) + obj_key = "%s_objs" % (t) + build_key = "%s_build_info" % (t) objects = [] build_info = [] if src_list is not None: src_list.sort() - objects = list(yield_objectfile_list(src_list, src_dir, osinfo.obj_suffix, options)) + objects = list( + yield_objectfile_list(src_list, src_dir, osinfo.obj_suffix, options) + ) build_info = _build_info(src_list, objects, t) for b in build_info: - if b['isa_flags'] != '': - out['isa_build_info'].append(b) + if b["isa_flags"] != "": + out["isa_build_info"].append(b) - if t == 'fuzzer': - fuzzer_bin = [b['exe'] for b in build_info] - elif t == 'examples': - example_bin = [b['exe'] for b in build_info] + if t == "fuzzer": + fuzzer_bin = [b["exe"] for b in build_info] + elif t == "examples": + example_bin = [b["exe"] for b in build_info] out[src_key] = src_list if src_list else [] out[obj_key] = objects out[build_key] = build_info - out['fuzzer_bin'] = ' '.join(fuzzer_bin) - out['example_bin'] = ' '.join(example_bin) - out['cli_headers'] = build_paths.cli_headers + out["fuzzer_bin"] = " ".join(fuzzer_bin) + out["example_bin"] = " ".join(example_bin) + out["cli_headers"] = build_paths.cli_headers return out -def create_template_vars(source_paths, build_paths, options, modules, disabled_modules, cc, arch, osinfo): + +def create_template_vars( + source_paths, build_paths, options, modules, disabled_modules, cc, arch, osinfo +): """ Create the template variables needed to process the makefile, build.h, etc """ def external_link_cmd(): - return ' '.join([cc.add_lib_dir_option + libdir for libdir in options.with_external_libdir]) + return " ".join( + [cc.add_lib_dir_option + libdir for libdir in options.with_external_libdir] + ) def link_to(module_member_name): """ Figure out what external libraries/frameworks are needed based on selected modules """ - if module_member_name not in ['libs', 'frameworks']: + if module_member_name not in ["libs", "frameworks"]: raise InternalError("Invalid argument") libs = set() for module in modules: - for (osname, module_link_to) in getattr(module, module_member_name).items(): - if osname in ['all', osinfo.basename]: + for osname, module_link_to in getattr(module, module_member_name).items(): + if osname in ["all", osinfo.basename]: libs |= set(module_link_to) else: - match = re.match('^all!(.*)', osname) + match = re.match("^all!(.*)", osname) if match is not None: - exceptions = match.group(1).split(',') + exceptions = match.group(1).split(",") if osinfo.basename not in exceptions: libs |= set(module_link_to) @@ -2094,13 +2581,15 @@ def configure_command_line(): # Cut absolute path from main executable (e.g. configure.py or python interpreter) # to get the same result when configuring the same thing on different machines main_executable = os.path.basename(sys.argv[0]) - quoted_args = [arg if ' ' not in arg else '\'' + arg + '\'' for arg in sys.argv[1:]] - return ' '.join([main_executable] + quoted_args) + quoted_args = [ + arg if " " not in arg else "'" + arg + "'" for arg in sys.argv[1:] + ] + return " ".join([main_executable] + quoted_args) def sysroot_option(): - if options.with_sysroot_dir == '': - return '' - if cc.add_sysroot_option == '': + if options.with_sysroot_dir == "": + return "" + if cc.add_sysroot_option == "": logging.error("This compiler doesn't support --sysroot option") return cc.add_sysroot_option + options.with_sysroot_dir @@ -2127,33 +2616,33 @@ def join_with_build_dir(path): # `normalize_source_path` will "fix" the path slashes but remove # a redundant `./` for the "trivial" relative path. normalized = normalize_source_path(os.path.join(build_dir, path)) - if build_dir == '.': - normalized = './%s' % normalized + if build_dir == ".": + normalized = "./%s" % normalized return normalized def all_targets(options): - yield 'libs' + yield "libs" if options.with_documentation: - yield 'docs' - if 'cli' in options.build_targets: - yield 'cli' - if 'tests' in options.build_targets: - yield 'tests' + yield "docs" + if "cli" in options.build_targets: + yield "cli" + if "tests" in options.build_targets: + yield "tests" if options.build_fuzzers: - yield 'fuzzers' - if 'bogo_shim' in options.build_targets: - yield 'bogo_shim' - if 'examples' in options.build_targets: - yield 'examples' - if 'ct_selftest' in options.build_targets: - yield 'ct_selftest' + yield "fuzzers" + if "bogo_shim" in options.build_targets: + yield "bogo_shim" + if "examples" in options.build_targets: + yield "examples" + if "ct_selftest" in options.build_targets: + yield "ct_selftest" def install_targets(options): - yield 'libs' - if 'cli' in options.build_targets: - yield 'cli' + yield "libs" + if "cli" in options.build_targets: + yield "cli" if options.with_documentation: - yield 'docs' + yield "docs" def absolute_install_dir(p): if os.path.isabs(p): @@ -2163,228 +2652,215 @@ def absolute_install_dir(p): def choose_python_exe(): return normalize_source_path(sys.executable) - def choose_cxx_exe(with_compiler_cache = False): + def choose_cxx_exe(with_compiler_cache=False): cxx = options.compiler_binary or cc.binary_name if options.compiler_cache is None or with_compiler_cache is False: return cxx else: - return '%s %s' % (normalize_source_path(options.compiler_cache), cxx) + return "%s %s" % (normalize_source_path(options.compiler_cache), cxx) def extra_libs(libs, cc): if libs is None: - return '' + return "" - return ' '.join([(cc.add_lib_option % lib) for lib in libs.split(',') if lib != '']) + return " ".join( + [(cc.add_lib_option % lib) for lib in libs.split(",") if lib != ""] + ) def test_exe_extra_ldflags(): if osinfo.matches_name("emscripten"): - return '--preload-file=%s@src/tests/data' % source_paths.test_data_dir + return "--preload-file=%s@src/tests/data" % source_paths.test_data_dir - return '' + return "" variables = { - 'version_major': Version.major(), - 'version_minor': Version.minor(), - 'version_patch': Version.patch(), - 'version_vc_rev': None if Version.vc_rev() == 'unknown' else Version.vc_rev(), - - 'version_vc_rev_or_unknown': 'unknown' if Version.datestamp() == 0 else Version.vc_rev(), - - 'abi_rev': Version.so_rev(), - - 'version': Version.as_string(), - 'release_type': Version.release_type(), - 'version_datestamp': Version.datestamp(), - - 'distribution_info': options.distribution_info, - 'distribution_info_or_unspecified': options.distribution_info or 'unspecified', - - 'full_version_string': Version.full_version_string(options), - 'short_version_string': Version.short_version_string(), - - 'macos_so_compat_ver': '%s.%s.0' % (Version.packed(), Version.so_rev()), - 'macos_so_current_ver': '%s.%s.%s' % (Version.packed(), Version.so_rev(), Version.patch()), - - 'all_targets': ' '.join(all_targets(options)), - 'install_targets': ' '.join(install_targets(options)), - - 'public_headers': sorted([os.path.basename(h) for h in build_paths.public_headers]), - 'internal_headers': sorted([os.path.basename(h) for h in build_paths.internal_headers]), - 'external_headers': sorted([os.path.basename(h) for h in build_paths.external_headers]), - - 'abs_root_dir': normalize_source_path(os.path.dirname(os.path.realpath(__file__))), - - 'base_dir': source_paths.base_dir, - 'src_dir': source_paths.src_dir, - 'test_data_dir': source_paths.test_data_dir, - 'doc_dir': source_paths.doc_dir, - 'scripts_dir': normalize_source_path(source_paths.scripts_dir), - 'python_dir': source_paths.python_dir, - - 'cli_exe_name': osinfo.cli_exe_name + program_suffix, - 'cli_exe': join_with_build_dir(osinfo.cli_exe_name + program_suffix), - 'build_cli_exe': bool('cli' in options.build_targets), - 'test_exe': join_with_build_dir('botan-test' + program_suffix), - - 'lib_prefix': osinfo.lib_prefix, - 'static_suffix': osinfo.static_suffix, - 'lib_suffix': options.library_suffix, - 'libname': osinfo.library_name.format(major=Version.major(), - minor=Version.minor(), - suffix=options.library_suffix), - - 'command_line': configure_command_line(), - - 'program_suffix': program_suffix, - - 'prefix': options.prefix or osinfo.install_root, - 'bindir': absolute_install_dir(options.bindir or osinfo.bin_dir), - 'libdir': absolute_install_dir(options.libdir or osinfo.lib_dir), - 'mandir': options.mandir or osinfo.man_dir, - 'includedir': options.includedir or osinfo.header_dir, - 'docdir': options.docdir or osinfo.doc_dir, - - 'with_documentation': options.with_documentation, - 'with_sphinx': options.with_sphinx, - 'with_pdf': options.with_pdf, - 'with_rst2man': options.with_rst2man, - 'sphinx_config_dir': source_paths.sphinx_config_dir, - 'with_doxygen': options.with_doxygen, - 'maintainer_mode': options.maintainer_mode, - - 'out_dir': normalize_source_path(build_dir), - 'build_dir': normalize_source_path(build_paths.build_dir), - 'module_info_dir': build_paths.doc_module_info, - - 'doc_stamp_file': normalize_source_path(os.path.join(build_paths.build_dir, 'doc.stamp')), - 'makefile_path': os.path.join(build_paths.build_dir, '..', 'Makefile'), - 'ninja_build_path': os.path.join(build_paths.build_dir, '..', 'build.ninja'), - + "version_major": Version.major(), + "version_minor": Version.minor(), + "version_patch": Version.patch(), + "version_vc_rev": None if Version.vc_rev() == "unknown" else Version.vc_rev(), + "version_vc_rev_or_unknown": ( + "unknown" if Version.datestamp() == 0 else Version.vc_rev() + ), + "abi_rev": Version.so_rev(), + "version": Version.as_string(), + "release_type": Version.release_type(), + "version_datestamp": Version.datestamp(), + "distribution_info": options.distribution_info, + "distribution_info_or_unspecified": options.distribution_info or "unspecified", + "full_version_string": Version.full_version_string(options), + "short_version_string": Version.short_version_string(), + "macos_so_compat_ver": "%s.%s.0" % (Version.packed(), Version.so_rev()), + "macos_so_current_ver": "%s.%s.%s" + % (Version.packed(), Version.so_rev(), Version.patch()), + "all_targets": " ".join(all_targets(options)), + "install_targets": " ".join(install_targets(options)), + "public_headers": sorted( + [os.path.basename(h) for h in build_paths.public_headers] + ), + "internal_headers": sorted( + [os.path.basename(h) for h in build_paths.internal_headers] + ), + "external_headers": sorted( + [os.path.basename(h) for h in build_paths.external_headers] + ), + "abs_root_dir": normalize_source_path( + os.path.dirname(os.path.realpath(__file__)) + ), + "base_dir": source_paths.base_dir, + "src_dir": source_paths.src_dir, + "test_data_dir": source_paths.test_data_dir, + "doc_dir": source_paths.doc_dir, + "scripts_dir": normalize_source_path(source_paths.scripts_dir), + "python_dir": source_paths.python_dir, + "cli_exe_name": osinfo.cli_exe_name + program_suffix, + "cli_exe": join_with_build_dir(osinfo.cli_exe_name + program_suffix), + "build_cli_exe": bool("cli" in options.build_targets), + "test_exe": join_with_build_dir("botan-test" + program_suffix), + "lib_prefix": osinfo.lib_prefix, + "static_suffix": osinfo.static_suffix, + "lib_suffix": options.library_suffix, + "libname": osinfo.library_name.format( + major=Version.major(), minor=Version.minor(), suffix=options.library_suffix + ), + "command_line": configure_command_line(), + "program_suffix": program_suffix, + "prefix": options.prefix or osinfo.install_root, + "bindir": absolute_install_dir(options.bindir or osinfo.bin_dir), + "libdir": absolute_install_dir(options.libdir or osinfo.lib_dir), + "mandir": options.mandir or osinfo.man_dir, + "includedir": options.includedir or osinfo.header_dir, + "docdir": options.docdir or osinfo.doc_dir, + "with_documentation": options.with_documentation, + "with_sphinx": options.with_sphinx, + "with_pdf": options.with_pdf, + "with_rst2man": options.with_rst2man, + "sphinx_config_dir": source_paths.sphinx_config_dir, + "with_doxygen": options.with_doxygen, + "maintainer_mode": options.maintainer_mode, + "out_dir": normalize_source_path(build_dir), + "build_dir": normalize_source_path(build_paths.build_dir), + "module_info_dir": build_paths.doc_module_info, + "doc_stamp_file": normalize_source_path( + os.path.join(build_paths.build_dir, "doc.stamp") + ), + "makefile_path": os.path.join(build_paths.build_dir, "..", "Makefile"), + "ninja_build_path": os.path.join(build_paths.build_dir, "..", "build.ninja"), # Use response files for the archive command on windows # Note: macOS (and perhaps other OSes) do not support this - 'build_static_lib_using_cmdline_args': options.build_static_lib and osinfo.basename != 'windows', - 'build_static_lib_using_response_file': options.build_static_lib and osinfo.basename == 'windows', - 'build_static_lib': options.build_static_lib, - 'build_shared_lib': options.build_shared_lib, - - 'build_fuzzers': options.build_fuzzers, - 'build_examples': 'examples' in options.build_targets, - - 'build_coverage' : options.with_coverage_info, - - 'symlink_shared_lib': options.build_shared_lib and osinfo.shared_lib_uses_symlinks, - - 'libobj_dir': build_paths.libobj_dir, - 'cliobj_dir': build_paths.cliobj_dir, - 'testobj_dir': build_paths.testobj_dir, - 'fuzzobj_dir': build_paths.fuzzobj_dir, - - 'fuzzer_output_dir': build_paths.fuzzer_output_dir if build_paths.fuzzer_output_dir else '', - 'doc_output_dir': build_paths.doc_output_dir, - 'handbook_output_dir': build_paths.handbook_output_dir, - 'doc_output_dir_doxygen': build_paths.doc_output_dir_doxygen, - 'response_file_dir': build_paths.response_file_dir, - - 'os': options.os, - 'arch': options.arch, - 'compiler': options.compiler, - 'cpu_family': arch.family, - - 'python_exe': choose_python_exe(), - 'python_version': options.python_version, - 'install_python_module': not options.no_install_python_module, - - 'cxx': choose_cxx_exe(True), - 'cxx_abi_flags': cc.mach_abi_link_flags(options), - 'linker': cc.linker_name or choose_cxx_exe(), - 'make_supports_phony': osinfo.basename != 'windows', - - 'cxx_supports_gcc_inline_asm': cc.supports_gcc_inline_asm and options.enable_asm, - 'compiler_assisted_stack_scrubbing': options.enable_stack_scrubbing, - - 'cxx_ct_value_barrier_type': cc.ct_value_barrier_type(options), - - 'sanitizer_types' : sorted(cc.sanitizer_types), - - 'dash_o': cc.output_to_object, - 'dash_c': cc.compile_flags, - - 'cc_lang_flags': cc.cc_lang_flags(), - 'cc_lang_binary_linker_flags': cc.cc_lang_binary_linker_flags(), - 'os_feature_macros': osinfo.macros(cc), - 'cc_sysroot': sysroot_option(), - 'cc_compile_flags': ' '.join(cc.cc_compile_flags(options)).strip(), - 'ldflags': ' '.join(cc.ldflags(options)).strip(), - 'test_exe_extra_ldflags': test_exe_extra_ldflags(), - 'extra_libs': extra_libs(options.extra_libs, cc), - 'cc_warning_flags': cc.cc_warning_flags(options), - 'output_to_exe': cc.output_to_exe, - 'cc_macro': cc.macro_name, - 'ninja_header_deps_style': cc.ninja_header_deps_style, - 'header_deps_flag': cc.header_deps_flag, - 'header_deps_out': cc.header_deps_out, - - 'visibility_attribute': cc.gen_visibility_attribute(options), - - 'lib_link_cmd': cc.so_link_command_for(osinfo.basename, options), - 'exe_link_cmd': cc.binary_link_command_for(osinfo.basename, options), - 'external_link_cmd': external_link_cmd(), - - 'ar_command': ar_command(), - 'ar_options': options.ar_options or cc.ar_options or osinfo.ar_options, - 'ar_output_to': cc.ar_output_to, - - 'link_to': ' '.join( - [(cc.add_lib_option % lib) for lib in link_to('libs')] + - [cc.add_framework_option + fw for fw in link_to('frameworks')] + "build_static_lib_using_cmdline_args": options.build_static_lib + and osinfo.basename != "windows", + "build_static_lib_using_response_file": options.build_static_lib + and osinfo.basename == "windows", + "build_static_lib": options.build_static_lib, + "build_shared_lib": options.build_shared_lib, + "build_fuzzers": options.build_fuzzers, + "build_examples": "examples" in options.build_targets, + "build_coverage": options.with_coverage_info, + "symlink_shared_lib": options.build_shared_lib + and osinfo.shared_lib_uses_symlinks, + "libobj_dir": build_paths.libobj_dir, + "cliobj_dir": build_paths.cliobj_dir, + "testobj_dir": build_paths.testobj_dir, + "fuzzobj_dir": build_paths.fuzzobj_dir, + "fuzzer_output_dir": ( + build_paths.fuzzer_output_dir if build_paths.fuzzer_output_dir else "" ), - - 'fuzzer_lib': (cc.add_lib_option % options.fuzzer_lib) if options.fuzzer_lib else '', - 'libs_used': [lib.replace('.lib', '') for lib in link_to('libs')], - - 'public_include_path': build_paths.public_include_dir, - 'internal_include_path': build_paths.internal_include_dir, - 'external_include_path': build_paths.external_include_dir, - - 'public_include_flags': build_paths.format_public_include_flags(cc), - 'internal_include_flags': build_paths.format_internal_include_flags(cc), - 'external_include_flags': build_paths.format_external_include_flags(cc, options.with_external_includedir), - 'module_defines': sorted(flatten([m.defines() for m in modules])), - 'module_internal_defines': sorted(flatten([m.internal_defines() for m in modules])), - - 'build_bogo_shim': bool('bogo_shim' in options.build_targets), - 'bogo_shim_src': os.path.join(source_paths.src_dir, 'bogo_shim', 'bogo_shim.cpp'), - - 'build_ct_selftest': bool('ct_selftest' in options.build_targets), - 'ct_selftest_src': os.path.join(source_paths.src_dir, 'ct_selftest', 'ct_selftest.cpp'), - - 'os_features': osinfo.enabled_features_internal(options), - 'os_features_public': osinfo.enabled_features_public(options), - 'os_name': osinfo.basename, - 'cpu_features': arch.supported_isa_extensions(cc, options), - 'system_cert_bundle': options.system_cert_bundle, - - 'enable_experimental_features': options.enable_experimental_features, - 'disable_deprecated_features': not options.enable_deprecated_features, - - 'fuzzer_mode': options.unsafe_fuzzer_mode, - 'building_fuzzers': options.build_fuzzers, - 'fuzzer_type': options.build_fuzzers.upper() if options.build_fuzzers else '', - - 'with_valgrind': options.with_valgrind, - 'with_debug_asserts': options.with_debug_asserts, - 'terminate_on_asserts': options.unsafe_terminate_on_asserts, - 'optimize_for_size': options.optimize_for_size, - - 'mod_list': sorted([m.basename for m in modules]), - 'disabled_mod_list': sorted([m.basename for m in disabled_modules]), + "doc_output_dir": build_paths.doc_output_dir, + "handbook_output_dir": build_paths.handbook_output_dir, + "doc_output_dir_doxygen": build_paths.doc_output_dir_doxygen, + "response_file_dir": build_paths.response_file_dir, + "os": options.os, + "arch": options.arch, + "compiler": options.compiler, + "cpu_family": arch.family, + "python_exe": choose_python_exe(), + "python_version": options.python_version, + "install_python_module": not options.no_install_python_module, + "cxx": choose_cxx_exe(True), + "cxx_abi_flags": cc.mach_abi_link_flags(options), + "linker": cc.linker_name or choose_cxx_exe(), + "make_supports_phony": osinfo.basename != "windows", + "cxx_supports_gcc_inline_asm": cc.supports_gcc_inline_asm + and options.enable_asm, + "compiler_assisted_stack_scrubbing": options.enable_stack_scrubbing, + "cxx_ct_value_barrier_type": cc.ct_value_barrier_type(options), + "sanitizer_types": sorted(cc.sanitizer_types), + "dash_o": cc.output_to_object, + "dash_c": cc.compile_flags, + "cc_lang_flags": cc.cc_lang_flags(), + "cc_lang_binary_linker_flags": cc.cc_lang_binary_linker_flags(), + "os_feature_macros": osinfo.macros(cc), + "cc_sysroot": sysroot_option(), + "cc_compile_flags": " ".join(cc.cc_compile_flags(options)).strip(), + "ldflags": " ".join(cc.ldflags(options)).strip(), + "test_exe_extra_ldflags": test_exe_extra_ldflags(), + "extra_libs": extra_libs(options.extra_libs, cc), + "cc_warning_flags": cc.cc_warning_flags(options), + "output_to_exe": cc.output_to_exe, + "cc_macro": cc.macro_name, + "ninja_header_deps_style": cc.ninja_header_deps_style, + "header_deps_flag": cc.header_deps_flag, + "header_deps_out": cc.header_deps_out, + "visibility_attribute": cc.gen_visibility_attribute(options), + "lib_link_cmd": cc.so_link_command_for(osinfo.basename, options), + "exe_link_cmd": cc.binary_link_command_for(osinfo.basename, options), + "external_link_cmd": external_link_cmd(), + "ar_command": ar_command(), + "ar_options": options.ar_options or cc.ar_options or osinfo.ar_options, + "ar_output_to": cc.ar_output_to, + "link_to": " ".join( + [(cc.add_lib_option % lib) for lib in link_to("libs")] + + [cc.add_framework_option + fw for fw in link_to("frameworks")] + ), + "fuzzer_lib": ( + (cc.add_lib_option % options.fuzzer_lib) if options.fuzzer_lib else "" + ), + "libs_used": [lib.replace(".lib", "") for lib in link_to("libs")], + "public_include_path": build_paths.public_include_dir, + "internal_include_path": build_paths.internal_include_dir, + "external_include_path": build_paths.external_include_dir, + "public_include_flags": build_paths.format_public_include_flags(cc), + "internal_include_flags": build_paths.format_internal_include_flags(cc), + "external_include_flags": build_paths.format_external_include_flags( + cc, options.with_external_includedir + ), + "module_defines": sorted(flatten([m.defines() for m in modules])), + "module_internal_defines": sorted( + flatten([m.internal_defines() for m in modules]) + ), + "build_bogo_shim": bool("bogo_shim" in options.build_targets), + "bogo_shim_src": os.path.join( + source_paths.src_dir, "bogo_shim", "bogo_shim.cpp" + ), + "build_ct_selftest": bool("ct_selftest" in options.build_targets), + "ct_selftest_src": os.path.join( + source_paths.src_dir, "ct_selftest", "ct_selftest.cpp" + ), + "os_features": osinfo.enabled_features_internal(options), + "os_features_public": osinfo.enabled_features_public(options), + "os_name": osinfo.basename, + "cpu_features": arch.supported_isa_extensions(cc, options), + "system_cert_bundle": options.system_cert_bundle, + "enable_experimental_features": options.enable_experimental_features, + "disable_deprecated_features": not options.enable_deprecated_features, + "fuzzer_mode": options.unsafe_fuzzer_mode, + "building_fuzzers": options.build_fuzzers, + "fuzzer_type": options.build_fuzzers.upper() if options.build_fuzzers else "", + "with_valgrind": options.with_valgrind, + "with_debug_asserts": options.with_debug_asserts, + "terminate_on_asserts": options.unsafe_terminate_on_asserts, + "optimize_for_size": options.optimize_for_size, + "mod_list": sorted([m.basename for m in modules]), + "disabled_mod_list": sorted([m.basename for m in disabled_modules]), } - variables['installed_include_dir'] = os.path.join( - variables['prefix'], - variables['includedir'], - 'botan-%d' % (Version.major()), 'botan') + variables["installed_include_dir"] = os.path.join( + variables["prefix"], + variables["includedir"], + "botan-%d" % (Version.major()), + "botan", + ) # On MSVC, the "ABI flags" should be passed to the compiler only, on other platforms, the # ABI flags are passed to both the compiler and the linker and the compiler flags are also @@ -2394,70 +2870,92 @@ def test_exe_extra_ldflags(): # different configurations, then /MD etc could be specified there rather than hijacking the ABI # flags for it and having to special-case their exclusion from the linker command line. - if cc.basename in ('msvc', 'clangcl'): + if cc.basename in ("msvc", "clangcl"): # Move the "ABI flags" (/MD etc) into the compiler flags to exclude it from linker invocations - variables['cc_compile_flags'] = '%s %s' % (variables['cxx_abi_flags'], variables['cc_compile_flags']) - variables['cxx_abi_flags'] = '' + variables["cc_compile_flags"] = "%s %s" % ( + variables["cxx_abi_flags"], + variables["cc_compile_flags"], + ) + variables["cxx_abi_flags"] = "" else: # Append the compiler flags to the linker flags - variables['ldflags'] = '%s %s' % (variables['ldflags'], variables['cc_compile_flags']) + variables["ldflags"] = "%s %s" % ( + variables["ldflags"], + variables["cc_compile_flags"], + ) - variables['lib_flags'] = cc.gen_lib_flags(options, variables) + variables["lib_flags"] = cc.gen_lib_flags(options, variables) if options.with_pkg_config: - variables['botan_pkgconfig'] = os.path.join(build_paths.build_dir, 'botan-%d.pc' % (Version.major())) + variables["botan_pkgconfig"] = os.path.join( + build_paths.build_dir, "botan-%d.pc" % (Version.major()) + ) if options.with_cmake_config: - variables['botan_cmake_config'] = os.path.join(build_paths.build_dir, 'cmake', 'botan-config.cmake') - variables['botan_cmake_version_config'] = os.path.join(build_paths.build_dir, 'cmake', 'botan-config-version.cmake') + variables["botan_cmake_config"] = os.path.join( + build_paths.build_dir, "cmake", "botan-config.cmake" + ) + variables["botan_cmake_version_config"] = os.path.join( + build_paths.build_dir, "cmake", "botan-config-version.cmake" + ) # The name is always set because Windows build needs it - variables['static_lib_name'] = '%s%s.%s' % (variables['lib_prefix'], variables['libname'], - variables['static_suffix']) + variables["static_lib_name"] = "%s%s.%s" % ( + variables["lib_prefix"], + variables["libname"], + variables["static_suffix"], + ) if options.build_shared_lib: if osinfo.soname_pattern_base is not None: - variables['soname_base'] = osinfo.soname_pattern_base.format(**variables) - variables['shared_lib_name'] = variables['soname_base'] + variables["soname_base"] = osinfo.soname_pattern_base.format(**variables) + variables["shared_lib_name"] = variables["soname_base"] if osinfo.soname_pattern_abi is not None: - variables['soname_abi'] = osinfo.soname_pattern_abi.format(**variables) - variables['shared_lib_name'] = variables['soname_abi'] + variables["soname_abi"] = osinfo.soname_pattern_abi.format(**variables) + variables["shared_lib_name"] = variables["soname_abi"] if osinfo.soname_pattern_patch is not None: - variables['soname_patch'] = osinfo.soname_pattern_patch.format(**variables) + variables["soname_patch"] = osinfo.soname_pattern_patch.format(**variables) - if options.os == 'windows': - variables['implib_name'] = variables['static_lib_name'] + if options.os == "windows": + variables["implib_name"] = variables["static_lib_name"] - variables['lib_link_cmd'] = variables['lib_link_cmd'].format(**variables) + variables["lib_link_cmd"] = variables["lib_link_cmd"].format(**variables) - for var in ['exe_link_cmd']: + for var in ["exe_link_cmd"]: variables[var] = variables[var].format(**variables) lib_targets = [] if options.build_static_lib: - lib_targets.append('static_lib_name') + lib_targets.append("static_lib_name") if options.build_shared_lib: - lib_targets.append('shared_lib_name') + lib_targets.append("shared_lib_name") - variables['library_targets'] = ' '.join([join_with_build_dir(variables[t]) for t in lib_targets]) + variables["library_targets"] = " ".join( + [join_with_build_dir(variables[t]) for t in lib_targets] + ) - if options.os == 'llvm' or options.compiler == 'msvc': + if options.os == "llvm" or options.compiler == "msvc": # llvm-link and msvc require just naming the file directly - variables['build_dir_link_path'] = '' - variables['link_to_botan'] = normalize_source_path(os.path.join(build_dir, variables['static_lib_name'])) + variables["build_dir_link_path"] = "" + variables["link_to_botan"] = normalize_source_path( + os.path.join(build_dir, variables["static_lib_name"]) + ) else: - variables['build_dir_link_path'] = '%s%s' % (cc.add_lib_dir_option, build_dir) - variables['link_to_botan'] = cc.add_lib_option % variables['libname'] + variables["build_dir_link_path"] = "%s%s" % (cc.add_lib_dir_option, build_dir) + variables["link_to_botan"] = cc.add_lib_option % variables["libname"] return variables + class ModulesChooser: """ Determine which modules to load based on options, target, etc """ - def __init__(self, modules, module_policy, archinfo, osinfo, ccinfo, cc_min_version, options): + def __init__( + self, modules, module_policy, archinfo, osinfo, ccinfo, cc_min_version, options + ): self._modules = modules self._module_policy = module_policy self._archinfo = archinfo @@ -2472,26 +2970,45 @@ def __init__(self, modules, module_policy, archinfo, osinfo, ccinfo, cc_min_vers self._not_using_because = collections.defaultdict(set) ModulesChooser._validate_dependencies_exist(self._modules) - self._options.enabled_modules = ModulesChooser._expand_wildcards_in_user_selection(self._modules, self._options.enabled_modules) - self._options.disabled_modules = ModulesChooser._expand_wildcards_in_user_selection(self._modules, self._options.disabled_modules) + self._options.enabled_modules = ( + ModulesChooser._expand_wildcards_in_user_selection( + self._modules, self._options.enabled_modules + ) + ) + self._options.disabled_modules = ( + ModulesChooser._expand_wildcards_in_user_selection( + self._modules, self._options.disabled_modules + ) + ) ModulesChooser._validate_user_selection( - self._modules, self._options.enabled_modules, self._options.disabled_modules) + self._modules, self._options.enabled_modules, self._options.disabled_modules + ) def _check_usable(self, module, modname): if not module.compatible_cpu(self._archinfo, self._options): - self._not_using_because['incompatible CPU'].add(modname) + self._not_using_because["incompatible CPU"].add(modname) return False elif not module.compatible_os(self._osinfo, self._options): - self._not_using_because['incompatible OS'].add(modname) + self._not_using_because["incompatible OS"].add(modname) return False - elif not module.compatible_compiler(self._ccinfo, self._cc_min_version, self._archinfo.basename): - self._not_using_because['incompatible compiler'].add(modname) + elif not module.compatible_compiler( + self._ccinfo, self._cc_min_version, self._archinfo.basename + ): + self._not_using_because["incompatible compiler"].add(modname) return False - elif module.is_deprecated() and not self._options.enable_deprecated_features and modname not in self._options.enabled_modules: - self._not_using_because['deprecated'].add(modname) + elif ( + module.is_deprecated() + and not self._options.enable_deprecated_features + and modname not in self._options.enabled_modules + ): + self._not_using_because["deprecated"].add(modname) return False - elif module.is_experimental() and modname not in self._options.enabled_modules and not self._options.enable_experimental_features: - self._not_using_because['experimental'].add(modname) + elif ( + module.is_experimental() + and modname not in self._options.enabled_modules + and not self._options.enable_experimental_features + ): + self._not_using_because["experimental"].add(modname) return False return True @@ -2502,40 +3019,48 @@ def _remove_virtual_modules(all_modules, modnames): @classmethod def _display_module_information_unused(cls, all_modules, skipped_modules): for reason in sorted(skipped_modules.keys()): - disabled_mods = cls._remove_virtual_modules(all_modules, sorted(skipped_modules[reason])) + disabled_mods = cls._remove_virtual_modules( + all_modules, sorted(skipped_modules[reason]) + ) if disabled_mods: - logging.info('Skipping (%s): %s', reason, ' '.join(disabled_mods)) + logging.info("Skipping (%s): %s", reason, " ".join(disabled_mods)) @classmethod def _display_module_information_to_load(cls, all_modules, modules_to_load): - sorted_modules_to_load = cls._remove_virtual_modules(all_modules, sorted(modules_to_load)) + sorted_modules_to_load = cls._remove_virtual_modules( + all_modules, sorted(modules_to_load) + ) deprecated = [] experimental = [] for modname in sorted_modules_to_load: if all_modules[modname].comment: - logging.info('%s: %s', modname, all_modules[modname].comment) + logging.info("%s: %s", modname, all_modules[modname].comment) if all_modules[modname].warning: - logging.warning('%s: %s', modname, all_modules[modname].warning) - if all_modules[modname].load_on == 'vendor': - logging.info('Enabling use of external dependency %s', modname) + logging.warning("%s: %s", modname, all_modules[modname].warning) + if all_modules[modname].load_on == "vendor": + logging.info("Enabling use of external dependency %s", modname) if all_modules[modname].is_deprecated(): deprecated.append(modname) if all_modules[modname].is_experimental(): experimental.append(modname) if deprecated: - logging.warning('These modules are deprecated and will be removed in a future release (consider disabling with --disable-deprecated-features): %s', - ' '.join(deprecated)) + logging.warning( + "These modules are deprecated and will be removed in a future release (consider disabling with --disable-deprecated-features): %s", + " ".join(deprecated), + ) if experimental: - logging.warning('These modules are experimental and may change or be removed in a future release: %s', - ' '.join(experimental)) + logging.warning( + "These modules are experimental and may change or be removed in a future release: %s", + " ".join(experimental), + ) if sorted_modules_to_load: - logging.info('Loading modules: %s', ' '.join(sorted_modules_to_load)) + logging.info("Loading modules: %s", " ".join(sorted_modules_to_load)) else: - logging.error('This configuration disables every submodule and is invalid') + logging.error("This configuration disables every submodule and is invalid") @staticmethod def _validate_state(used_modules, unused_modules): @@ -2544,7 +3069,8 @@ def _validate_state(used_modules, unused_modules): if intersection: raise InternalError( "Disabled modules (%s) and modules to load have common elements: %s" - % (reason, intersection)) + % (reason, intersection) + ) @staticmethod def _validate_dependencies_exist(modules): @@ -2553,18 +3079,32 @@ def _validate_dependencies_exist(modules): @staticmethod def _expand_wildcards_in_user_selection(modules, user_selected_modules): - valid_module_name_with_wildcard = re.compile(r'^[a-z0-9_*]+$') - public_modules = [modname for modname, modinfo in modules.items() if modinfo.is_public()] + valid_module_name_with_wildcard = re.compile(r"^[a-z0-9_*]+$") + public_modules = [ + modname for modname, modinfo in modules.items() if modinfo.is_public() + ] + def expand(user_selected_module): if not valid_module_name_with_wildcard.match(user_selected_module): - logging.error("Invalid module name with wildcard: %s", user_selected_module) + logging.error( + "Invalid module name with wildcard: %s", user_selected_module + ) return [] - regex_from_wildcards = re.compile("^%s$" % user_selected_module.replace('*', '[a-z0-9_]+')) - matching_modules = [mod for mod in public_modules if regex_from_wildcards.match(mod)] + regex_from_wildcards = re.compile( + "^%s$" % user_selected_module.replace("*", "[a-z0-9_]+") + ) + matching_modules = [ + mod for mod in public_modules if regex_from_wildcards.match(mod) + ] if not matching_modules: - logging.warning("Wildcard '%s' did not match any modules", user_selected_module) + logging.warning( + "Wildcard '%s' did not match any modules", user_selected_module + ) return matching_modules - return flatten([expand(mod) if '*' in mod else [mod] for mod in user_selected_modules]) + + return flatten( + [expand(mod) if "*" in mod else [mod] for mod in user_selected_modules] + ) @staticmethod def _validate_user_selection(modules, enabled_modules, disabled_modules): @@ -2582,32 +3122,41 @@ def _handle_by_module_policy(self, modname, module, usable): if self._module_policy is not None: if modname in self._module_policy.required: if not usable: - logging.error('Module policy requires module %s not usable on this platform', modname) + logging.error( + "Module policy requires module %s not usable on this platform", + modname, + ) elif modname in self._options.disabled_modules: - logging.error('Module %s was disabled but is required by policy', modname) + logging.error( + "Module %s was disabled but is required by policy", modname + ) elif module.is_virtual(): logging.error("Module %s is meant for internal use only", modname) self._to_load.add(modname) return True elif modname in self._module_policy.if_available: if modname in self._options.disabled_modules: - self._not_using_because['disabled by user'].add(modname) + self._not_using_because["disabled by user"].add(modname) elif module.is_virtual(): logging.error("Module %s is meant for internal use only", modname) elif usable: - logging.debug('Enabling optional module %s', modname) + logging.debug("Enabling optional module %s", modname) self._to_load.add(modname) return True elif modname in self._module_policy.prohibited: if modname in self._options.enabled_modules: - logging.error('Module %s was requested but is prohibited by policy', modname) - self._not_using_because['prohibited by module policy'].add(modname) + logging.error( + "Module %s was requested but is prohibited by policy", modname + ) + self._not_using_because["prohibited by module policy"].add(modname) return True return False @staticmethod - def resolve_dependencies(available_modules, dependency_table, module, loaded_modules=None): + def resolve_dependencies( + available_modules, dependency_table, module, loaded_modules=None + ): """ Parameters - available_modules: modules to choose from. Constant. @@ -2625,7 +3174,7 @@ def resolve_dependencies(available_modules, dependency_table, module, loaded_mod loaded_modules.add(module) for dependency in dependency_table[module]: - dependency_choices = set(dependency.split('|')) + dependency_choices = set(dependency.split("|")) dependency_met = False @@ -2636,7 +3185,8 @@ def resolve_dependencies(available_modules, dependency_table, module, loaded_mod for mod in possible_mods: ok, dependency_modules = ModulesChooser.resolve_dependencies( - available_modules, dependency_table, mod, loaded_modules) + available_modules, dependency_table, mod, loaded_modules + ) if ok: dependency_met = True loaded_modules.add(mod) @@ -2651,7 +3201,9 @@ def resolve_dependencies(available_modules, dependency_table, module, loaded_mod def _modules_dependency_table(self): out = {} for modname in self._modules: - out[modname] = self._modules[modname].dependencies(self._osinfo, self._archinfo) + out[modname] = self._modules[modname].dependencies( + self._osinfo, self._archinfo + ) return out def _resolve_dependencies_for_all_modules(self): @@ -2662,7 +3214,9 @@ def _resolve_dependencies_for_all_modules(self): for modname in self._to_load: # This will try to recursively load all dependencies of modname - ok, modules = self.resolve_dependencies(available_modules, dependency_table, modname) + ok, modules = self.resolve_dependencies( + available_modules, dependency_table, modname + ) if ok: successfully_loaded.add(modname) successfully_loaded.update(modules) @@ -2670,35 +3224,36 @@ def _resolve_dependencies_for_all_modules(self): # Skip this module pass - self._not_using_because['dependency failure'].update(self._to_load - successfully_loaded) + self._not_using_because["dependency failure"].update( + self._to_load - successfully_loaded + ) self._to_load = successfully_loaded self._maybe_dep -= successfully_loaded def _handle_by_load_on(self, module): modname = module.basename - if module.load_on == 'never': - self._not_using_because['disabled as buggy'].add(modname) - elif module.load_on == 'request': - self._not_using_because['by request only'].add(modname) - elif module.load_on == 'vendor': - self._not_using_because['requires external dependency'].add(modname) - elif module.load_on == 'dep': + if module.load_on == "never": + self._not_using_because["disabled as buggy"].add(modname) + elif module.load_on == "request": + self._not_using_because["by request only"].add(modname) + elif module.load_on == "vendor": + self._not_using_because["requires external dependency"].add(modname) + elif module.load_on == "dep": self._maybe_dep.add(modname) - elif module.load_on == 'always': + elif module.load_on == "always": self._to_load.add(modname) - elif module.load_on == 'auto': + elif module.load_on == "auto": if self._options.no_autoload or self._module_policy is not None: self._maybe_dep.add(modname) else: self._to_load.add(modname) else: - logging.error('Unknown load_on %s in %s', - module.load_on, modname) + logging.error("Unknown load_on %s in %s", module.load_on, modname) def choose(self): - for (modname, module) in self._modules.items(): + for modname, module in self._modules.items(): usable = self._check_usable(module, modname) module_handled = self._handle_by_module_policy(modname, module, usable) @@ -2706,38 +3261,51 @@ def choose(self): continue if modname in self._options.disabled_modules: - self._not_using_because['disabled by user'].add(modname) + self._not_using_because["disabled by user"].add(modname) elif usable: if modname in self._options.enabled_modules: - self._to_load.add(modname) # trust the user + self._to_load.add(modname) # trust the user else: self._handle_by_load_on(module) - if 'compression' in self._to_load: + if "compression" in self._to_load: # Confirm that we have at least one compression library enabled # Otherwise we leave a lot of useless support code compiled in, plus a # make_compressor call that always fails - if 'zlib' not in self._to_load and 'bzip2' not in self._to_load and 'lzma' not in self._to_load: - self._to_load.remove('compression') - self._not_using_because['no enabled compression schemes'].add('compression') + if ( + "zlib" not in self._to_load + and "bzip2" not in self._to_load + and "lzma" not in self._to_load + ): + self._to_load.remove("compression") + self._not_using_because["no enabled compression schemes"].add( + "compression" + ) # The AVX2 implementation of Argon2 fails when compiled by GCC in # amalgamation mode. - if 'argon2_avx2' in self._to_load and self._options.amalgamation and self._options.compiler == 'gcc': - self._to_load.remove('argon2_avx2') - self._not_using_because['disabled due to compiler bug'].add('argon2_avx2') + if ( + "argon2_avx2" in self._to_load + and self._options.amalgamation + and self._options.compiler == "gcc" + ): + self._to_load.remove("argon2_avx2") + self._not_using_because["disabled due to compiler bug"].add("argon2_avx2") self._resolve_dependencies_for_all_modules() for not_a_dep in self._maybe_dep: - self._not_using_because['not requested'].add(not_a_dep) + self._not_using_because["not requested"].add(not_a_dep) ModulesChooser._validate_state(self._to_load, self._not_using_because) - ModulesChooser._display_module_information_unused(self._modules, self._not_using_because) + ModulesChooser._display_module_information_unused( + self._modules, self._not_using_because + ) ModulesChooser._display_module_information_to_load(self._modules, self._to_load) return self._to_load + def choose_link_method(options): """ Choose the link method based on system availability and user request @@ -2746,7 +3314,6 @@ def choose_link_method(options): req = options.link_method def useable_methods(): - # Symbolic link support on Windows was introduced in Windows 6.0 (Vista) # and Python 3.2. Furthermore, the SeCreateSymbolicLinkPrivilege is # required in order to successfully create symlinks. So only try to use @@ -2756,28 +3323,31 @@ def useable_methods(): # so there we only use it if requested. # MinGW declares itself as 'Windows' - host_is_windows = python_platform_identifier() in ['windows', 'cygwin'] + host_is_windows = python_platform_identifier() in ["windows", "cygwin"] - if 'symlink' in os.__dict__: + if "symlink" in os.__dict__: if host_is_windows: - if req == 'symlink': - yield 'symlink' + if req == "symlink": + yield "symlink" else: - yield 'symlink' + yield "symlink" - if 'link' in os.__dict__ and req == 'hardlink': - yield 'hardlink' + if "link" in os.__dict__ and req == "hardlink": + yield "hardlink" - yield 'copy' + yield "copy" for method in useable_methods(): if req is None or req == method: - logging.info('Using %s to link files into build dir ' \ - '(use --link-method to change)', method) + logging.info( + "Using %s to link files into build dir (use --link-method to change)", + method, + ) return method logging.warning('Could not use link method "%s", will copy instead', req) - return 'copy' + return "copy" + def portable_symlink(file_path, target_dir, method): """ @@ -2785,30 +3355,30 @@ def portable_symlink(file_path, target_dir, method): """ if not os.access(file_path, os.R_OK): - logging.warning('Missing file %s', file_path) + logging.warning("Missing file %s", file_path) return - if method == 'symlink': + if method == "symlink": rel_file_path = os.path.relpath(file_path, start=target_dir) os.symlink(rel_file_path, os.path.join(target_dir, os.path.basename(file_path))) - elif method == 'hardlink': + elif method == "hardlink": os.link(file_path, os.path.join(target_dir, os.path.basename(file_path))) - elif method == 'copy': + elif method == "copy": shutil.copy(file_path, target_dir) else: - raise UserError('Unknown link method %s' % (method)) + raise UserError("Unknown link method %s" % (method)) class AmalgamationHelper: # All include types may have trailing comment like e.g. '#include // IWYU pragma: export' - _any_include = re.compile(r'#include <(.*)>') - _botan_include = re.compile(r'#include ') + _any_include = re.compile(r"#include <(.*)>") + _botan_include = re.compile(r"#include ") # Only matches at the beginning of the line. By convention, this means that the include # is not wrapped by condition macros - _unconditional_any_include = re.compile(r'^#include <(.*)>') + _unconditional_any_include = re.compile(r"^#include <(.*)>") # stddef.h is included in ffi.h - _unconditional_std_include = re.compile(r'^#include <([^/\.]+|stddef.h)>') + _unconditional_std_include = re.compile(r"^#include <([^/\.]+|stddef.h)>") @staticmethod def is_any_include(cpp_source_line): @@ -2835,18 +3405,20 @@ def is_unconditional_std_include(cpp_source_line): @staticmethod def write_banner(fd): - fd.write("""/* + fd.write( + """/* * Botan %s Amalgamation * (C) 1999-2023 The Botan Authors * * Botan is released under the Simplified BSD License (see license.txt) */ -""" % (Version.as_string())) +""" + % (Version.as_string()) + ) class AmalgamationHeader: def __init__(self, input_filepaths): - self.included_already = set() self.all_std_includes = set() @@ -2856,19 +3428,21 @@ def __init__(self, input_filepaths): contents = AmalgamationGenerator.read_header(filepath) self.file_contents[os.path.basename(filepath)] = contents except IOError as ex: - logging.error('Error processing file %s for amalgamation: %s', filepath, ex) + logging.error( + "Error processing file %s for amalgamation: %s", filepath, ex + ) - self.contents = '' + self.contents = "" for name in sorted(self.file_contents): - self.contents += ''.join(list(self.header_contents(name))) + self.contents += "".join(list(self.header_contents(name))) - self.header_includes = '' + self.header_includes = "" for std_header in sorted(self.all_std_includes): - self.header_includes += '#include <%s>\n' % (std_header) - self.header_includes += '\n' + self.header_includes += "#include <%s>\n" % (std_header) + self.header_includes += "\n" def header_contents(self, name): - name = name.replace('internal/', '') + name = name.replace("internal/", "") if name in self.included_already: return @@ -2878,7 +3452,7 @@ def header_contents(self, name): if name not in self.file_contents: return - depr_marker = 'BOTAN_DEPRECATED_HEADER(%s)\n' % (name) + depr_marker = "BOTAN_DEPRECATED_HEADER(%s)\n" % (name) if depr_marker in self.file_contents[name]: logging.debug("Ignoring deprecated header %s", name) return @@ -2896,7 +3470,7 @@ def header_contents(self, name): yield line def write_to_file(self, filepath, include_guard): - with open(filepath, 'w', encoding='utf8') as f: + with open(filepath, "w", encoding="utf8") as f: AmalgamationHelper.write_banner(f) f.write("\n#ifndef %s\n#define %s\n\n" % (include_guard, include_guard)) f.write(self.header_includes) @@ -2905,18 +3479,18 @@ def write_to_file(self, filepath, include_guard): class AmalgamationGenerator: - _header_guard_pattern = re.compile(r'^#define BOTAN_.*_H_\s*$') - _header_endif_pattern = re.compile(r'^#endif.*$') + _header_guard_pattern = re.compile(r"^#define BOTAN_.*_H_\s*$") + _header_endif_pattern = re.compile(r"^#endif.*$") @staticmethod def read_header(filepath): - with open(filepath, encoding='utf8') as f: + with open(filepath, encoding="utf8") as f: raw_content = f.readlines() return AmalgamationGenerator.strip_header_goop(filepath, raw_content) @staticmethod def strip_header_goop(header_name, header_lines): - lines = copy.copy(header_lines) # defensive copy + lines = copy.copy(header_lines) # defensive copy start_header_guard_index = None for index, line in enumerate(lines): @@ -2929,11 +3503,11 @@ def strip_header_goop(header_name, header_lines): end_header_guard_index = None for index, line in enumerate(lines): if AmalgamationGenerator._header_endif_pattern.match(line): - end_header_guard_index = index # override with last found + end_header_guard_index = index # override with last found if end_header_guard_index is None: raise InternalError("No header guard end found in " + header_name) - lines = lines[start_header_guard_index+1 : end_header_guard_index] + lines = lines[start_header_guard_index + 1 : end_header_guard_index] # Strip leading and trailing empty lines while lines[0].strip() == "": @@ -2951,9 +3525,11 @@ def __init__(self, prefix, build_paths, modules, options): def generate(self): pub_header_amalag = AmalgamationHeader(self._build_paths.public_headers) - amalgamation_header_fsname = '%s.h' % (self._filename_prefix) - logging.info('Writing amalgamation header to %s', amalgamation_header_fsname) - pub_header_amalag.write_to_file(amalgamation_header_fsname, "BOTAN_AMALGAMATION_H_") + amalgamation_header_fsname = "%s.h" % (self._filename_prefix) + logging.info("Writing amalgamation header to %s", amalgamation_header_fsname) + pub_header_amalag.write_to_file( + amalgamation_header_fsname, "BOTAN_AMALGAMATION_H_" + ) internal_headers_list = [] @@ -2961,10 +3537,10 @@ def generate(self): internal_headers_list.append(hdr) # file descriptors for all `amalgamation_sources` - amalgamation_fsname = '%s.cpp' % (self._filename_prefix) - logging.info('Writing amalgamation source to %s', amalgamation_fsname) + amalgamation_fsname = "%s.cpp" % (self._filename_prefix) + logging.info("Writing amalgamation source to %s", amalgamation_fsname) - amalgamation_file = open(amalgamation_fsname, 'w', encoding='utf8') + amalgamation_file = open(amalgamation_fsname, "w", encoding="utf8") AmalgamationHelper.write_banner(amalgamation_file) amalgamation_file.write('\n#include "%s"\n\n' % (amalgamation_header_fsname)) @@ -2977,18 +3553,23 @@ def generate(self): for mod in sorted(self._modules, key=lambda module: module.basename): for src in sorted(mod.source): - with open(src, 'r', encoding='utf8') as f: + with open(src, "r", encoding="utf8") as f: for line in f: if AmalgamationHelper.is_botan_include(line): # Botan headers are inlined in amalgamation headers continue - if AmalgamationHelper.is_any_include(line) in unconditional_headers: + if ( + AmalgamationHelper.is_any_include(line) + in unconditional_headers + ): # This include (conditional or unconditional) was unconditionally added before continue amalgamation_file.write(line) - unconditional_header = AmalgamationHelper.is_unconditional_any_include(line) + unconditional_header = ( + AmalgamationHelper.is_unconditional_any_include(line) + ) if unconditional_header: unconditional_headers.add(unconditional_header) @@ -3006,19 +3587,19 @@ def exe_test(path, program): exe_file = os.path.join(path, program) if os.path.exists(exe_file) and os.access(exe_file, os.X_OK): - logging.debug('Found program %s in %s', program, path) + logging.debug("Found program %s in %s", program, path) return True else: return False - exe_suffixes = ['', '.exe'] + exe_suffixes = ["", ".exe"] - for path in os.environ['PATH'].split(os.pathsep): + for path in os.environ["PATH"].split(os.pathsep): for suffix in exe_suffixes: if exe_test(path, program + suffix): return True - logging.debug('Program %s not found', program) + logging.debug("Program %s not found", program) return False @@ -3040,7 +3621,7 @@ def setup_logging(options): log_level = logging.INFO lh = BotanConfigureLogHandler(sys.stdout) - lh.setFormatter(logging.Formatter('%(levelname) 7s: %(message)s')) + lh.setFormatter(logging.Formatter("%(levelname) 7s: %(message)s")) logging.getLogger().addHandler(lh) logging.getLogger().setLevel(log_level) @@ -3054,7 +3635,7 @@ def filename_matches(filename): else: return filename_matcher.match(filename) is not None - for (dirpath, _, filenames) in os.walk(search_dir): + for dirpath, _, filenames in os.walk(search_dir): for filename in filenames: filepath = os.path.join(dirpath, filename) if filename_matches(filename): @@ -3062,17 +3643,19 @@ def filename_matches(filename): info[info_obj.basename] = info_obj if info: - infotxt_basenames = ' '.join(sorted(info.keys())) - logging.debug('Loaded %d %s files: %s', len(info), descr, infotxt_basenames) + infotxt_basenames = " ".join(sorted(info.keys())) + logging.debug("Loaded %d %s files: %s", len(info), descr, infotxt_basenames) else: - logging.warning('Failed to load any %s files', descr) + logging.warning("Failed to load any %s files", descr) return info def load_build_data_info_files(source_paths, descr, subdir, class_t): - matcher = re.compile(r'[_a-z0-9]+\.txt$') - return load_info_files(os.path.join(source_paths.build_data_dir, subdir), descr, matcher, class_t) + matcher = re.compile(r"[_a-z0-9]+\.txt$") + return load_info_files( + os.path.join(source_paths.build_data_dir, subdir), descr, matcher, class_t + ) # Workaround for Windows systems where antivirus is enabled GH #353 @@ -3103,62 +3686,65 @@ def robust_makedirs(directory, max_retries=5): # Final attempt, pass any exceptions up to caller. os.makedirs(directory) + def python_platform_identifier(): system_from_python = platform.system().lower() - if re.match('^cygwin_.*', system_from_python): - return 'cygwin' + if re.match("^cygwin_.*", system_from_python): + return "cygwin" return system_from_python + def run_compiler(options, ccinfo, default_return, flags=None): if flags is None: flags = [] cc_bin = options.compiler_binary or ccinfo.binary_name - cmd = cc_bin.split(' ') + flags + cmd = cc_bin.split(" ") + flags try: - logging.debug("Running '%s'", ' '.join(cmd)) + logging.debug("Running '%s'", " ".join(cmd)) stdout, _ = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True).communicate() + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True + ).communicate() cc_output = stdout except OSError as ex: - logging.warning('Could not execute %s: %s', cmd, ex) + logging.warning("Could not execute %s: %s", cmd, ex) return default_return return cc_output + # This is for options that have --with-XYZ and --without-XYZ. If user does not # set any of those, we choose a default here. # Mutates `options` def set_defaults_for_unset_options(options, info_arch, info_cc, info_os): if options.os is None: options.os = python_platform_identifier() - logging.info('Guessing target OS is %s (use --os to set)', options.os) + logging.info("Guessing target OS is %s (use --os to set)", options.os) if options.os not in info_os: + def find_canonical_os_name(os_name_variant): - for (canonical_os_name, os_info) in info_os.items(): + for canonical_os_name, os_info in info_os.items(): if os_info.matches_name(os_name_variant): return canonical_os_name - return os_name_variant # not found + return os_name_variant # not found + options.os = find_canonical_os_name(options.os) def deduce_compiler_type_from_cc_bin(options): cc_bin = options.compiler_binary - if cc_bin.find('clang') != -1 or cc_bin in ['emcc', 'em++']: - return 'clang' - if cc_bin.find('g++') != -1: - return 'gcc' + if cc_bin.find("clang") != -1 or cc_bin in ["emcc", "em++"]: + return "clang" + if cc_bin.find("g++") != -1: + return "gcc" - vers = run_compiler(options, None, '', ['--version']) - if vers.find('clang') != -1: - return 'clang' - if vers.find('Free Software Foundation') != -1: - return 'gcc' + vers = run_compiler(options, None, "", ["--version"]) + if vers.find("clang") != -1: + return "clang" + if vers.find("Free Software Foundation") != -1: + return "gcc" return None @@ -3166,37 +3752,48 @@ def deduce_compiler_type_from_cc_bin(options): options.compiler = deduce_compiler_type_from_cc_bin(options) if options.compiler is None: - logging.error("Could not figure out what compiler type '%s' is, use --cc to set", - options.compiler_binary) + logging.error( + "Could not figure out what compiler type '%s' is, use --cc to set", + options.compiler_binary, + ) - if options.compiler == 'clang' and run_compiler(options, info_cc['clang'], '?', ['--version']).startswith('Apple clang'): - options.compiler = 'xcode' + if options.compiler == "clang" and run_compiler( + options, info_cc["clang"], "?", ["--version"] + ).startswith("Apple clang"): + options.compiler = "xcode" if options.compiler is None and options.os in info_os: options.compiler = info_os[options.os].default_compiler if not have_program(info_cc[options.compiler].binary_name): - logging.error("Default compiler is %s but could not find '%s'; use --cc to set", - options.compiler, info_cc[options.compiler].binary_name) + logging.error( + "Default compiler is %s but could not find '%s'; use --cc to set", + options.compiler, + info_cc[options.compiler].binary_name, + ) - logging.info('Guessing to use compiler %s (use --cc or CXX to set)', options.compiler) + logging.info( + "Guessing to use compiler %s (use --cc or CXX to set)", options.compiler + ) if options.cpu is None: (arch, cpu) = guess_processor(info_arch) options.arch = arch options.cpu = cpu - logging.info('Guessing target processor is a %s (use --cpu to set)', options.arch) + logging.info( + "Guessing target processor is a %s (use --cpu to set)", options.arch + ) # OpenBSD uses an old binutils that does not support AVX2 - if options.os == 'openbsd': - del info_cc['gcc'].isa_flags['avx2'] + if options.os == "openbsd": + del info_cc["gcc"].isa_flags["avx2"] if options.with_documentation is True: - if options.with_sphinx is None and have_program('sphinx-build'): - logging.info('Found sphinx-build (use --without-sphinx to disable)') + if options.with_sphinx is None and have_program("sphinx-build"): + logging.info("Found sphinx-build (use --without-sphinx to disable)") options.with_sphinx = True - if options.with_rst2man is None and have_program('rst2man'): - logging.info('Found rst2man (use --without-rst2man to disable)') + if options.with_rst2man is None and have_program("rst2man"): + logging.info("Found rst2man (use --without-rst2man to disable)") options.with_rst2man = True if options.with_pkg_config is None and options.os in info_os: @@ -3204,25 +3801,29 @@ def deduce_compiler_type_from_cc_bin(options): if options.system_cert_bundle is None: default_paths = [ - '/etc/ssl/certs/ca-certificates.crt', # Ubuntu, Debian, Arch, Gentoo - '/etc/pki/tls/certs/ca-bundle.crt', # RHEL - '/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem', # Fedora - '/etc/ssl/ca-bundle.pem', # SuSE - '/etc/ssl/cert.pem', # OpenBSD, FreeBSD, Alpine - '/etc/certs/ca-certificates.crt', # Solaris - '/opt/local/share/curl/curl-ca-bundle.crt', # macOS with macPorts + "/etc/ssl/certs/ca-certificates.crt", # Ubuntu, Debian, Arch, Gentoo + "/etc/pki/tls/certs/ca-bundle.crt", # RHEL + "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", # Fedora + "/etc/ssl/ca-bundle.pem", # SuSE + "/etc/ssl/cert.pem", # OpenBSD, FreeBSD, Alpine + "/etc/certs/ca-certificates.crt", # Solaris + "/opt/local/share/curl/curl-ca-bundle.crt", # macOS with macPorts ] for path in default_paths: if os.access(path, os.R_OK): - logging.info('Using %s as system certificate store', path) + logging.info("Using %s as system certificate store", path) options.system_cert_bundle = path break else: if not os.access(options.system_cert_bundle, os.R_OK): - logging.warning('System cert bundle "%s" not found, ignoring', options.system_cert_bundle) + logging.warning( + 'System cert bundle "%s" not found, ignoring', + options.system_cert_bundle, + ) options.system_cert_bundle = None + # Mutates `options` def canonicalize_options(options, info_os, info_arch): # canonical ARCH/CPU @@ -3231,7 +3832,7 @@ def canonicalize_options(options, info_os, info_arch): raise UserError('Unknown or unidentifiable processor "%s"' % (options.cpu)) if options.cpu != options.arch: - logging.info('Canonicalized CPU target %s to %s', options.cpu, options.arch) + logging.info("Canonicalized CPU target %s to %s", options.cpu, options.arch) # select and sanity check build targets def canonicalize_build_targets(options): @@ -3240,7 +3841,9 @@ def canonicalize_build_targets(options): return ["cli", "tests"] # flatten the list of multiple --build-targets="" and comma separation - build_targets = [t.strip().lower() for ts in options.build_targets for t in ts.split(",")] + build_targets = [ + t.strip().lower() for ts in options.build_targets for t in ts.split(",") + ] # validate that all requested build targets are available for build_target in build_targets: @@ -3251,28 +3854,44 @@ def canonicalize_build_targets(options): if options.build_shared_lib is None: options.build_shared_lib = "shared" in build_targets elif bool(options.build_shared_lib) != bool("shared" in build_targets): - raise UserError("inconsistent usage of --enable/disable-shared-library and --build-targets") + raise UserError( + "inconsistent usage of --enable/disable-shared-library and --build-targets" + ) # building the static lib desired and without contradiction? if options.build_static_lib is None: options.build_static_lib = "static" in build_targets elif bool(options.build_static_lib) != bool("static" in build_targets): - raise UserError("inconsistent usage of --enable/disable-static-library and --build-targets") + raise UserError( + "inconsistent usage of --enable/disable-static-library and --build-targets" + ) return build_targets options.build_targets = canonicalize_build_targets(options) - shared_libs_supported = options.os in info_os and info_os[options.os].building_shared_supported() + shared_libs_supported = ( + options.os in info_os and info_os[options.os].building_shared_supported() + ) if not shared_libs_supported: if options.build_shared_lib is True: - logging.warning('Shared libs not supported on %s, disabling shared lib support', options.os) + logging.warning( + "Shared libs not supported on %s, disabling shared lib support", + options.os, + ) options.build_shared_lib = False elif options.build_shared_lib is None: - logging.info('Shared libs not supported on %s, disabling shared lib support', options.os) - - if options.os == 'windows' and options.build_shared_lib is None and options.build_static_lib is None: + logging.info( + "Shared libs not supported on %s, disabling shared lib support", + options.os, + ) + + if ( + options.os == "windows" + and options.build_shared_lib is None + and options.build_static_lib is None + ): options.build_shared_lib = True if options.with_stack_protector is None: @@ -3280,36 +3899,37 @@ def canonicalize_build_targets(options): options.with_stack_protector = info_os[options.os].use_stack_protector if options.build_shared_lib is None: - if options.os == 'windows' and options.build_static_lib: + if options.os == "windows" and options.build_static_lib: pass else: options.build_shared_lib = shared_libs_supported if options.build_static_lib is None: - if options.os == 'windows' and options.build_shared_lib: + if options.os == "windows" and options.build_shared_lib: pass else: options.build_static_lib = True if options.ldflags is not None: extra_libs = [] - link_to_lib = re.compile('^-l(.*)') - for flag in options.ldflags.split(' '): + link_to_lib = re.compile("^-l(.*)") + for flag in options.ldflags.split(" "): match = link_to_lib.match(flag) if match: extra_libs.append(match.group(1)) - options.extra_libs += ','.join(extra_libs) + options.extra_libs += ",".join(extra_libs) + # Checks user options for consistency # This method DOES NOT change options on behalf of the user but explains # why the given configuration does not work. def validate_options(options, info_os, info_cc, cc_version, available_module_policies): - if options.name_amalgamation != 'botan_all': - if options.name_amalgamation == '': - raise UserError('Amalgamation basename must be non-empty') + if options.name_amalgamation != "botan_all": + if options.name_amalgamation == "": + raise UserError("Amalgamation basename must be non-empty") - acceptable_name_re = re.compile('^[a-zA-Z0-9_]+$') + acceptable_name_re = re.compile("^[a-zA-Z0-9_]+$") if acceptable_name_re.match(options.name_amalgamation) is None: raise UserError("Amalgamation basename must match [a-zA-Z0-9_]+") @@ -3317,113 +3937,161 @@ def validate_options(options, info_os, info_cc, cc_version, available_module_pol raise UserError("Jython detected: need --os and --cpu to set target") if options.os not in info_os: - raise UserError('Unknown OS "%s"; available options: %s' % ( - options.os, ' '.join(sorted(info_os.keys())))) + raise UserError( + 'Unknown OS "%s"; available options: %s' + % (options.os, " ".join(sorted(info_os.keys()))) + ) if options.compiler not in info_cc: - raise UserError('Unknown compiler "%s"; available options: %s' % ( - options.compiler, ' '.join(sorted(info_cc.keys())))) - - if options.cc_min_version is not None and not re.match(r'^[0-9]+\.[0-9]+$', options.cc_min_version): + raise UserError( + 'Unknown compiler "%s"; available options: %s' + % (options.compiler, " ".join(sorted(info_cc.keys()))) + ) + + if options.cc_min_version is not None and not re.match( + r"^[0-9]+\.[0-9]+$", options.cc_min_version + ): raise UserError("--cc-min-version must have the format MAJOR.MINOR") if options.module_policy and options.module_policy not in available_module_policies: raise UserError("Unknown module set %s" % options.module_policy) - if options.cpu == 'llvm' or options.os == 'llvm': - if options.compiler != 'clang': - raise UserError('LLVM target requires using Clang') + if options.cpu == "llvm" or options.os == "llvm": + if options.compiler != "clang": + raise UserError("LLVM target requires using Clang") - if options.cpu != 'llvm': - raise UserError('LLVM target requires CPU target set to LLVM bitcode (llvm)') + if options.cpu != "llvm": + raise UserError( + "LLVM target requires CPU target set to LLVM bitcode (llvm)" + ) - if options.os != 'llvm': - raise UserError('Target OS is not an LLVM bitcode target') + if options.os != "llvm": + raise UserError("Target OS is not an LLVM bitcode target") - if options.cpu == 'wasm' or options.os == 'emscripten': - if options.compiler != 'emcc': - raise UserError('Emscripten target requires using emcc') + if options.cpu == "wasm" or options.os == "emscripten": + if options.compiler != "emcc": + raise UserError("Emscripten target requires using emcc") - if options.cpu != 'wasm': - raise UserError('Emscripten target requires CPU target set to LLVM bitcode (wasm)') + if options.cpu != "wasm": + raise UserError( + "Emscripten target requires CPU target set to LLVM bitcode (wasm)" + ) - if options.os != 'emscripten': - raise UserError('Target OS is not emscripten') + if options.os != "emscripten": + raise UserError("Target OS is not emscripten") if options.build_fuzzers is not None: - if options.build_fuzzers not in ['libfuzzer', 'afl', 'klee', 'test']: - raise UserError('Bad value to --build-fuzzers') + if options.build_fuzzers not in ["libfuzzer", "afl", "klee", "test"]: + raise UserError("Bad value to --build-fuzzers") - if options.build_fuzzers == 'klee' and options.os != 'llvm': - raise UserError('Building for KLEE requires targeting LLVM') + if options.build_fuzzers == "klee" and options.os != "llvm": + raise UserError("Building for KLEE requires targeting LLVM") - if options.build_tool not in ['make', 'ninja']: + if options.build_tool not in ["make", "ninja"]: raise UserError("Unknown --build-tool option (possible values: make, ninja)") if options.build_static_lib is False and options.build_shared_lib is False: - raise UserError('With both --disable-static-library and --disable-shared-library, nothing to do') + raise UserError( + "With both --disable-static-library and --disable-shared-library, nothing to do" + ) - if options.os == 'windows' and options.build_static_lib is True and options.build_shared_lib is True: - raise UserError('On Windows only one of static lib and DLL can be selected') + if ( + options.os == "windows" + and options.build_static_lib is True + and options.build_shared_lib is True + ): + raise UserError("On Windows only one of static lib and DLL can be selected") - if 'examples' in options.build_targets and 'boost' not in options.enabled_modules: - raise UserError('Target examples requires --with-boost') + if "examples" in options.build_targets and "boost" not in options.enabled_modules: + raise UserError("Target examples requires --with-boost") if options.with_documentation is False: if options.with_doxygen: - raise UserError('Using --with-doxygen plus --without-documentation makes no sense') + raise UserError( + "Using --with-doxygen plus --without-documentation makes no sense" + ) if options.with_sphinx: - raise UserError('Using --with-sphinx plus --without-documentation makes no sense') + raise UserError( + "Using --with-sphinx plus --without-documentation makes no sense" + ) if options.with_pdf: - raise UserError('Using --with-pdf plus --without-documentation makes no sense') + raise UserError( + "Using --with-pdf plus --without-documentation makes no sense" + ) if options.with_pdf and not options.with_sphinx: - raise UserError('Option --with-pdf requires --with-sphinx') + raise UserError("Option --with-pdf requires --with-sphinx") if options.ct_value_barrier_type: - if options.ct_value_barrier_type not in ['asm', 'volatile', 'none']: - raise UserError('Unknown setting "%s" for --ct-value-barrier-type' % (options.ct_value_barrier_type)) - - if options.enable_stack_scrubbing and (options.compiler not in ['gcc'] or float(cc_version) < 14): - raise UserError('Your compiler does not support stack scrubbing. Only GCC 14 and newer support this at the moment.') + if options.ct_value_barrier_type not in ["asm", "volatile", "none"]: + raise UserError( + 'Unknown setting "%s" for --ct-value-barrier-type' + % (options.ct_value_barrier_type) + ) + + if options.enable_stack_scrubbing and ( + options.compiler not in ["gcc"] or float(cc_version) < 14 + ): + raise UserError( + "Your compiler does not support stack scrubbing. Only GCC 14 and newer support this at the moment." + ) # Warnings - if options.os == 'windows' and options.compiler not in ('msvc', 'clangcl'): - logging.warning('The windows target is oriented towards MSVC; maybe you want --os=cygwin or --os=mingw') + if options.os == "windows" and options.compiler not in ("msvc", "clangcl"): + logging.warning( + "The windows target is oriented towards MSVC; maybe you want --os=cygwin or --os=mingw" + ) if options.msvc_runtime: - if options.compiler not in ('msvc', 'clangcl'): - raise UserError("Makes no sense to specify MSVC runtime for %s" % (options.compiler)) + if options.compiler not in ("msvc", "clangcl"): + raise UserError( + "Makes no sense to specify MSVC runtime for %s" % (options.compiler) + ) - if options.msvc_runtime not in ['MT', 'MD', 'MTd', 'MDd']: + if options.msvc_runtime not in ["MT", "MD", "MTd", "MDd"]: logging.warning("MSVC runtime option '%s' not known", options.msvc_runtime) - if 'threads' in options.without_os_features: - logging.warning('Disabling thread support will cause data races if threads are used by the application') + if "threads" in options.without_os_features: + logging.warning( + "Disabling thread support will cause data races if threads are used by the application" + ) + -def run_compiler_preproc(options, ccinfo, source_file, default_return, extra_flags=None): +def run_compiler_preproc( + options, ccinfo, source_file, default_return, extra_flags=None +): if extra_flags is None: extra_flags = [] - cc_output = run_compiler(options, ccinfo, default_return, ccinfo.preproc_flags.split(' ') + extra_flags + [source_file]) + cc_output = run_compiler( + options, + ccinfo, + default_return, + ccinfo.preproc_flags.split(" ") + extra_flags + [source_file], + ) def cleanup_output(output): - return ('\n'.join([line for line in output.splitlines() if not line.startswith('#')])).strip() + return ( + "\n".join( + [line for line in output.splitlines() if not line.startswith("#")] + ) + ).strip() return cleanup_output(cc_output) + def calculate_cc_min_version(options, ccinfo, source_paths): version_patterns = { - 'msvc': r'^ *MSVC ([0-9]{2})([0-9]{2})$', - 'gcc': r'^ *GCC ([0-9]+) ([0-9]+)$', - 'clang': r'^ *CLANG ([0-9]+) ([0-9]+)$', - 'clangcl': r'^ *CLANG ([0-9]+) ([0-9]+)$', - 'xcode': r'^ *XCODE ([0-9]+) ([0-9]+)$', - 'xlc': r'^ *XLC ([0-9]+) ([0-9]+)$', - 'emcc': r'^ *EMCC ([0-9]+) ([0-9]+)$', + "msvc": r"^ *MSVC ([0-9]{2})([0-9]{2})$", + "gcc": r"^ *GCC ([0-9]+) ([0-9]+)$", + "clang": r"^ *CLANG ([0-9]+) ([0-9]+)$", + "clangcl": r"^ *CLANG ([0-9]+) ([0-9]+)$", + "xcode": r"^ *XCODE ([0-9]+) ([0-9]+)$", + "xlc": r"^ *XLC ([0-9]+) ([0-9]+)$", + "emcc": r"^ *EMCC ([0-9]+) ([0-9]+)$", } - unknown_pattern = r'UNKNOWN 0 0' + unknown_pattern = r"UNKNOWN 0 0" cxx = ccinfo.basename @@ -3431,18 +4099,21 @@ def calculate_cc_min_version(options, ccinfo, source_paths): logging.info("No compiler version detection available for %s", cxx) return "0.0" - detect_version_source = os.path.join(source_paths.build_data_dir, "detect_version.cpp") + detect_version_source = os.path.join( + source_paths.build_data_dir, "detect_version.cpp" + ) cc_output = run_compiler_preproc(options, ccinfo, detect_version_source, "0.0") if re.search(unknown_pattern, cc_output) is not None: - logging.warning('Failed to get version for %s from macro check', cxx) + logging.warning("Failed to get version for %s from macro check", cxx) return "0.0" match = re.search(version_patterns[cxx], cc_output, flags=re.MULTILINE) if match is None: - logging.warning("Tried to get %s version, but output '%s' is unexpected", - cxx, cc_output) + logging.warning( + "Tried to get %s version, but output '%s' is unexpected", cxx, cc_output + ) return "0.0" major_version = int(match.group(1), 0) @@ -3450,8 +4121,8 @@ def calculate_cc_min_version(options, ccinfo, source_paths): cc_version = "%d.%d" % (major_version, minor_version) - if cc_version != '0.0': - logging.info('Auto-detected compiler version %s %s', cxx, cc_version) + if cc_version != "0.0": + logging.info("Auto-detected compiler version %s %s", cxx, cc_version) if ccinfo.minimum_supported_version: # compare as floats @@ -3459,22 +4130,32 @@ def calculate_cc_min_version(options, ccinfo, source_paths): our_ver = float(cc_version) if our_ver < min_ver: - logging.error("This version of Botan requires at least %s %s", - cxx, ccinfo.minimum_supported_version) + logging.error( + "This version of Botan requires at least %s %s", + cxx, + ccinfo.minimum_supported_version, + ) return cc_version -def check_compiler_arch(options, ccinfo, archinfo, source_paths): - detect_version_source = os.path.join(source_paths.build_data_dir, 'detect_arch.cpp') - - abi_flags = ccinfo.mach_abi_link_flags(options).split(' ') - cc_output = run_compiler_preproc(options, ccinfo, detect_version_source, 'UNKNOWN', abi_flags).lower() - if cc_output == '': - cc_output = run_compiler_preproc(options, ccinfo, detect_version_source, 'UNKNOWN').lower() - - if cc_output == 'unknown': - logging.warning('Unable to detect target architecture via compiler macro checks') +def check_compiler_arch(options, ccinfo, archinfo, source_paths): + detect_version_source = os.path.join(source_paths.build_data_dir, "detect_arch.cpp") + + abi_flags = ccinfo.mach_abi_link_flags(options).split(" ") + cc_output = run_compiler_preproc( + options, ccinfo, detect_version_source, "UNKNOWN", abi_flags + ).lower() + + if cc_output == "": + cc_output = run_compiler_preproc( + options, ccinfo, detect_version_source, "UNKNOWN" + ).lower() + + if cc_output == "unknown": + logging.warning( + "Unable to detect target architecture via compiler macro checks" + ) return None if cc_output not in archinfo: @@ -3482,15 +4163,26 @@ def check_compiler_arch(options, ccinfo, archinfo, source_paths): logging.warning("Error detecting compiler target arch: '%s'", cc_output) return None - logging.info('Auto-detected compiler arch %s', cc_output) + logging.info("Auto-detected compiler arch %s", cc_output) return cc_output -def do_io_for_build(cc, arch, osinfo, using_mods, info_modules, build_paths, source_paths, template_vars, options): + +def do_io_for_build( + cc, + arch, + osinfo, + using_mods, + info_modules, + build_paths, + source_paths, + template_vars, + options, +): try: robust_rmtree(build_paths.build_dir) except OSError as ex: if ex.errno != errno.ENOENT: - logging.error('Problem while removing build dir: %s', ex) + logging.error("Problem while removing build dir: %s", ex) for build_dir in build_paths.build_dirs(): try: @@ -3499,128 +4191,161 @@ def do_io_for_build(cc, arch, osinfo, using_mods, info_modules, build_paths, sou if ex.errno != errno.EEXIST: logging.error('Error while creating "%s": %s', build_dir, ex) - def write_template_with_variables(sink, template, variables, postproc_fn = None): + def write_template_with_variables(sink, template, variables, postproc_fn=None): output = process_template(template, variables) if postproc_fn: output = postproc_fn(output) - with open(sink, 'w', encoding='utf8') as f: + with open(sink, "w", encoding="utf8") as f: f.write(output) - def write_template(sink, template, postproc_fn = None): + def write_template(sink, template, postproc_fn=None): write_template_with_variables(sink, template, template_vars, postproc_fn) def in_build_dir(p): return os.path.join(build_paths.build_dir, p) + def in_build_data(p): return os.path.join(source_paths.build_data_dir, p) + def in_build_module_info(p): return os.path.join(build_paths.doc_module_info, p) - write_template(in_build_dir('build.h'), in_build_data('buildh.in')) - write_template(in_build_dir('target_info.h'), in_build_data('target_info.h.in')) - write_template(in_build_dir('version_info.h'), in_build_data('version_info.h.in')) - write_template(in_build_dir('botan.doxy'), in_build_data('botan.doxy.in')) + write_template(in_build_dir("build.h"), in_build_data("buildh.in")) + write_template(in_build_dir("target_info.h"), in_build_data("target_info.h.in")) + write_template(in_build_dir("version_info.h"), in_build_data("version_info.h.in")) + write_template(in_build_dir("botan.doxy"), in_build_data("botan.doxy.in")) if options.with_cmake_config: robust_makedirs(in_build_dir("cmake")) - write_template(in_build_dir('cmake/botan-config.cmake'), in_build_data('botan-config.cmake.in')) - write_template(in_build_dir('cmake/botan-config-version.cmake'), in_build_data('botan-config-version.cmake.in')) - - if 'botan_pkgconfig' in template_vars: - write_template(template_vars['botan_pkgconfig'], in_build_data('botan.pc.in')) + write_template( + in_build_dir("cmake/botan-config.cmake"), + in_build_data("botan-config.cmake.in"), + ) + write_template( + in_build_dir("cmake/botan-config-version.cmake"), + in_build_data("botan-config-version.cmake.in"), + ) + + if "botan_pkgconfig" in template_vars: + write_template(template_vars["botan_pkgconfig"], in_build_data("botan.pc.in")) link_method = choose_link_method(options) def link_headers(headers, visibility, directory): - logging.debug('Linking %d %s header files in %s', len(headers), visibility, directory) + logging.debug( + "Linking %d %s header files in %s", len(headers), visibility, directory + ) for header_file in headers: try: portable_symlink(header_file, directory, link_method) except OSError as ex: if ex.errno != errno.EEXIST: - raise UserError('Error linking %s into %s: %s' % (header_file, directory, ex)) from ex + raise UserError( + "Error linking %s into %s: %s" % (header_file, directory, ex) + ) from ex - link_headers(build_paths.public_headers, 'public', - build_paths.public_include_dir) + link_headers(build_paths.public_headers, "public", build_paths.public_include_dir) - link_headers(build_paths.internal_headers, 'internal', - build_paths.internal_include_dir) + link_headers( + build_paths.internal_headers, "internal", build_paths.internal_include_dir + ) - link_headers(build_paths.external_headers, 'external', - build_paths.external_include_dir) + link_headers( + build_paths.external_headers, "external", build_paths.external_include_dir + ) if options.amalgamation: (amalg_cpp_files, amalg_headers) = AmalgamationGenerator( - options.name_amalgamation, build_paths, using_mods, options).generate() + options.name_amalgamation, build_paths, using_mods, options + ).generate() build_paths.lib_sources = amalg_cpp_files - template_vars['generated_files'] = ' '.join(amalg_cpp_files + amalg_headers) + template_vars["generated_files"] = " ".join(amalg_cpp_files + amalg_headers) - link_headers(amalg_headers, 'public', build_paths.public_include_dir) + link_headers(amalg_headers, "public", build_paths.public_include_dir) # Inserting an amalgamation generated using DLL visibility flags into a # binary project will either cause errors (on Windows) or unnecessary overhead. # Provide a hint if options.build_shared_lib: - logging.warning('Unless you are building a DLL or .so from the amalgamation, use --disable-shared as well') + logging.warning( + "Unless you are building a DLL or .so from the amalgamation, use --disable-shared as well" + ) - template_vars.update(generate_build_info(build_paths, using_mods, cc, arch, osinfo, options)) + template_vars.update( + generate_build_info(build_paths, using_mods, cc, arch, osinfo, options) + ) - with open(os.path.join(build_paths.build_dir, 'build_config.json'), 'w', encoding='utf8') as f: + with open( + os.path.join(build_paths.build_dir, "build_config.json"), "w", encoding="utf8" + ) as f: json.dump(template_vars, f, sort_keys=True, indent=2) if options.with_compilation_database: - write_template(in_build_dir('compile_commands.json'), in_build_data('compile_commands.json.in')) + write_template( + in_build_dir("compile_commands.json"), + in_build_data("compile_commands.json.in"), + ) + + if options.build_tool == "make": + write_template(template_vars["makefile_path"], in_build_data("makefile.in")) + elif options.build_tool == "ninja": - if options.build_tool == 'make': - write_template(template_vars['makefile_path'], in_build_data('makefile.in')) - elif options.build_tool == 'ninja': def escape_build_lines(contents): - ninja_build_line = re.compile('^build (.*): (.*)') + ninja_build_line = re.compile("^build (.*): (.*)") output = [] - for line in contents.split('\n'): + for line in contents.split("\n"): match = ninja_build_line.match(line) if match: - escaped1 = match.group(1).replace(':', '$:') - escaped2 = match.group(2).replace(':', '$:') - output.append('build %s: %s' % (escaped1, escaped2)) + escaped1 = match.group(1).replace(":", "$:") + escaped2 = match.group(2).replace(":", "$:") + output.append("build %s: %s" % (escaped1, escaped2)) else: output.append(line) return "\n".join(output) - write_template(template_vars['ninja_build_path'], in_build_data('ninja.in'), escape_build_lines) + write_template( + template_vars["ninja_build_path"], + in_build_data("ninja.in"), + escape_build_lines, + ) if options.with_doxygen: for module_name, info in info_modules.items(): - dependencies = [dep for dep in info.requires if not dep.endswith('?dyn_load')] - write_template_with_variables(in_build_module_info(module_name + '.dox'), in_build_data('module_info.in'), - { - 'parent': info.parent_module, - 'identifier': module_name, - 'title': info.name, - 'internal': info.is_internal(), - 'virtual': info.is_virtual(), - 'deprecated': info.is_deprecated(), - 'experimental': info.is_experimental(), - 'brief': info.brief, - 'public_headers': info.header_public, - 'internal_headers': info.header_internal, - 'sources': info.sources(), - 'dependencies': dependencies, - 'os_features': info.os_features, - 'cpu_features': info.isa, - 'arch_requirements': info.arch, - 'compiler_requirements': info.cc - }) + dependencies = [ + dep for dep in info.requires if not dep.endswith("?dyn_load") + ] + write_template_with_variables( + in_build_module_info(module_name + ".dox"), + in_build_data("module_info.in"), + { + "parent": info.parent_module, + "identifier": module_name, + "title": info.name, + "internal": info.is_internal(), + "virtual": info.is_virtual(), + "deprecated": info.is_deprecated(), + "experimental": info.is_experimental(), + "brief": info.brief, + "public_headers": info.header_public, + "internal_headers": info.header_internal, + "sources": info.sources(), + "dependencies": dependencies, + "os_features": info.os_features, + "cpu_features": info.isa, + "arch_requirements": info.arch, + "compiler_requirements": info.cc, + }, + ) if options.with_rst2man: - rst2man_file = os.path.join(build_paths.build_dir, 'botan.rst') - cli_doc = os.path.join(source_paths.doc_dir, 'cli.rst') + rst2man_file = os.path.join(build_paths.build_dir, "botan.rst") + cli_doc = os.path.join(source_paths.doc_dir, "cli.rst") - cli_doc_contents = open(cli_doc, encoding='utf8').readlines() + cli_doc_contents = open(cli_doc, encoding="utf8").readlines() while cli_doc_contents[0] != "\n": cli_doc_contents.pop(0) @@ -3634,19 +4359,21 @@ def escape_build_lines(contents): """.strip() - with open(rst2man_file, 'w', encoding='utf8') as f: + with open(rst2man_file, "w", encoding="utf8") as f: f.write(rst2man_header) f.write("\n") for line in cli_doc_contents: f.write(line) - date = 'dated %d' % (Version.datestamp()) if Version.datestamp() != 0 else 'undated' + date = "dated %d" % (Version.datestamp()) if Version.datestamp() != 0 else "undated" - logging.info('Botan %s (revision %s) (%s %s) build setup is complete', - Version.as_string(), - Version.vc_rev(), - Version.release_type(), - date) + logging.info( + "Botan %s (revision %s) (%s %s) build setup is complete", + Version.as_string(), + Version.vc_rev(), + Version.release_type(), + date, + ) # Warn about build modes that are not safe for production: if options.unsafe_fuzzer_mode: @@ -3655,15 +4382,18 @@ def escape_build_lines(contents): if options.unsafe_terminate_on_asserts: logging.warning("Terminating on assertion failures is NOT SAFE FOR PRODUCTION") + def list_os_features(all_os_features, info_os): for feat in all_os_features: os_with_feat = [o for o in info_os.keys() if feat in info_os[o].target_features] - os_without_feat = [o for o in info_os.keys() if feat not in info_os[o].target_features] + os_without_feat = [ + o for o in info_os.keys() if feat not in info_os[o].target_features + ] if len(os_with_feat) < len(os_without_feat): - print("%s: %s" % (feat, ' '.join(sorted(os_with_feat)))) + print("%s: %s" % (feat, " ".join(sorted(os_with_feat)))) else: - print("%s: %s" % (feat, '!' + ' !'.join(sorted(os_without_feat)))) + print("%s: %s" % (feat, "!" + " !".join(sorted(os_without_feat)))) return 0 @@ -3678,20 +4408,30 @@ def main(argv): source_paths = SourcePaths(os.path.dirname(argv[0])) - info_modules = load_info_files(source_paths.lib_dir, 'Modules', "info.txt", ModuleInfo) + info_modules = load_info_files( + source_paths.lib_dir, "Modules", "info.txt", ModuleInfo + ) if options.list_modules: - public_modules = [name for (name, info) in info_modules.items() if info.is_public()] + public_modules = [ + name for (name, info) in info_modules.items() if info.is_public() + ] for mod in sorted(public_modules): print(mod) return 0 - info_arch = load_build_data_info_files(source_paths, 'CPU info', 'arch', ArchInfo) - info_os = load_build_data_info_files(source_paths, 'OS info', 'os', OsInfo) - info_cc = load_build_data_info_files(source_paths, 'compiler info', 'cc', CompilerInfo) - info_module_policies = load_build_data_info_files(source_paths, 'module policy', 'policy', ModulePolicyInfo) - - all_os_features = sorted(set(flatten([o.target_features for o in info_os.values()]))) + info_arch = load_build_data_info_files(source_paths, "CPU info", "arch", ArchInfo) + info_os = load_build_data_info_files(source_paths, "OS info", "os", OsInfo) + info_cc = load_build_data_info_files( + source_paths, "compiler info", "cc", CompilerInfo + ) + info_module_policies = load_build_data_info_files( + source_paths, "module policy", "policy", ModulePolicyInfo + ) + + all_os_features = sorted( + set(flatten([o.target_features for o in info_os.values()])) + ) all_defined_isas = set(flatten([a.isa_extensions for a in info_arch.values()])) if options.list_os_features: @@ -3706,18 +4446,27 @@ def main(argv): for policy in info_module_policies.values(): policy.cross_check(info_modules) - logging.info('%s invoked with options "%s"', argv[0], ' '.join(argv[1:])) - logging.info('Configuring to build Botan %s (revision %s)', - Version.as_string(), Version.vc_rev()) - logging.info('Python version: "%s"', sys.version.replace('\n', '')) + logging.info('%s invoked with options "%s"', argv[0], " ".join(argv[1:])) + logging.info( + "Configuring to build Botan %s (revision %s)", + Version.as_string(), + Version.vc_rev(), + ) + logging.info('Python version: "%s"', sys.version.replace("\n", "")) take_options_from_env(options) - logging.info('Autodetected platform information: OS="%s" machine="%s" proc="%s"', - platform.system(), platform.machine(), platform.processor()) + logging.info( + 'Autodetected platform information: OS="%s" machine="%s" proc="%s"', + platform.system(), + platform.machine(), + platform.processor(), + ) - cpu_names = sorted(flatten([[ainfo.basename] + ainfo.aliases for ainfo in info_arch.values()])) - logging.debug('Known CPU names: %s', ' '.join(cpu_names)) + cpu_names = sorted( + flatten([[ainfo.basename] + ainfo.aliases for ainfo in info_arch.values()]) + ) + logging.debug("Known CPU names: %s", " ".join(cpu_names)) set_defaults_for_unset_options(options, info_arch, info_cc, info_os) canonicalize_options(options, info_os, info_arch) @@ -3725,54 +4474,91 @@ def main(argv): cc = info_cc[options.compiler] arch = info_arch[options.arch] osinfo = info_os[options.os] - module_policy = info_module_policies[options.module_policy] if options.module_policy else None + module_policy = ( + info_module_policies[options.module_policy] if options.module_policy else None + ) if options.enable_cc_tests: - cc_min_version = options.cc_min_version or calculate_cc_min_version(options, cc, source_paths) + cc_min_version = options.cc_min_version or calculate_cc_min_version( + options, cc, source_paths + ) - if options.arch not in ['generic', 'llvm']: + if options.arch not in ["generic", "llvm"]: cc_arch = check_compiler_arch(options, cc, info_arch, source_paths) if cc_arch is not None and cc_arch != options.arch: - logging.error("Configured target is %s but compiler probe indicates %s", options.arch, cc_arch) + logging.error( + "Configured target is %s but compiler probe indicates %s", + options.arch, + cc_arch, + ) else: cc_min_version = options.cc_min_version or "0.0" validate_options(options, info_os, info_cc, cc_min_version, info_module_policies) - logging.info('Target is %s:%s-%s-%s', - options.compiler, cc_min_version, options.os, options.arch) - - chooser = ModulesChooser(info_modules, module_policy, arch, osinfo, cc, cc_min_version, options) + logging.info( + "Target is %s:%s-%s-%s", + options.compiler, + cc_min_version, + options.os, + options.arch, + ) + + chooser = ModulesChooser( + info_modules, module_policy, arch, osinfo, cc, cc_min_version, options + ) loaded_module_names = chooser.choose() using_mods = [info_modules[modname] for modname in loaded_module_names] - not_using_mods = [modinfo for modname, modinfo in info_modules.items() if modname not in loaded_module_names] + not_using_mods = [ + modinfo + for modname, modinfo in info_modules.items() + if modname not in loaded_module_names + ] build_paths = BuildPaths(source_paths, options, using_mods) - build_paths.public_headers.append(os.path.join(build_paths.build_dir, 'build.h')) - for internal_headers in ['target_info.h', 'version_info.h']: - build_paths.internal_headers.append(os.path.join(build_paths.build_dir, internal_headers)) + build_paths.public_headers.append(os.path.join(build_paths.build_dir, "build.h")) + for internal_headers in ["target_info.h", "version_info.h"]: + build_paths.internal_headers.append( + os.path.join(build_paths.build_dir, internal_headers) + ) - template_vars = create_template_vars(source_paths, build_paths, options, using_mods, not_using_mods, cc, arch, osinfo) + template_vars = create_template_vars( + source_paths, build_paths, options, using_mods, not_using_mods, cc, arch, osinfo + ) # Now we start writing to disk - do_io_for_build(cc, arch, osinfo, using_mods, info_modules, build_paths, source_paths, template_vars, options) + do_io_for_build( + cc, + arch, + osinfo, + using_mods, + info_modules, + build_paths, + source_paths, + template_vars, + options, + ) return 0 -if __name__ == '__main__': + +if __name__ == "__main__": try: sys.exit(main(argv=sys.argv)) except UserError as e: logging.debug(traceback.format_exc()) logging.error(e) - except Exception: # pylint: disable=broad-except + except Exception: # pylint: disable=broad-except # error() will stop script, so wrap all information into one call - logging.error("""%s + logging.error( + """%s An internal error occurred. Don't panic, this is probably not your fault! Please open an issue with the entire output at https://github.com/randombit/botan -You'll meet friendly people happy to help!""", traceback.format_exc()) +You'll meet friendly people happy to help!""", + traceback.format_exc(), + ) sys.exit(0) diff --git a/src/configs/sphinx/conf.py b/src/configs/sphinx/conf.py index 30455619f1d..e31057dcb27 100644 --- a/src/configs/sphinx/conf.py +++ b/src/configs/sphinx/conf.py @@ -4,12 +4,13 @@ import sys import re -#import sphinx +# import sphinx + def check_for_tag(tag): # Nasty hack :( try: - opt_t = sys.argv.index('-t') + opt_t = sys.argv.index("-t") opt_tag = sys.argv.index(tag) return opt_t + 1 == opt_tag except ValueError: @@ -22,76 +23,77 @@ def parse_version_file(version_path): results = {} for line in version_file.readlines(): - if not line or line[0] == '#': + if not line or line[0] == "#": continue match = key_and_val.match(line) if match: key = match.group(1) val = match.group(2) - if val == 'None': + if val == "None": val = None elif val.startswith("'") and val.endswith("'"): - val = val[1:len(val)-1] + val = val[1 : len(val) - 1] else: val = int(val) results[key] = val return results -version_info = parse_version_file('../../build-data/version.txt') -version_major = version_info['release_major'] -version_minor = version_info['release_minor'] -version_patch = version_info['release_patch'] -version_suffix = version_info['release_suffix'] +version_info = parse_version_file("../../build-data/version.txt") + +version_major = version_info["release_major"] +version_minor = version_info["release_minor"] +version_patch = version_info["release_patch"] +version_suffix = version_info["release_suffix"] -is_website_build = check_for_tag('website') +is_website_build = check_for_tag("website") -needs_sphinx = '1.2' +needs_sphinx = "1.2" -templates_path = ['templates'] +templates_path = ["templates"] -source_suffix = '.rst' +source_suffix = ".rst" -source_encoding = 'utf-8-sig' +source_encoding = "utf-8-sig" -master_doc = 'contents' +master_doc = "contents" -project = u'botan' -copyright = u'2000-2023, The Botan Authors' +project = "botan" +copyright = "2000-2023, The Botan Authors" -version = '%d.%d' % (version_major, version_minor) -release = '%d.%d.%d%s' % (version_major, version_minor, version_patch, version_suffix) +version = "%d.%d" % (version_major, version_minor) +release = "%d.%d.%d%s" % (version_major, version_minor, version_patch, version_suffix) -#today = '' -today_fmt = '%Y-%m-%d' +# today = '' +today_fmt = "%Y-%m-%d" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = [] # The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. add_function_parentheses = False # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False -highlight_language = 'cpp' +highlight_language = "cpp" # The name of the Pygments (syntax highlighting) style to use. -#pygments_style = 'sphinx' +# pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- @@ -101,36 +103,37 @@ def parse_version_file(version_path): try: # On Arch this is python-sphinx-furo - import furo # noqa: F401 + import furo # noqa: F401 + html_theme = "furo" # Add a small edit button to each document to allow visitors to easily # propose changes to that document using the repository’s source control system. html_theme_options = { - 'source_repository': 'https://github.com/randombit/botan/', - 'source_branch': 'master', - 'source_directory': 'doc/', + "source_repository": "https://github.com/randombit/botan/", + "source_branch": "master", + "source_directory": "doc/", } except ImportError: print("Could not import furo theme; falling back to agago") - html_theme = 'agogo' + html_theme = "agogo" html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -html_title = 'Botan' +html_title = "Botan" # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, @@ -139,27 +142,27 @@ def parse_version_file(version_path): # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -html_last_updated_fmt = '%Y-%m-%d' +html_last_updated_fmt = "%Y-%m-%d" # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. html_show_sourcelink = False @@ -174,58 +177,56 @@ def parse_version_file(version_path): # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. if is_website_build: - html_use_opensearch = 'https://botan.randombit.net/' + html_use_opensearch = "https://botan.randombit.net/" else: - html_use_opensearch = '' + html_use_opensearch = "" # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Output file base name for HTML help builder. -htmlhelp_basename = 'botandoc' +htmlhelp_basename = "botandoc" # -- Options for LaTeX output -------------------------------------------------- # The paper size ('letter' or 'a4'). -#latex_paper_size = 'letter' +# latex_paper_size = 'letter' # The font size ('10pt', '11pt' or '12pt'). -#latex_font_size = '10pt' +# latex_font_size = '10pt' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). -authors = u'The Botan Authors' +authors = "The Botan Authors" latex_documents = [ - ('contents', 'botan.tex', u'Botan Reference Guide', authors, 'manual'), + ("contents", "botan.tex", "Botan Reference Guide", authors, "manual"), ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. latex_show_pagerefs = False # If true, show URL addresses after external links. -latex_show_urls = 'inline' +latex_show_urls = "inline" # Additional stuff for the LaTeX preamble. -#latex_preamble = '' +# latex_preamble = '' # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. latex_domain_indices = False -latex_elements = { - 'printindex': '\\footnotesize\\raggedright\\printindex' -} +latex_elements = {"printindex": "\\footnotesize\\raggedright\\printindex"} # Give all sections a label, so we can reference them extensions = [ diff --git a/src/ct_selftest/ct_selftest.py b/src/ct_selftest/ct_selftest.py index 436be820909..45d25162b90 100755 --- a/src/ct_selftest/ct_selftest.py +++ b/src/ct_selftest/ct_selftest.py @@ -17,26 +17,28 @@ from enum import StrEnum, auto import json -def run_command(cmd: list[str], is_text = True): - """ Run the command . """ + +def run_command(cmd: list[str], is_text=True): + """Run the command .""" return subprocess.run(cmd, capture_output=True, text=is_text, check=False) + def run_with_valgrind(cmd: list[str]): - """ Run a command with valgrind. """ - valgrind_args = ['valgrind', - '-v', - '--error-exitcode=2'] + """Run a command with valgrind.""" + valgrind_args = ["valgrind", "-v", "--error-exitcode=2"] res = run_command(valgrind_args + cmd, is_text=False) # valgrind may output non-utf-8 characters res.stdout = res.stdout.decode("utf-8", errors="replace") res.stderr = res.stderr.decode("utf-8", errors="replace") return res + class ValgrindTest: - """ A single test from ct_selftest """ + """A single test from ct_selftest""" class Status(StrEnum): - """ Defines the test result status """ + """Defines the test result status""" + OK = auto() WARNING = auto() ERROR = auto() @@ -55,12 +57,14 @@ def __init__(self, name, expect_failure, needs_special_config): def from_line(line: str): info = line.split("\t") assert len(info) == 3 - return ValgrindTest(name=info[2], - expect_failure=info[0] == "true", - needs_special_config=info[1] == "true") + return ValgrindTest( + name=info[2], + expect_failure=info[0] == "true", + needs_special_config=info[1] == "true", + ) def runnable(self, build_cfg): - """ Decide whether or not to run this test given build config info """ + """Decide whether or not to run this test given build config info""" if not self.needs_special_config: return True @@ -68,12 +72,15 @@ def runnable(self, build_cfg): return False # test has special build requirements, but we have no info if self.name == "clang_vs_bare_metal_ct_mask": - return build_cfg["cc_macro"] == "CLANG" and "-Os" in build_cfg["cc_compile_flags"] + return ( + build_cfg["cc_macro"] == "CLANG" + and "-Os" in build_cfg["cc_compile_flags"] + ) raise LookupError(f"Unknown special config test '{self.name}'") def run(self, exe_path: str, build_config): - """ Run the test and return whether it succeeded """ + """Run the test and return whether it succeeded""" if not self.runnable(build_config): self.status = self.Status.SKIP @@ -98,15 +105,21 @@ def run(self, exe_path: str, build_config): @staticmethod def read_test_list(ct_selftest_test_list: str) -> list[Self]: - """ Read the list of tests from the output of `ct_selftest --list`. """ + """Read the list of tests from the output of `ct_selftest --list`.""" - return [ValgrindTest.from_line(line) for line in ct_selftest_test_list.split("\n")[2:] if line] + return [ + ValgrindTest.from_line(line) + for line in ct_selftest_test_list.split("\n")[2:] + if line + ] -def main(): # pylint: disable=missing-function-docstring +def main(): # pylint: disable=missing-function-docstring parser = argparse.ArgumentParser("ct_selftests") parser.add_argument("ct_selftest_path", help="Path to the ct_selftest executable") - parser.add_argument("--build-config-path", help="Path to Botan's build-config.json file", default="") + parser.add_argument( + "--build-config-path", help="Path to Botan's build-config.json file", default="" + ) args = parser.parse_args() @@ -145,5 +158,5 @@ def open_build_config(build_config_path): print(test.stderr) -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(main()) diff --git a/src/editors/vscode/scripts/bogo.py b/src/editors/vscode/scripts/bogo.py index 103c4f0f695..5ad70a51494 100755 --- a/src/editors/vscode/scripts/bogo.py +++ b/src/editors/vscode/scripts/bogo.py @@ -18,18 +18,29 @@ def main(): if not os.path.isdir(BORING_PATH): # check out our fork of boring ssl - run_cmd("git clone --depth 1 --branch %s %s %s" % - (BORING_BRANCH, BORING_REPO, BORING_PATH)) + run_cmd( + "git clone --depth 1 --branch %s %s %s" + % (BORING_BRANCH, BORING_REPO, BORING_PATH) + ) # make doubly sure we're on the correct branch run_cmd("git -C %s checkout %s" % (BORING_PATH, BORING_BRANCH)) - extra_args = "-debug -test '%s'" % ';'.join( - sys.argv[1:]) if len(sys.argv) > 1 else '' + extra_args = ( + "-debug -test '%s'" % ";".join(sys.argv[1:]) if len(sys.argv) > 1 else "" + ) - run_cmd("go test -pipe -num-workers %d -shim-path %s -shim-config %s %s" % - (get_concurrency(), os.path.abspath(SHIM_PATH), os.path.abspath(SHIM_CONFIG), extra_args), BOGO_PATH) + run_cmd( + "go test -pipe -num-workers %d -shim-path %s -shim-config %s %s" + % ( + get_concurrency(), + os.path.abspath(SHIM_PATH), + os.path.abspath(SHIM_CONFIG), + extra_args, + ), + BOGO_PATH, + ) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/src/editors/vscode/scripts/common.py b/src/editors/vscode/scripts/common.py index 55fbf44941c..eb897164ad9 100644 --- a/src/editors/vscode/scripts/common.py +++ b/src/editors/vscode/scripts/common.py @@ -4,6 +4,7 @@ import re import multiprocessing + class BuildError(Exception): pass @@ -23,34 +24,33 @@ def get_concurrency(): def run_cmd(cmd, workingdir=None): if isinstance(cmd, str): - print('> running: ' + cmd) + print("> running: " + cmd) shell = True else: - print('> running: ' + ' '.join(cmd)) + print("> running: " + " ".join(cmd)) shell = False sys.stdout.flush() try: subprocess.run(cmd, shell=shell, check=True, cwd=workingdir) except subprocess.CalledProcessError as ex: - raise BuildError('External command failed, aborting...') from ex + raise BuildError("External command failed, aborting...") from ex def _find_regex_in_makefile(regex): if not os.path.exists(_MAKEFILE): - raise BuildError('No Makefile found. Maybe run ./configure.py?') + raise BuildError("No Makefile found. Maybe run ./configure.py?") - with open(_MAKEFILE, 'r', encoding="utf-8") as f: + with open(_MAKEFILE, "r", encoding="utf-8") as f: return re.search(regex, f.read()) def get_test_binary_name(): - match = _find_regex_in_makefile(r'TEST\s*=\s*([^\n]+)\n') + match = _find_regex_in_makefile(r"TEST\s*=\s*([^\n]+)\n") if not match: - raise BuildError('Test binary name not found in Makefile') + raise BuildError("Test binary name not found in Makefile") test_file = os.path.split(match.group(1))[1] if not test_file: - raise BuildError( - 'Cannot make sense of test binary name: ' + match.group(0)) + raise BuildError("Cannot make sense of test binary name: " + match.group(0)) return test_file diff --git a/src/editors/vscode/scripts/test.py b/src/editors/vscode/scripts/test.py index c74b12a2e7d..3f0ad2a5c32 100644 --- a/src/editors/vscode/scripts/test.py +++ b/src/editors/vscode/scripts/test.py @@ -13,33 +13,35 @@ def get_test_names_from(test_file): if not os.path.dirname(test_file) == TESTS_DIR: raise common.BuildError( - 'Given file path is not a Botan unit test: ' + test_file) + "Given file path is not a Botan unit test: " + test_file + ) - with open(test_file, 'r', encoding='utf-8') as f: - find_test_registration = \ - re.compile( - r'BOTAN_REGISTER_TEST(_FN)?\s*\(\s*\"(.+)\",\s*\"(.+)\",[^)]+\)') + with open(test_file, "r", encoding="utf-8") as f: + find_test_registration = re.compile( + r"BOTAN_REGISTER_TEST(_FN)?\s*\(\s*\"(.+)\",\s*\"(.+)\",[^)]+\)" + ) matches = find_test_registration.findall(f.read()) tests = [match[-1] for match in matches] if not tests: raise common.BuildError( - 'Failed to find a BOTAN_REGISTER_TEST in the given test file: ' + test_file) + "Failed to find a BOTAN_REGISTER_TEST in the given test file: " + test_file + ) return tests def main(): - test_binary = os.path.join('.', common.get_test_binary_name()) + test_binary = os.path.join(".", common.get_test_binary_name()) if len(sys.argv) == 2: test_src_file = sys.argv[1] test_names = get_test_names_from(test_src_file) - common.run_cmd("%s %s" % (test_binary, ' '.join(test_names))) + common.run_cmd("%s %s" % (test_binary, " ".join(test_names))) else: common.run_cmd(test_binary) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/src/python/botan3.py b/src/python/botan3.py index 27602d4cd60..5b8c93217ef 100755 --- a/src/python/botan3.py +++ b/src/python/botan3.py @@ -18,8 +18,24 @@ """ from __future__ import annotations -from ctypes import CDLL, CFUNCTYPE, POINTER, byref, create_string_buffer, \ - c_void_p, c_size_t, c_uint8, c_uint32, c_uint64, c_int, c_uint, c_char, c_char_p, addressof, Array +from ctypes import ( + CDLL, + CFUNCTYPE, + POINTER, + byref, + create_string_buffer, + c_void_p, + c_size_t, + c_uint8, + c_uint32, + c_uint64, + c_int, + c_uint, + c_char, + c_char_p, + addressof, + Array, +) from typing import Callable, Any, Union, List from sys import platform @@ -33,56 +49,58 @@ # 3.10.0 - introduced botan_pubkey_load_ec*_sec1() BOTAN_FFI_VERSION = 20250829 + # # Base exception for all exceptions raised from this module # class BotanException(Exception): - def __init__(self, message, rc=0): - self.__rc = rc if rc == 0: super().__init__(message) else: - exn_msg = _DLL.botan_error_last_exception_message().decode('ascii') - err_descr = _DLL.botan_error_description(rc).decode('ascii') + exn_msg = _DLL.botan_error_last_exception_message().decode("ascii") + err_descr = _DLL.botan_error_description(rc).decode("ascii") formatted_msg = "%s: %d (%s)" % (message, rc, err_descr) if exn_msg != "": - formatted_msg += ': ' + exn_msg + formatted_msg += ": " + exn_msg super().__init__(formatted_msg) def error_code(self) -> int: return self.__rc + # # Module initialization # -def _load_botan_dll(expected_version): +def _load_botan_dll(expected_version): possible_dll_names = [] - if platform in ['win32', 'cygwin', 'msys']: - possible_dll_names.append('botan-3.dll') - possible_dll_names.append('libbotan-3.dll') - possible_dll_names.append('botan.dll') - elif platform in ['darwin', 'macos']: - possible_dll_names.append('libbotan-3.dylib') + if platform in ["win32", "cygwin", "msys"]: + possible_dll_names.append("botan-3.dll") + possible_dll_names.append("libbotan-3.dll") + possible_dll_names.append("botan.dll") + elif platform in ["darwin", "macos"]: + possible_dll_names.append("libbotan-3.dylib") else: # assumed to be some Unix/Linux system - possible_dll_names.append('libbotan-3.so') + possible_dll_names.append("libbotan-3.so") - min_minor = 8 # minimum supported FFI - max_minor = 32 # arbitrary but probably large enough - possible_dll_names += ['libbotan-3.so.%d' % (v) for v in reversed(range(min_minor, max_minor))] + min_minor = 8 # minimum supported FFI + max_minor = 32 # arbitrary but probably large enough + possible_dll_names += [ + "libbotan-3.so.%d" % (v) for v in reversed(range(min_minor, max_minor)) + ] for dll_name in possible_dll_names: try: dll = CDLL(dll_name) - if hasattr(dll, 'botan_ffi_supports_api'): + if hasattr(dll, "botan_ffi_supports_api"): dll.botan_ffi_supports_api.argtypes = [c_uint32] dll.botan_ffi_supports_api.restype = c_int if dll.botan_ffi_supports_api(expected_version) == 0: @@ -92,16 +110,19 @@ def _load_botan_dll(expected_version): raise BotanException("Could not find a usable Botan shared object library") + VIEW_BIN_CALLBACK = CFUNCTYPE(c_int, c_void_p, POINTER(c_char), c_size_t) VIEW_STR_CALLBACK = CFUNCTYPE(c_int, c_void_p, c_char_p, c_size_t) + def _errcheck(rc, fn, _args): # This errcheck should only be used for int-returning functions assert isinstance(rc, int) if rc >= 0 or rc in fn.allowed_errors: return rc - raise BotanException('%s failed' % (fn.__name__), rc) + raise BotanException("%s failed" % (fn.__name__), rc) + def _set_prototypes(dll): def ffi_api(fn, args, allowed_errors=None): @@ -174,7 +195,10 @@ def ffi_api(fn, args, allowed_errors=None): ffi_api(dll.botan_mac_final, [c_void_p, c_char_p]) ffi_api(dll.botan_mac_clear, [c_void_p]) ffi_api(dll.botan_mac_name, [c_void_p, c_char_p, POINTER(c_size_t)]) - ffi_api(dll.botan_mac_get_keyspec, [c_void_p, POINTER(c_size_t), POINTER(c_size_t), POINTER(c_size_t)]) + ffi_api( + dll.botan_mac_get_keyspec, + [c_void_p, POINTER(c_size_t), POINTER(c_size_t), POINTER(c_size_t)], + ) ffi_api(dll.botan_mac_destroy, [c_void_p]) # CIPHER @@ -187,33 +211,114 @@ def ffi_api(fn, args, allowed_errors=None): ffi_api(dll.botan_cipher_is_authenticated, [c_void_p]) ffi_api(dll.botan_cipher_requires_entire_message, [c_void_p]) ffi_api(dll.botan_cipher_get_update_granularity, [c_void_p, POINTER(c_size_t)]) - ffi_api(dll.botan_cipher_get_ideal_update_granularity, [c_void_p, POINTER(c_size_t)]) - ffi_api(dll.botan_cipher_query_keylen, [c_void_p, POINTER(c_size_t), POINTER(c_size_t)]) - ffi_api(dll.botan_cipher_get_keyspec, [c_void_p, POINTER(c_size_t), POINTER(c_size_t), POINTER(c_size_t)]) + ffi_api( + dll.botan_cipher_get_ideal_update_granularity, [c_void_p, POINTER(c_size_t)] + ) + ffi_api( + dll.botan_cipher_query_keylen, [c_void_p, POINTER(c_size_t), POINTER(c_size_t)] + ) + ffi_api( + dll.botan_cipher_get_keyspec, + [c_void_p, POINTER(c_size_t), POINTER(c_size_t), POINTER(c_size_t)], + ) ffi_api(dll.botan_cipher_set_key, [c_void_p, c_char_p, c_size_t]) ffi_api(dll.botan_cipher_reset, [c_void_p]) ffi_api(dll.botan_cipher_set_associated_data, [c_void_p, c_char_p, c_size_t]) ffi_api(dll.botan_cipher_start, [c_void_p, c_char_p, c_size_t]) - ffi_api(dll.botan_cipher_update, - [c_void_p, c_uint32, c_char_p, c_size_t, POINTER(c_size_t), c_char_p, c_size_t, POINTER(c_size_t)]) + ffi_api( + dll.botan_cipher_update, + [ + c_void_p, + c_uint32, + c_char_p, + c_size_t, + POINTER(c_size_t), + c_char_p, + c_size_t, + POINTER(c_size_t), + ], + ) ffi_api(dll.botan_cipher_clear, [c_void_p]) ffi_api(dll.botan_cipher_destroy, [c_void_p]) - ffi_api(dll.botan_pbkdf, - [c_char_p, c_char_p, c_size_t, c_char_p, c_char_p, c_size_t, c_size_t]) - ffi_api(dll.botan_pbkdf_timed, - [c_char_p, c_char_p, c_size_t, c_char_p, c_char_p, c_size_t, c_size_t, POINTER(c_size_t)]) + ffi_api( + dll.botan_pbkdf, + [c_char_p, c_char_p, c_size_t, c_char_p, c_char_p, c_size_t, c_size_t], + ) + ffi_api( + dll.botan_pbkdf_timed, + [ + c_char_p, + c_char_p, + c_size_t, + c_char_p, + c_char_p, + c_size_t, + c_size_t, + POINTER(c_size_t), + ], + ) - ffi_api(dll.botan_pwdhash, - [c_char_p, c_size_t, c_size_t, c_size_t, c_char_p, c_size_t, c_char_p, c_size_t, c_char_p, c_size_t]) - ffi_api(dll.botan_pwdhash_timed, - [c_char_p, c_uint32, POINTER(c_size_t), POINTER(c_size_t), POINTER(c_size_t), c_char_p, c_size_t, c_char_p, c_size_t, c_char_p, c_size_t]) + ffi_api( + dll.botan_pwdhash, + [ + c_char_p, + c_size_t, + c_size_t, + c_size_t, + c_char_p, + c_size_t, + c_char_p, + c_size_t, + c_char_p, + c_size_t, + ], + ) + ffi_api( + dll.botan_pwdhash_timed, + [ + c_char_p, + c_uint32, + POINTER(c_size_t), + POINTER(c_size_t), + POINTER(c_size_t), + c_char_p, + c_size_t, + c_char_p, + c_size_t, + c_char_p, + c_size_t, + ], + ) - ffi_api(dll.botan_scrypt, - [c_char_p, c_size_t, c_char_p, c_char_p, c_size_t, c_size_t, c_size_t, c_size_t]) + ffi_api( + dll.botan_scrypt, + [ + c_char_p, + c_size_t, + c_char_p, + c_char_p, + c_size_t, + c_size_t, + c_size_t, + c_size_t, + ], + ) - ffi_api(dll.botan_kdf, - [c_char_p, c_char_p, c_size_t, c_char_p, c_size_t, c_char_p, c_size_t, c_char_p, c_size_t]) + ffi_api( + dll.botan_kdf, + [ + c_char_p, + c_char_p, + c_size_t, + c_char_p, + c_size_t, + c_char_p, + c_size_t, + c_char_p, + c_size_t, + ], + ) # BLOCK ffi_api(dll.botan_block_cipher_init, [c_void_p, c_char_p]) @@ -221,10 +326,17 @@ def ffi_api(fn, args, allowed_errors=None): ffi_api(dll.botan_block_cipher_clear, [c_void_p]) ffi_api(dll.botan_block_cipher_set_key, [c_void_p, c_char_p, c_size_t]) ffi_api(dll.botan_block_cipher_block_size, [c_void_p]) - ffi_api(dll.botan_block_cipher_encrypt_blocks, [c_void_p, c_char_p, c_char_p, c_size_t]) - ffi_api(dll.botan_block_cipher_decrypt_blocks, [c_void_p, c_char_p, c_char_p, c_size_t]) + ffi_api( + dll.botan_block_cipher_encrypt_blocks, [c_void_p, c_char_p, c_char_p, c_size_t] + ) + ffi_api( + dll.botan_block_cipher_decrypt_blocks, [c_void_p, c_char_p, c_char_p, c_size_t] + ) ffi_api(dll.botan_block_cipher_name, [c_void_p, c_char_p, POINTER(c_size_t)]) - ffi_api(dll.botan_block_cipher_get_keyspec, [c_void_p, POINTER(c_size_t), POINTER(c_size_t), POINTER(c_size_t)]) + ffi_api( + dll.botan_block_cipher_get_keyspec, + [c_void_p, POINTER(c_size_t), POINTER(c_size_t), POINTER(c_size_t)], + ) # MP ffi_api(dll.botan_mp_init, [c_void_p]) @@ -269,8 +381,10 @@ def ffi_api(fn, args, allowed_errors=None): ffi_api(dll.botan_mp_set_bit, [c_void_p, c_size_t]) ffi_api(dll.botan_mp_clear_bit, [c_void_p, c_size_t]) - ffi_api(dll.botan_bcrypt_generate, - [c_char_p, POINTER(c_size_t), c_char_p, c_void_p, c_size_t, c_uint32]) + ffi_api( + dll.botan_bcrypt_generate, + [c_char_p, POINTER(c_size_t), c_char_p, c_void_p, c_size_t, c_uint32], + ) ffi_api(dll.botan_bcrypt_is_valid, [c_char_p, c_char_p]) # OID @@ -286,8 +400,19 @@ def ffi_api(fn, args, allowed_errors=None): ffi_api(dll.botan_ec_group_destroy, [c_void_p]) ffi_api(dll.botan_ec_group_supports_application_specific_group, [POINTER(c_int)]) ffi_api(dll.botan_ec_group_supports_named_group, [c_char_p, POINTER(c_int)]) - ffi_api(dll.botan_ec_group_from_params, - [c_void_p, c_void_p, c_void_p, c_void_p, c_void_p, c_void_p, c_void_p, c_void_p]) + ffi_api( + dll.botan_ec_group_from_params, + [ + c_void_p, + c_void_p, + c_void_p, + c_void_p, + c_void_p, + c_void_p, + c_void_p, + c_void_p, + ], + ) ffi_api(dll.botan_ec_group_from_ber, [c_void_p, c_char_p, c_size_t]) ffi_api(dll.botan_ec_group_from_pem, [c_void_p, c_char_p]) ffi_api(dll.botan_ec_group_from_oid, [c_void_p, c_void_p]) @@ -314,8 +439,7 @@ def ffi_api(fn, args, allowed_errors=None): ffi_api(dll.botan_privkey_create_dh, [c_void_p, c_void_p, c_char_p]) ffi_api(dll.botan_privkey_create_dsa, [c_void_p, c_void_p, c_size_t, c_size_t]) ffi_api(dll.botan_privkey_create_elgamal, [c_void_p, c_void_p, c_size_t, c_size_t]) - ffi_api(dll.botan_privkey_load, - [c_void_p, c_void_p, c_char_p, c_size_t, c_char_p]) + ffi_api(dll.botan_privkey_load, [c_void_p, c_void_p, c_char_p, c_size_t, c_char_p]) ffi_api(dll.botan_privkey_destroy, [c_void_p]) ffi_api(dll.botan_privkey_view_der, [c_void_p, c_void_p, VIEW_BIN_CALLBACK]) @@ -323,23 +447,94 @@ def ffi_api(fn, args, allowed_errors=None): ffi_api(dll.botan_privkey_view_raw, [c_void_p, c_void_p, VIEW_BIN_CALLBACK]) ffi_api(dll.botan_privkey_algo_name, [c_void_p, c_char_p, POINTER(c_size_t)]) - ffi_api(dll.botan_privkey_export_encrypted, - [c_void_p, c_char_p, POINTER(c_size_t), c_void_p, c_char_p, c_char_p, c_uint32]) + ffi_api( + dll.botan_privkey_export_encrypted, + [c_void_p, c_char_p, POINTER(c_size_t), c_void_p, c_char_p, c_char_p, c_uint32], + ) - ffi_api(dll.botan_privkey_export_encrypted_pbkdf_msec, - [c_void_p, c_char_p, POINTER(c_size_t), c_void_p, c_char_p, c_uint32, POINTER(c_size_t), c_char_p, c_char_p, c_uint32]) - ffi_api(dll.botan_privkey_export_encrypted_pbkdf_iter, - [c_void_p, c_char_p, POINTER(c_size_t), c_void_p, c_char_p, c_size_t, c_char_p, c_char_p, c_uint32]) + ffi_api( + dll.botan_privkey_export_encrypted_pbkdf_msec, + [ + c_void_p, + c_char_p, + POINTER(c_size_t), + c_void_p, + c_char_p, + c_uint32, + POINTER(c_size_t), + c_char_p, + c_char_p, + c_uint32, + ], + ) + ffi_api( + dll.botan_privkey_export_encrypted_pbkdf_iter, + [ + c_void_p, + c_char_p, + POINTER(c_size_t), + c_void_p, + c_char_p, + c_size_t, + c_char_p, + c_char_p, + c_uint32, + ], + ) - ffi_api(dll.botan_privkey_view_encrypted_der, - [c_void_p, c_void_p, c_char_p, c_char_p, c_char_p, c_size_t, c_void_p, VIEW_BIN_CALLBACK]) - ffi_api(dll.botan_privkey_view_encrypted_pem, - [c_void_p, c_void_p, c_char_p, c_char_p, c_char_p, c_size_t, c_void_p, VIEW_STR_CALLBACK]) + ffi_api( + dll.botan_privkey_view_encrypted_der, + [ + c_void_p, + c_void_p, + c_char_p, + c_char_p, + c_char_p, + c_size_t, + c_void_p, + VIEW_BIN_CALLBACK, + ], + ) + ffi_api( + dll.botan_privkey_view_encrypted_pem, + [ + c_void_p, + c_void_p, + c_char_p, + c_char_p, + c_char_p, + c_size_t, + c_void_p, + VIEW_STR_CALLBACK, + ], + ) - ffi_api(dll.botan_privkey_view_encrypted_der_timed, - [c_void_p, c_void_p, c_char_p, c_char_p, c_char_p, c_size_t, c_void_p, VIEW_BIN_CALLBACK]) - ffi_api(dll.botan_privkey_view_encrypted_pem_timed, - [c_void_p, c_void_p, c_char_p, c_char_p, c_char_p, c_size_t, c_void_p, VIEW_STR_CALLBACK]) + ffi_api( + dll.botan_privkey_view_encrypted_der_timed, + [ + c_void_p, + c_void_p, + c_char_p, + c_char_p, + c_char_p, + c_size_t, + c_void_p, + VIEW_BIN_CALLBACK, + ], + ) + ffi_api( + dll.botan_privkey_view_encrypted_pem_timed, + [ + c_void_p, + c_void_p, + c_char_p, + c_char_p, + c_char_p, + c_size_t, + c_void_p, + VIEW_STR_CALLBACK, + ], + ) ffi_api(dll.botan_privkey_export_pubkey, [c_void_p, c_void_p]) ffi_api(dll.botan_pubkey_load, [c_void_p, c_char_p, c_size_t]) @@ -351,7 +546,9 @@ def ffi_api(fn, args, allowed_errors=None): ffi_api(dll.botan_pubkey_algo_name, [c_void_p, c_char_p, POINTER(c_size_t)]) ffi_api(dll.botan_pubkey_check_key, [c_void_p, c_void_p, c_uint32], [-1]) ffi_api(dll.botan_pubkey_estimated_strength, [c_void_p, POINTER(c_size_t)]) - ffi_api(dll.botan_pubkey_fingerprint, [c_void_p, c_char_p, c_char_p, POINTER(c_size_t)]) + ffi_api( + dll.botan_pubkey_fingerprint, [c_void_p, c_char_p, c_char_p, POINTER(c_size_t)] + ) ffi_api(dll.botan_pubkey_destroy, [c_void_p]) ffi_api(dll.botan_pubkey_get_field, [c_void_p, c_void_p, c_char_p]) ffi_api(dll.botan_privkey_get_field, [c_void_p, c_void_p, c_char_p]) @@ -366,14 +563,19 @@ def ffi_api(fn, args, allowed_errors=None): ffi_api(dll.botan_privkey_rsa_get_d, [c_void_p, c_void_p]) ffi_api(dll.botan_privkey_rsa_get_n, [c_void_p, c_void_p]) ffi_api(dll.botan_privkey_rsa_get_e, [c_void_p, c_void_p]) - ffi_api(dll.botan_privkey_rsa_get_privkey, [c_void_p, c_char_p, POINTER(c_size_t), c_uint32]) + ffi_api( + dll.botan_privkey_rsa_get_privkey, + [c_void_p, c_char_p, POINTER(c_size_t), c_uint32], + ) ffi_api(dll.botan_pubkey_load_rsa, [c_void_p, c_void_p, c_void_p]) ffi_api(dll.botan_pubkey_rsa_get_e, [c_void_p, c_void_p]) ffi_api(dll.botan_pubkey_rsa_get_n, [c_void_p, c_void_p]) - ffi_api(dll.botan_privkey_load_dsa, - [c_void_p, c_void_p, c_void_p, c_void_p, c_void_p]) - ffi_api(dll.botan_pubkey_load_dsa, - [c_void_p, c_void_p, c_void_p, c_void_p, c_void_p]) + ffi_api( + dll.botan_privkey_load_dsa, [c_void_p, c_void_p, c_void_p, c_void_p, c_void_p] + ) + ffi_api( + dll.botan_pubkey_load_dsa, [c_void_p, c_void_p, c_void_p, c_void_p, c_void_p] + ) ffi_api(dll.botan_privkey_dsa_get_x, [c_void_p, c_void_p]) ffi_api(dll.botan_pubkey_dsa_get_p, [c_void_p, c_void_p]) ffi_api(dll.botan_pubkey_dsa_get_q, [c_void_p, c_void_p]) @@ -405,14 +607,22 @@ def ffi_api(fn, args, allowed_errors=None): ffi_api(dll.botan_pubkey_load_slh_dsa, [c_void_p, c_void_p, c_int, c_char_p]) ffi_api(dll.botan_privkey_load_kyber, [c_void_p, c_char_p, c_int]) ffi_api(dll.botan_pubkey_load_kyber, [c_void_p, c_char_p, c_int]) - ffi_api(dll.botan_privkey_view_kyber_raw_key, [c_void_p, c_void_p, VIEW_BIN_CALLBACK]) - ffi_api(dll.botan_pubkey_view_kyber_raw_key, [c_void_p, c_void_p, VIEW_BIN_CALLBACK]) + ffi_api( + dll.botan_privkey_view_kyber_raw_key, [c_void_p, c_void_p, VIEW_BIN_CALLBACK] + ) + ffi_api( + dll.botan_pubkey_view_kyber_raw_key, [c_void_p, c_void_p, VIEW_BIN_CALLBACK] + ) ffi_api(dll.botan_privkey_load_ml_kem, [c_void_p, c_void_p, c_int, c_char_p]) ffi_api(dll.botan_pubkey_load_ml_kem, [c_void_p, c_void_p, c_int, c_char_p]) ffi_api(dll.botan_privkey_load_frodokem, [c_void_p, c_void_p, c_int, c_char_p]) ffi_api(dll.botan_pubkey_load_frodokem, [c_void_p, c_void_p, c_int, c_char_p]) - ffi_api(dll.botan_privkey_load_classic_mceliece, [c_void_p, c_void_p, c_int, c_char_p]) - ffi_api(dll.botan_pubkey_load_classic_mceliece, [c_void_p, c_void_p, c_int, c_char_p]) + ffi_api( + dll.botan_privkey_load_classic_mceliece, [c_void_p, c_void_p, c_int, c_char_p] + ) + ffi_api( + dll.botan_pubkey_load_classic_mceliece, [c_void_p, c_void_p, c_int, c_char_p] + ) ffi_api(dll.botan_privkey_load_ecdsa, [c_void_p, c_void_p, c_char_p]) ffi_api(dll.botan_pubkey_load_ecdsa, [c_void_p, c_void_p, c_void_p, c_char_p]) ffi_api(dll.botan_pubkey_load_ecdsa_sec1, [c_void_p, c_void_p, c_size_t, c_char_p]) @@ -424,83 +634,168 @@ def ffi_api(fn, args, allowed_errors=None): ffi_api(dll.botan_privkey_load_sm2, [c_void_p, c_void_p, c_char_p]) ffi_api(dll.botan_pubkey_load_sm2_enc, [c_void_p, c_void_p, c_void_p, c_char_p]) ffi_api(dll.botan_privkey_load_sm2_enc, [c_void_p, c_void_p, c_char_p]) - ffi_api(dll.botan_pubkey_sm2_compute_za, - [c_char_p, POINTER(c_size_t), c_char_p, c_char_p, c_void_p]) - ffi_api(dll.botan_pubkey_view_ec_public_point, - [c_void_p, c_void_p, VIEW_BIN_CALLBACK]) + ffi_api( + dll.botan_pubkey_sm2_compute_za, + [c_char_p, POINTER(c_size_t), c_char_p, c_char_p, c_void_p], + ) + ffi_api( + dll.botan_pubkey_view_ec_public_point, [c_void_p, c_void_p, VIEW_BIN_CALLBACK] + ) # PK ffi_api(dll.botan_pk_op_encrypt_create, [c_void_p, c_void_p, c_char_p, c_uint32]) ffi_api(dll.botan_pk_op_encrypt_destroy, [c_void_p]) - ffi_api(dll.botan_pk_op_encrypt_output_length, [c_void_p, c_size_t, POINTER(c_size_t)]) - ffi_api(dll.botan_pk_op_encrypt, - [c_void_p, c_void_p, c_char_p, POINTER(c_size_t), c_char_p, c_size_t]) + ffi_api( + dll.botan_pk_op_encrypt_output_length, [c_void_p, c_size_t, POINTER(c_size_t)] + ) + ffi_api( + dll.botan_pk_op_encrypt, + [c_void_p, c_void_p, c_char_p, POINTER(c_size_t), c_char_p, c_size_t], + ) ffi_api(dll.botan_pk_op_decrypt_create, [c_void_p, c_void_p, c_char_p, c_uint32]) ffi_api(dll.botan_pk_op_decrypt_destroy, [c_void_p]) - ffi_api(dll.botan_pk_op_decrypt_output_length, [c_void_p, c_size_t, POINTER(c_size_t)]) - ffi_api(dll.botan_pk_op_decrypt, - [c_void_p, c_char_p, POINTER(c_size_t), c_char_p, c_size_t]) + ffi_api( + dll.botan_pk_op_decrypt_output_length, [c_void_p, c_size_t, POINTER(c_size_t)] + ) + ffi_api( + dll.botan_pk_op_decrypt, + [c_void_p, c_char_p, POINTER(c_size_t), c_char_p, c_size_t], + ) ffi_api(dll.botan_pk_op_sign_create, [c_void_p, c_void_p, c_char_p, c_uint32]) ffi_api(dll.botan_pk_op_sign_destroy, [c_void_p]) ffi_api(dll.botan_pk_op_sign_output_length, [c_void_p, POINTER(c_size_t)]) ffi_api(dll.botan_pk_op_sign_update, [c_void_p, c_char_p, c_size_t]) - ffi_api(dll.botan_pk_op_sign_finish, [c_void_p, c_void_p, c_char_p, POINTER(c_size_t)]) + ffi_api( + dll.botan_pk_op_sign_finish, [c_void_p, c_void_p, c_char_p, POINTER(c_size_t)] + ) ffi_api(dll.botan_pk_op_verify_create, [c_void_p, c_void_p, c_char_p, c_uint32]) ffi_api(dll.botan_pk_op_verify_destroy, [c_void_p]) ffi_api(dll.botan_pk_op_verify_update, [c_void_p, c_char_p, c_size_t]) ffi_api(dll.botan_pk_op_verify_finish, [c_void_p, c_char_p, c_size_t]) - ffi_api(dll.botan_pk_op_key_agreement_create, [c_void_p, c_void_p, c_char_p, c_uint32]) + ffi_api( + dll.botan_pk_op_key_agreement_create, [c_void_p, c_void_p, c_char_p, c_uint32] + ) ffi_api(dll.botan_pk_op_key_agreement_destroy, [c_void_p]) - ffi_api(dll.botan_pk_op_key_agreement_view_public, [c_void_p, c_void_p, VIEW_BIN_CALLBACK]) + ffi_api( + dll.botan_pk_op_key_agreement_view_public, + [c_void_p, c_void_p, VIEW_BIN_CALLBACK], + ) ffi_api(dll.botan_pk_op_key_agreement_size, [c_void_p, POINTER(c_size_t)]) - ffi_api(dll.botan_pk_op_key_agreement, - [c_void_p, c_char_p, POINTER(c_size_t), c_char_p, c_size_t, c_char_p, c_size_t]) + ffi_api( + dll.botan_pk_op_key_agreement, + [c_void_p, c_char_p, POINTER(c_size_t), c_char_p, c_size_t, c_char_p, c_size_t], + ) ffi_api(dll.botan_pkcs_hash_id, [c_char_p, c_char_p, POINTER(c_size_t)]) ffi_api(dll.botan_pk_op_kem_encrypt_create, [c_void_p, c_void_p, c_char_p]) ffi_api(dll.botan_pk_op_kem_encrypt_destroy, [c_void_p]) - ffi_api(dll.botan_pk_op_kem_encrypt_shared_key_length, [c_void_p, c_size_t, POINTER(c_size_t)]) - ffi_api(dll.botan_pk_op_kem_encrypt_encapsulated_key_length, [c_void_p, POINTER(c_size_t)]) + ffi_api( + dll.botan_pk_op_kem_encrypt_shared_key_length, + [c_void_p, c_size_t, POINTER(c_size_t)], + ) + ffi_api( + dll.botan_pk_op_kem_encrypt_encapsulated_key_length, + [c_void_p, POINTER(c_size_t)], + ) - ffi_api(dll.botan_pk_op_kem_encrypt_create_shared_key, - [c_void_p, c_void_p, c_char_p, c_size_t, c_size_t, - c_char_p, POINTER(c_size_t), c_char_p, POINTER(c_size_t)]) + ffi_api( + dll.botan_pk_op_kem_encrypt_create_shared_key, + [ + c_void_p, + c_void_p, + c_char_p, + c_size_t, + c_size_t, + c_char_p, + POINTER(c_size_t), + c_char_p, + POINTER(c_size_t), + ], + ) ffi_api(dll.botan_pk_op_kem_decrypt_create, [c_void_p, c_void_p, c_char_p]) ffi_api(dll.botan_pk_op_kem_decrypt_destroy, [c_void_p]) - ffi_api(dll.botan_pk_op_kem_decrypt_shared_key_length, [c_void_p, c_size_t, POINTER(c_size_t)]) + ffi_api( + dll.botan_pk_op_kem_decrypt_shared_key_length, + [c_void_p, c_size_t, POINTER(c_size_t)], + ) - ffi_api(dll.botan_pk_op_kem_decrypt_shared_key, - [c_void_p, c_char_p, c_size_t, c_char_p, c_size_t, c_size_t, c_char_p, POINTER(c_size_t)]) + ffi_api( + dll.botan_pk_op_kem_decrypt_shared_key, + [ + c_void_p, + c_char_p, + c_size_t, + c_char_p, + c_size_t, + c_size_t, + c_char_p, + POINTER(c_size_t), + ], + ) # X509 ffi_api(dll.botan_x509_cert_load, [c_void_p, c_char_p, c_size_t]) ffi_api(dll.botan_x509_cert_load_file, [c_void_p, c_char_p]) ffi_api(dll.botan_x509_cert_destroy, [c_void_p]) ffi_api(dll.botan_x509_cert_dup, [c_void_p, c_void_p]) - ffi_api(dll.botan_x509_cert_get_time_starts, [c_void_p, c_char_p, POINTER(c_size_t)]) - ffi_api(dll.botan_x509_cert_get_time_expires, [c_void_p, c_char_p, POINTER(c_size_t)]) + ffi_api( + dll.botan_x509_cert_get_time_starts, [c_void_p, c_char_p, POINTER(c_size_t)] + ) + ffi_api( + dll.botan_x509_cert_get_time_expires, [c_void_p, c_char_p, POINTER(c_size_t)] + ) ffi_api(dll.botan_x509_cert_not_before, [c_void_p, POINTER(c_uint64)]) ffi_api(dll.botan_x509_cert_not_after, [c_void_p, POINTER(c_uint64)]) - ffi_api(dll.botan_x509_cert_get_fingerprint, [c_void_p, c_char_p, c_char_p, POINTER(c_size_t)]) - ffi_api(dll.botan_x509_cert_get_serial_number, [c_void_p, c_char_p, POINTER(c_size_t)]) - ffi_api(dll.botan_x509_cert_get_authority_key_id, [c_void_p, c_char_p, POINTER(c_size_t)]) - ffi_api(dll.botan_x509_cert_get_subject_key_id, [c_void_p, c_char_p, POINTER(c_size_t)]) - ffi_api(dll.botan_x509_cert_view_public_key_bits, [c_void_p, c_void_p, VIEW_BIN_CALLBACK]) + ffi_api( + dll.botan_x509_cert_get_fingerprint, + [c_void_p, c_char_p, c_char_p, POINTER(c_size_t)], + ) + ffi_api( + dll.botan_x509_cert_get_serial_number, [c_void_p, c_char_p, POINTER(c_size_t)] + ) + ffi_api( + dll.botan_x509_cert_get_authority_key_id, + [c_void_p, c_char_p, POINTER(c_size_t)], + ) + ffi_api( + dll.botan_x509_cert_get_subject_key_id, [c_void_p, c_char_p, POINTER(c_size_t)] + ) + ffi_api( + dll.botan_x509_cert_view_public_key_bits, + [c_void_p, c_void_p, VIEW_BIN_CALLBACK], + ) ffi_api(dll.botan_x509_cert_get_public_key, [c_void_p, c_void_p]) - ffi_api(dll.botan_x509_cert_get_issuer_dn, - [c_void_p, c_char_p, c_size_t, c_char_p, POINTER(c_size_t)]) - ffi_api(dll.botan_x509_cert_get_subject_dn, - [c_void_p, c_char_p, c_size_t, c_char_p, POINTER(c_size_t)]) + ffi_api( + dll.botan_x509_cert_get_issuer_dn, + [c_void_p, c_char_p, c_size_t, c_char_p, POINTER(c_size_t)], + ) + ffi_api( + dll.botan_x509_cert_get_subject_dn, + [c_void_p, c_char_p, c_size_t, c_char_p, POINTER(c_size_t)], + ) ffi_api(dll.botan_x509_cert_view_as_string, [c_void_p, c_void_p, VIEW_STR_CALLBACK]) ffi_api(dll.botan_x509_cert_allowed_usage, [c_void_p, c_uint]) ffi_api(dll.botan_x509_cert_hostname_match, [c_void_p, c_char_p], [-1]) - ffi_api(dll.botan_x509_cert_verify, - [POINTER(c_int), c_void_p, c_void_p, c_size_t, c_void_p, c_size_t, c_char_p, c_size_t, c_char_p, c_uint64]) + ffi_api( + dll.botan_x509_cert_verify, + [ + POINTER(c_int), + c_void_p, + c_void_p, + c_size_t, + c_void_p, + c_size_t, + c_char_p, + c_size_t, + c_char_p, + c_uint64, + ], + ) dll.botan_x509_cert_validation_status.argtypes = [c_int] dll.botan_x509_cert_validation_status.restype = c_char_p @@ -510,32 +805,74 @@ def ffi_api(fn, args, allowed_errors=None): ffi_api(dll.botan_x509_crl_load_file, [c_void_p, c_char_p]) ffi_api(dll.botan_x509_crl_destroy, [c_void_p]) ffi_api(dll.botan_x509_is_revoked, [c_void_p, c_void_p], [-1]) - ffi_api(dll.botan_x509_cert_verify_with_crl, - [POINTER(c_int), c_void_p, c_void_p, c_size_t, c_void_p, c_size_t, c_void_p, c_size_t, c_char_p, c_size_t, c_char_p, c_uint64]) + ffi_api( + dll.botan_x509_cert_verify_with_crl, + [ + POINTER(c_int), + c_void_p, + c_void_p, + c_size_t, + c_void_p, + c_size_t, + c_void_p, + c_size_t, + c_char_p, + c_size_t, + c_char_p, + c_uint64, + ], + ) - ffi_api(dll.botan_nist_kw_enc, - [c_char_p, c_int, c_char_p, c_size_t, c_char_p, c_size_t, c_char_p, POINTER(c_size_t)]) - ffi_api(dll.botan_nist_kw_dec, - [c_char_p, c_int, c_char_p, c_size_t, c_char_p, c_size_t, c_char_p, POINTER(c_size_t)]) + ffi_api( + dll.botan_nist_kw_enc, + [ + c_char_p, + c_int, + c_char_p, + c_size_t, + c_char_p, + c_size_t, + c_char_p, + POINTER(c_size_t), + ], + ) + ffi_api( + dll.botan_nist_kw_dec, + [ + c_char_p, + c_int, + c_char_p, + c_size_t, + c_char_p, + c_size_t, + c_char_p, + POINTER(c_size_t), + ], + ) # HOTP - ffi_api(dll.botan_hotp_init, - [c_void_p, c_char_p, c_size_t, c_char_p, c_size_t]) + ffi_api(dll.botan_hotp_init, [c_void_p, c_char_p, c_size_t, c_char_p, c_size_t]) ffi_api(dll.botan_hotp_destroy, [c_void_p]) ffi_api(dll.botan_hotp_generate, [c_void_p, POINTER(c_uint32), c_uint64]) - ffi_api(dll.botan_hotp_check, - [c_void_p, POINTER(c_uint64), c_uint32, c_uint64, c_size_t]) + ffi_api( + dll.botan_hotp_check, + [c_void_p, POINTER(c_uint64), c_uint32, c_uint64, c_size_t], + ) # TOTP - ffi_api(dll.botan_totp_init, - [c_void_p, c_char_p, c_size_t, c_char_p, c_size_t, c_size_t]) + ffi_api( + dll.botan_totp_init, + [c_void_p, c_char_p, c_size_t, c_char_p, c_size_t, c_size_t], + ) ffi_api(dll.botan_totp_destroy, [c_void_p]) ffi_api(dll.botan_totp_generate, [c_void_p, POINTER(c_uint32), c_uint64]) ffi_api(dll.botan_totp_check, [c_void_p, c_uint32, c_uint64, c_size_t]) # FPE - ffi_api(dll.botan_fpe_fe1_init, - [c_void_p, c_void_p, c_char_p, c_size_t, c_size_t, c_uint32]) + ffi_api( + dll.botan_fpe_fe1_init, + [c_void_p, c_void_p, c_char_p, c_size_t, c_size_t, c_uint32], + ) ffi_api(dll.botan_fpe_destroy, [c_void_p]) ffi_api(dll.botan_fpe_encrypt, [c_void_p, c_void_p, c_char_p, c_size_t]) ffi_api(dll.botan_fpe_decrypt, [c_void_p, c_void_p, c_char_p, c_size_t]) @@ -543,22 +880,72 @@ def ffi_api(fn, args, allowed_errors=None): # SRP6-a ffi_api(dll.botan_srp6_server_session_init, [c_void_p]) ffi_api(dll.botan_srp6_server_session_destroy, [c_void_p]) - ffi_api(dll.botan_srp6_server_session_step1, - [c_void_p, c_char_p, c_size_t, c_char_p, c_char_p, c_void_p, c_char_p, POINTER(c_size_t)]) - ffi_api(dll.botan_srp6_server_session_step2, - [c_void_p, c_char_p, c_size_t, c_char_p, POINTER(c_size_t)]) - ffi_api(dll.botan_srp6_generate_verifier, - [c_char_p, c_char_p, c_char_p, c_size_t, c_char_p, c_char_p, c_char_p, POINTER(c_size_t)]) - ffi_api(dll.botan_srp6_client_agree, - [c_char_p, c_char_p, c_char_p, c_char_p, c_char_p, c_size_t, c_char_p, c_size_t, c_void_p, - c_char_p, POINTER(c_size_t), c_char_p, POINTER(c_size_t)]) + ffi_api( + dll.botan_srp6_server_session_step1, + [ + c_void_p, + c_char_p, + c_size_t, + c_char_p, + c_char_p, + c_void_p, + c_char_p, + POINTER(c_size_t), + ], + ) + ffi_api( + dll.botan_srp6_server_session_step2, + [c_void_p, c_char_p, c_size_t, c_char_p, POINTER(c_size_t)], + ) + ffi_api( + dll.botan_srp6_generate_verifier, + [ + c_char_p, + c_char_p, + c_char_p, + c_size_t, + c_char_p, + c_char_p, + c_char_p, + POINTER(c_size_t), + ], + ) + ffi_api( + dll.botan_srp6_client_agree, + [ + c_char_p, + c_char_p, + c_char_p, + c_char_p, + c_char_p, + c_size_t, + c_char_p, + c_size_t, + c_void_p, + c_char_p, + POINTER(c_size_t), + c_char_p, + POINTER(c_size_t), + ], + ) ffi_api(dll.botan_srp6_group_size, [c_char_p, POINTER(c_size_t)]) # ZFEC - ffi_api(dll.botan_zfec_encode, - [c_size_t, c_size_t, c_char_p, c_size_t, POINTER(c_char_p)]) - ffi_api(dll.botan_zfec_decode, - [c_size_t, c_size_t, POINTER(c_size_t), POINTER(c_char_p), c_size_t, POINTER(c_char_p)]) + ffi_api( + dll.botan_zfec_encode, + [c_size_t, c_size_t, c_char_p, c_size_t, POINTER(c_char_p)], + ) + ffi_api( + dll.botan_zfec_decode, + [ + c_size_t, + c_size_t, + POINTER(c_size_t), + POINTER(c_char_p), + c_size_t, + POINTER(c_char_p), + ], + ) # TPM2 ffi_api(dll.botan_tpm2_supports_crypto_backend, []) @@ -572,11 +959,13 @@ def ffi_api(fn, args, allowed_errors=None): return dll + # # Load the DLL and set prototypes on it # _DLL = _set_prototypes(_load_botan_dll(BOTAN_FFI_VERSION)) + # # Internal utilities # @@ -585,8 +974,8 @@ def _call_fn_returning_sz(fn) -> int: fn(byref(sz)) return int(sz.value) -def _call_fn_returning_vec(guess, fn) -> bytes: +def _call_fn_returning_vec(guess, fn) -> bytes: buf = create_string_buffer(guess) buf_len = c_size_t(len(buf)) @@ -595,10 +984,10 @@ def _call_fn_returning_vec(guess, fn) -> bytes: return _call_fn_returning_vec(buf_len.value, fn) assert buf_len.value <= len(buf) - return buf.raw[0:int(buf_len.value)] + return buf.raw[0 : int(buf_len.value)] -def _call_fn_returning_vec_pair(guess1, guess2, fn) -> tuple[bytes, bytes]: +def _call_fn_returning_vec_pair(guess1, guess2, fn) -> tuple[bytes, bytes]: buf1 = create_string_buffer(guess1) buf1_len = c_size_t(len(buf1)) @@ -615,58 +1004,71 @@ def _call_fn_returning_vec_pair(guess1, guess2, fn) -> tuple[bytes, bytes]: assert buf1_len.value <= len(buf1) assert buf2_len.value <= len(buf2) - return (buf1.raw[0:int(buf1_len.value)], buf2.raw[0:int(buf2_len.value)]) + return (buf1.raw[0 : int(buf1_len.value)], buf2.raw[0 : int(buf2_len.value)]) + def _call_fn_returning_str(guess, fn): # Assumes that anything called with this is returning plain ASCII strings # (base64 data, algorithm names, etc) v = _call_fn_returning_vec(guess, fn) - return v.decode('ascii')[:-1] + return v.decode("ascii")[:-1] + @VIEW_BIN_CALLBACK def _view_bin_fn(_ctx, buf_val, buf_len): _view_bin_fn.output = buf_val[0:buf_len] return 0 + def _call_fn_viewing_vec(fn) -> bytes: fn(None, _view_bin_fn) result = _view_bin_fn.output _view_bin_fn.output = None return result + @VIEW_STR_CALLBACK def _view_str_fn(_ctx, str_val, _str_len): _view_str_fn.output = str_val return 0 + def _call_fn_viewing_str(fn) -> str: fn(None, _view_str_fn) - result = _view_str_fn.output.decode('utf8') + result = _view_str_fn.output.decode("utf8") _view_str_fn.output = None return result + def _ctype_str(s: str | None) -> bytes | None: if s is None: return None assert isinstance(s, str) - return s.encode('utf-8') + return s.encode("utf-8") + def _ctype_to_str(s: bytes) -> str: - return s.decode('utf-8') + return s.decode("utf-8") + def _ctype_bits(s: str | bytes) -> bytes: if isinstance(s, bytes): return s elif isinstance(s, str): - return s.encode('utf-8') + return s.encode("utf-8") else: - raise Exception("Internal error - unexpected type %s provided to _ctype_bits" % (type(s).__name__)) + raise Exception( + "Internal error - unexpected type %s provided to _ctype_bits" + % (type(s).__name__) + ) + def _ctype_bufout(buf): return buf.raw + def _hex_encode(buf): - return hexlify(buf).decode('ascii') + return hexlify(buf).decode("ascii") # @@ -674,23 +1076,29 @@ def _hex_encode(buf): # _MPIArg = Union[str, "MPI", Any, None] + # # Versioning # def version_major() -> int: return int(_DLL.botan_version_major()) + def version_minor() -> int: return int(_DLL.botan_version_minor()) + def version_patch() -> int: return int(_DLL.botan_version_patch()) + def ffi_api_version() -> int: return int(_DLL.botan_ffi_api_version()) + def version_string() -> int: - return _DLL.botan_version_string().decode('ascii') + return _DLL.botan_version_string().decode("ascii") + # # Utilities @@ -705,35 +1113,46 @@ def const_time_compare(x: str | bytes, y: str | bytes) -> bool: rc = _DLL.botan_constant_time_compare(xbits, ybits, c_size_t(len_x)) return rc == 0 + # # TPM2 # + class TPM2Object: def __init__(self, obj: c_void_p, destroyer: Callable[[c_void_p], None]): self.__obj = obj self.__destroyer = destroyer def __del__(self): - if hasattr(self, '__obj') and hasattr(self, '__destroyer'): + if hasattr(self, "__obj") and hasattr(self, "__destroyer"): self.__destroyer(self.__obj) def handle_(self): return self.__obj + class TPM2Context(TPM2Object): """TPM 2.0 Context object""" - def __init__(self, tcti_name_maybe_with_conf: str | None = None, tcti_conf: str | None = None): + def __init__( + self, tcti_name_maybe_with_conf: str | None = None, tcti_conf: str | None = None + ): """Construct a TPM2Context object with optional TCTI name and configuration.""" obj = c_void_p(0) if tcti_conf is not None: - rc = _DLL.botan_tpm2_ctx_init_ex(byref(obj), _ctype_str(tcti_name_maybe_with_conf), _ctype_str(tcti_conf)) + rc = _DLL.botan_tpm2_ctx_init_ex( + byref(obj), _ctype_str(tcti_name_maybe_with_conf), _ctype_str(tcti_conf) + ) else: - rc = _DLL.botan_tpm2_ctx_init(byref(obj), _ctype_str(tcti_name_maybe_with_conf)) - if rc == -40: # 'Not Implemented' - raise BotanException("TPM2 is not implemented in this build configuration", rc) + rc = _DLL.botan_tpm2_ctx_init( + byref(obj), _ctype_str(tcti_name_maybe_with_conf) + ) + if rc == -40: # 'Not Implemented' + raise BotanException( + "TPM2 is not implemented in this build configuration", rc + ) self.rng_ = None super().__init__(obj, _DLL.botan_tpm2_ctx_destroy) @@ -751,6 +1170,7 @@ def enable_botan_crypto_backend(self, rng: RandomNumberGenerator): self.rng_ = rng _DLL.botan_tpm2_ctx_enable_crypto_backend(self.handle_(), self.rng_.handle_()) + class TPM2Session(TPM2Object): """Basic TPM 2.0 Session object, typically users will instantiate a derived class.""" @@ -760,7 +1180,8 @@ def __init__(self, obj: c_void_p): @staticmethod def session_bundle_(*args): """Transforms a session bundle passed by the downstream user into a 3-tuple of session handles. - Users might pass a bare TPM2Session object or an iterable list of such objects.""" + Users might pass a bare TPM2Session object or an iterable list of such objects. + """ if len(args) == 1: if isinstance(args[0], Iterable): args = list(args[0]) @@ -771,25 +1192,31 @@ def session_bundle_(*args): sessions = list(args) while len(sessions) < 3: sessions.append(None) - return (s.handle_() if isinstance(s, TPM2Session) else None for s in sessions) + return ( + s.handle_() if isinstance(s, TPM2Session) else None for s in sessions + ) else: - raise BotanException("session bundle arguments must be 0 to 3 TPM2Session objects") + raise BotanException( + "session bundle arguments must be 0 to 3 TPM2Session objects" + ) class TPM2UnauthenticatedSession(TPM2Session): """Session object that is not bound to any authentication credential. It provides basic parameter encryption between the application and the TPM.""" + def __init__(self, ctx: TPM2Context): obj = c_void_p(0) _DLL.botan_tpm2_unauthenticated_session_init(byref(obj), ctx.handle_()) super().__init__(obj) + # # RNG # class RandomNumberGenerator: # Can also use type "system" - def __init__(self, rng_type: str = 'system', **kwargs): + def __init__(self, rng_type: str = "system", **kwargs): """Constructs a RandomNumberGenerator of type rng_type Available RNG types are:: @@ -802,17 +1229,24 @@ def __init__(self, rng_type: str = 'system', **kwargs): (needs additional named arguments tpm2_context= and, optionally, tpm2_sessions=) """ self.__obj = c_void_p(0) - if rng_type == 'tpm2': + if rng_type == "tpm2": ctx = kwargs.pop("tpm2_context", None) if not ctx or not isinstance(ctx, TPM2Context): - raise BotanException("Cannot instantiate a TPM2-based RNG without a TPM2 context, pass tpm2_context= argument?") + raise BotanException( + "Cannot instantiate a TPM2-based RNG without a TPM2 context, pass tpm2_context= argument?" + ) sessions = TPM2Session.session_bundle_(kwargs.pop("tpm2_sessions", None)) if kwargs: - raise BotanException("Unexpected arguments for TPM2 RNG: %s" % (", ".join(kwargs.keys()))) + raise BotanException( + "Unexpected arguments for TPM2 RNG: %s" % (", ".join(kwargs.keys())) + ) _DLL.botan_tpm2_rng_init(byref(self.__obj), ctx.handle_(), *sessions) else: if kwargs: - raise BotanException("Unexpected arguments for RNG type %s: %s" % (rng_type, ", ".join(kwargs.keys()))) + raise BotanException( + "Unexpected arguments for RNG type %s: %s" + % (rng_type, ", ".join(kwargs.keys())) + ) _DLL.botan_rng_init(byref(self.__obj), _ctype_str(rng_type)) def __del__(self): @@ -836,23 +1270,25 @@ def get(self, length: int) -> bytes: _DLL.botan_rng_get(self.__obj, out, c_size_t(length)) return _ctype_bufout(out) + # # Block cipher # class BlockCipher: def __init__(self, algo: str | c_void_p): - if isinstance(algo, c_void_p): self.__obj = algo else: - flags = c_uint32(0) # always zero in this API version + flags = c_uint32(0) # always zero in this API version self.__obj = c_void_p(0) _DLL.botan_block_cipher_init(byref(self.__obj), _ctype_str(algo), flags) min_keylen = c_size_t(0) max_keylen = c_size_t(0) mod_keylen = c_size_t(0) - _DLL.botan_block_cipher_get_keyspec(self.__obj, byref(min_keylen), byref(max_keylen), byref(mod_keylen)) + _DLL.botan_block_cipher_get_keyspec( + self.__obj, byref(min_keylen), byref(max_keylen), byref(mod_keylen) + ) self.__min_keylen = min_keylen.value self.__max_keylen = max_keylen.value @@ -885,7 +1321,9 @@ def decrypt(self, ct: bytes) -> Array[c_char]: return output def algo_name(self) -> str: - return _call_fn_returning_str(32, lambda b, bl: _DLL.botan_block_cipher_name(self.__obj, b, bl)) + return _call_fn_returning_str( + 32, lambda b, bl: _DLL.botan_block_cipher_name(self.__obj, b, bl) + ) def clear(self): _DLL.botan_block_cipher_clear(self.__obj) @@ -908,16 +1346,19 @@ def keylength_modulo(self) -> int: # class HashFunction: def __init__(self, algo: str | c_void_p): - if isinstance(algo, c_void_p): self.__obj = algo else: - flags = c_uint32(0) # always zero in this API version + flags = c_uint32(0) # always zero in this API version self.__obj = c_void_p(0) _DLL.botan_hash_init(byref(self.__obj), _ctype_str(algo), flags) - self.__output_length = _call_fn_returning_sz(lambda length: _DLL.botan_hash_output_length(self.__obj, length)) - self.__block_size = _call_fn_returning_sz(lambda length: _DLL.botan_hash_block_size(self.__obj, length)) + self.__output_length = _call_fn_returning_sz( + lambda length: _DLL.botan_hash_output_length(self.__obj, length) + ) + self.__block_size = _call_fn_returning_sz( + lambda length: _DLL.botan_hash_block_size(self.__obj, length) + ) def __del__(self): _DLL.botan_hash_destroy(self.__obj) @@ -928,7 +1369,9 @@ def copy_state(self) -> HashFunction: return HashFunction(copy) def algo_name(self) -> str: - return _call_fn_returning_str(32, lambda b, bl: _DLL.botan_hash_name(self.__obj, b, bl)) + return _call_fn_returning_str( + 32, lambda b, bl: _DLL.botan_hash_name(self.__obj, b, bl) + ) def clear(self): _DLL.botan_hash_clear(self.__obj) @@ -948,19 +1391,22 @@ def final(self) -> bytes: _DLL.botan_hash_final(self.__obj, out) return _ctype_bufout(out) + # # Message authentication codes # class MsgAuthCode: def __init__(self, algo: str): - flags = c_uint32(0) # always zero in this API version + flags = c_uint32(0) # always zero in this API version self.__obj = c_void_p(0) _DLL.botan_mac_init(byref(self.__obj), _ctype_str(algo), flags) min_keylen = c_size_t(0) max_keylen = c_size_t(0) mod_keylen = c_size_t(0) - _DLL.botan_mac_get_keyspec(self.__obj, byref(min_keylen), byref(max_keylen), byref(mod_keylen)) + _DLL.botan_mac_get_keyspec( + self.__obj, byref(min_keylen), byref(max_keylen), byref(mod_keylen) + ) self.__min_keylen = min_keylen.value self.__max_keylen = max_keylen.value @@ -977,7 +1423,9 @@ def clear(self): _DLL.botan_mac_clear(self.__obj) def algo_name(self) -> str: - return _call_fn_returning_str(32, lambda b, bl: _DLL.botan_mac_name(self.__obj, b, bl)) + return _call_fn_returning_str( + 32, lambda b, bl: _DLL.botan_mac_name(self.__obj, b, bl) + ) def output_length(self) -> int: return self.__output_length @@ -1006,19 +1454,22 @@ def final(self) -> bytes: _DLL.botan_mac_final(self.__obj, out) return _ctype_bufout(out) + class SymmetricCipher: def __init__(self, algo: str, encrypt: bool = True): flags = 0 if encrypt else 1 self.__obj = c_void_p(0) _DLL.botan_cipher_init(byref(self.__obj), _ctype_str(algo), flags) - self._is_cbc = algo.find('/CBC') > 0 + self._is_cbc = algo.find("/CBC") > 0 self._is_encrypt = encrypt def __del__(self): _DLL.botan_cipher_destroy(self.__obj) def algo_name(self) -> str: - return _call_fn_returning_str(32, lambda b, bl: _DLL.botan_cipher_name(self.__obj, b, bl)) + return _call_fn_returning_str( + 32, lambda b, bl: _DLL.botan_cipher_name(self.__obj, b, bl) + ) def default_nonce_length(self) -> int: length = c_size_t(0) @@ -1080,8 +1531,7 @@ def start(self, nonce: bytes): _DLL.botan_cipher_start(self.__obj, nonce, len(nonce)) def _update(self, txt: str | bytes | None, final: bool): - - inp = txt if txt else '' + inp = txt if txt else "" bits = _ctype_bits(inp) inp_sz = c_size_t(len(bits)) inp_consumed = c_size_t(0) @@ -1099,13 +1549,20 @@ def _update(self, txt: str | bytes | None, final: bool): out_written = c_size_t(0) flags = c_uint32(1 if final else 0) - _DLL.botan_cipher_update(self.__obj, flags, - out, out_sz, byref(out_written), - bits, inp_sz, byref(inp_consumed)) + _DLL.botan_cipher_update( + self.__obj, + flags, + out, + out_sz, + byref(out_written), + bits, + inp_sz, + byref(inp_consumed), + ) # buffering not supported yet assert inp_consumed.value == inp_sz.value - return out.raw[0:int(out_written.value)] + return out.raw[0 : int(out_written.value)] def update(self, txt: str | bytes): return self._update(txt, False) @@ -1113,6 +1570,7 @@ def update(self, txt: str | bytes): def finish(self, txt: str | bytes | None = None): return self._update(txt, True) + def bcrypt(passwd: str, rng_obj: RandomNumberGenerator, work_factor=10): """ Bcrypt password hashing @@ -1120,61 +1578,115 @@ def bcrypt(passwd: str, rng_obj: RandomNumberGenerator, work_factor=10): out_len = c_size_t(64) out = create_string_buffer(out_len.value) flags = c_uint32(0) - _DLL.botan_bcrypt_generate(out, byref(out_len), _ctype_str(passwd), - rng_obj.handle_(), c_size_t(work_factor), flags) - b = out.raw[0:int(out_len.value)-1] - if b[-1] == '\x00': + _DLL.botan_bcrypt_generate( + out, + byref(out_len), + _ctype_str(passwd), + rng_obj.handle_(), + c_size_t(work_factor), + flags, + ) + b = out.raw[0 : int(out_len.value) - 1] + if b[-1] == "\x00": b = b[:-1] return _ctype_to_str(b) + def check_bcrypt(passwd: str, passwd_hash: str): rc = _DLL.botan_bcrypt_is_valid(_ctype_str(passwd), _ctype_str(passwd_hash)) return rc == 0 + # # PBKDF # -def pbkdf(algo: str, password: str, out_len: int, iterations: int = 100000, salt: bytes | None = None) -> tuple[bytes, int, bytes]: +def pbkdf( + algo: str, + password: str, + out_len: int, + iterations: int = 100000, + salt: bytes | None = None, +) -> tuple[bytes, int, bytes]: if salt is None: salt = RandomNumberGenerator().get(12) out_buf = create_string_buffer(out_len) - _DLL.botan_pwdhash(_ctype_str(algo), iterations, 0, 0, - out_buf, out_len, - _ctype_str(password), len(password), - salt, len(salt)) + _DLL.botan_pwdhash( + _ctype_str(algo), + iterations, + 0, + 0, + out_buf, + out_len, + _ctype_str(password), + len(password), + salt, + len(salt), + ) return (salt, iterations, out_buf.raw) -def pbkdf_timed(algo: str, password: str, out_len: int, ms_to_run: int = 300, salt: bytes | None = None) -> tuple[bytes, int, bytes]: + +def pbkdf_timed( + algo: str, + password: str, + out_len: int, + ms_to_run: int = 300, + salt: bytes | None = None, +) -> tuple[bytes, int, bytes]: if salt is None: salt = RandomNumberGenerator().get(12) out_buf = create_string_buffer(out_len) iterations = c_size_t(0) - _DLL.botan_pwdhash_timed(_ctype_str(algo), c_uint32(ms_to_run), - byref(iterations), None, None, - out_buf, out_len, - _ctype_str(password), len(password), - salt, len(salt)) + _DLL.botan_pwdhash_timed( + _ctype_str(algo), + c_uint32(ms_to_run), + byref(iterations), + None, + None, + out_buf, + out_len, + _ctype_str(password), + len(password), + salt, + len(salt), + ) return (salt, iterations.value, out_buf.raw) + # # Scrypt # -def scrypt(out_len: int, password: str, salt: str | bytes, n: int = 1024, r: int = 8, p: int = 8) -> bytes: +def scrypt( + out_len: int, + password: str, + salt: str | bytes, + n: int = 1024, + r: int = 8, + p: int = 8, +) -> bytes: out_buf = create_string_buffer(out_len) passbits = _ctype_bits(password) saltbits = _ctype_bits(salt) - _DLL.botan_pwdhash(_ctype_str("Scrypt"), n, r, p, - out_buf, out_len, - passbits, len(passbits), - saltbits, len(saltbits)) + _DLL.botan_pwdhash( + _ctype_str("Scrypt"), + n, + r, + p, + out_buf, + out_len, + passbits, + len(passbits), + saltbits, + len(saltbits), + ) return out_buf.raw + # Argon2 # # The variant param should be "Argon2i", "Argon2d", or "Argon2id" @@ -1184,34 +1696,59 @@ def scrypt(out_len: int, password: str, salt: str | bytes, n: int = 1024, r: int # p specifies the parallelism # # returns an output of out_len bytes -def argon2(variant: str, out_len: int, password: str, salt: str | bytes, m: int = 256, t: int = 1, p: int = 1) -> bytes: +def argon2( + variant: str, + out_len: int, + password: str, + salt: str | bytes, + m: int = 256, + t: int = 1, + p: int = 1, +) -> bytes: out_buf = create_string_buffer(out_len) passbits = _ctype_bits(password) saltbits = _ctype_bits(salt) - _DLL.botan_pwdhash(_ctype_str(variant), m, t, p, - out_buf, out_len, - passbits, len(passbits), - saltbits, len(saltbits)) + _DLL.botan_pwdhash( + _ctype_str(variant), + m, + t, + p, + out_buf, + out_len, + passbits, + len(passbits), + saltbits, + len(saltbits), + ) return out_buf.raw + # # KDF # def kdf(algo: str, secret: bytes, out_len: int, salt: bytes, label: bytes) -> bytes: out_buf = create_string_buffer(out_len) out_sz = c_size_t(out_len) - _DLL.botan_kdf(_ctype_str(algo), out_buf, out_sz, - secret, len(secret), - salt, len(salt), - label, len(label)) - return out_buf.raw[0:int(out_sz.value)] + _DLL.botan_kdf( + _ctype_str(algo), + out_buf, + out_sz, + secret, + len(secret), + salt, + len(salt), + label, + len(label), + ) + return out_buf.raw[0 : int(out_sz.value)] + # # Public key # -class PublicKey: # pylint: disable=invalid-name +class PublicKey: # pylint: disable=invalid-name def __init__(self, obj: c_void_p | None = None): if not obj: obj = c_void_p(0) @@ -1239,7 +1776,9 @@ def load_dsa(cls, p: _MPIArg, q: _MPIArg, g: _MPIArg, y: _MPIArg) -> PublicKey: q = MPI(q) g = MPI(g) y = MPI(y) - _DLL.botan_pubkey_load_dsa(byref(pub.handle_()), p.handle_(), q.handle_(), g.handle_(), y.handle_()) + _DLL.botan_pubkey_load_dsa( + byref(pub.handle_()), p.handle_(), q.handle_(), g.handle_(), y.handle_() + ) return pub @classmethod @@ -1248,7 +1787,9 @@ def load_dh(cls, p: _MPIArg, g: _MPIArg, y: _MPIArg) -> PublicKey: p = MPI(p) g = MPI(g) y = MPI(y) - _DLL.botan_pubkey_load_dh(byref(pub.handle_()), p.handle_(), g.handle_(), y.handle_()) + _DLL.botan_pubkey_load_dh( + byref(pub.handle_()), p.handle_(), g.handle_(), y.handle_() + ) return pub @classmethod @@ -1258,7 +1799,9 @@ def load_elgamal(cls, p: _MPIArg, q: _MPIArg, g: _MPIArg, y: _MPIArg) -> PublicK q = MPI(q) g = MPI(g) y = MPI(y) - _DLL.botan_pubkey_load_elgamal(byref(pub.handle_()), p.handle_(), q.handle_(), g.handle_(), y.handle_()) + _DLL.botan_pubkey_load_elgamal( + byref(pub.handle_()), p.handle_(), q.handle_(), g.handle_(), y.handle_() + ) return pub @classmethod @@ -1266,13 +1809,20 @@ def load_ecdsa(cls, curve: str, pub_x: _MPIArg, pub_y: _MPIArg) -> PublicKey: pub = PublicKey() pub_x = MPI(pub_x) pub_y = MPI(pub_y) - _DLL.botan_pubkey_load_ecdsa(byref(pub.handle_()), pub_x.handle_(), pub_y.handle_(), _ctype_str(curve)) + _DLL.botan_pubkey_load_ecdsa( + byref(pub.handle_()), pub_x.handle_(), pub_y.handle_(), _ctype_str(curve) + ) return pub @classmethod def load_ecdsa_sec1(cls, curve: str, sec1_encoding: str | bytes) -> PublicKey: pub = PublicKey() - _DLL.botan_pubkey_load_ecdsa_sec1(byref(pub.handle_()), _ctype_bits(sec1_encoding), len(sec1_encoding), _ctype_str(curve)) + _DLL.botan_pubkey_load_ecdsa_sec1( + byref(pub.handle_()), + _ctype_bits(sec1_encoding), + len(sec1_encoding), + _ctype_str(curve), + ) return pub @classmethod @@ -1280,13 +1830,20 @@ def load_ecdh(cls, curve: str, pub_x: _MPIArg, pub_y: _MPIArg) -> PublicKey: pub = PublicKey() pub_x = MPI(pub_x) pub_y = MPI(pub_y) - _DLL.botan_pubkey_load_ecdh(byref(pub.handle_()), pub_x.handle_(), pub_y.handle_(), _ctype_str(curve)) + _DLL.botan_pubkey_load_ecdh( + byref(pub.handle_()), pub_x.handle_(), pub_y.handle_(), _ctype_str(curve) + ) return pub @classmethod def load_ecdh_sec1(cls, curve: str, sec1_encoding: str | bytes) -> PublicKey: pub = PublicKey() - _DLL.botan_pubkey_load_ecdh_sec1(byref(pub.handle_()), _ctype_bits(sec1_encoding), len(sec1_encoding), _ctype_str(curve)) + _DLL.botan_pubkey_load_ecdh_sec1( + byref(pub.handle_()), + _ctype_bits(sec1_encoding), + len(sec1_encoding), + _ctype_str(curve), + ) return pub @classmethod @@ -1294,13 +1851,20 @@ def load_sm2(cls, curve: str, pub_x: _MPIArg, pub_y: _MPIArg) -> PublicKey: pub = PublicKey() pub_x = MPI(pub_x) pub_y = MPI(pub_y) - _DLL.botan_pubkey_load_sm2(byref(pub.handle_()), pub_x.handle_(), pub_y.handle_(), _ctype_str(curve)) + _DLL.botan_pubkey_load_sm2( + byref(pub.handle_()), pub_x.handle_(), pub_y.handle_(), _ctype_str(curve) + ) return pub @classmethod def load_sm2_sec1(cls, curve: str, sec1_encoding: str | bytes) -> PublicKey: pub = PublicKey() - _DLL.botan_pubkey_load_sm2_sec1(byref(pub.handle_()), _ctype_bits(sec1_encoding), len(sec1_encoding), _ctype_str(curve)) + _DLL.botan_pubkey_load_sm2_sec1( + byref(pub.handle_()), + _ctype_bits(sec1_encoding), + len(sec1_encoding), + _ctype_str(curve), + ) return pub @classmethod @@ -1312,31 +1876,41 @@ def load_kyber(cls, key: bytes) -> PublicKey: @classmethod def load_ml_kem(cls, mlkem_mode: str, key: bytes) -> PublicKey: pub = PublicKey() - _DLL.botan_pubkey_load_ml_kem(byref(pub.handle_()), key, len(key), _ctype_str(mlkem_mode)) + _DLL.botan_pubkey_load_ml_kem( + byref(pub.handle_()), key, len(key), _ctype_str(mlkem_mode) + ) return pub @classmethod def load_ml_dsa(cls, mldsa_mode: str, key: bytes) -> PublicKey: pub = PublicKey() - _DLL.botan_pubkey_load_ml_dsa(byref(pub.handle_()), key, len(key), _ctype_str(mldsa_mode)) + _DLL.botan_pubkey_load_ml_dsa( + byref(pub.handle_()), key, len(key), _ctype_str(mldsa_mode) + ) return pub @classmethod def load_slh_dsa(cls, slhdsa_mode: str, key: bytes) -> PublicKey: pub = PublicKey() - _DLL.botan_pubkey_load_slh_dsa(byref(pub.handle_()), key, len(key), _ctype_str(slhdsa_mode)) + _DLL.botan_pubkey_load_slh_dsa( + byref(pub.handle_()), key, len(key), _ctype_str(slhdsa_mode) + ) return pub @classmethod def load_frodokem(cls, frodo_mode: str, key: bytes) -> PublicKey: pub = PublicKey() - _DLL.botan_pubkey_load_frodokem(byref(pub.handle_()), key, len(key), _ctype_str(frodo_mode)) + _DLL.botan_pubkey_load_frodokem( + byref(pub.handle_()), key, len(key), _ctype_str(frodo_mode) + ) return pub @classmethod def load_classic_mceliece(cls, cmce_mode: str, key: bytes) -> PublicKey: pub = PublicKey() - _DLL.botan_pubkey_load_classic_mceliece(byref(pub.handle_()), key, len(key), _ctype_str(cmce_mode)) + _DLL.botan_pubkey_load_classic_mceliece( + byref(pub.handle_()), key, len(key), _ctype_str(cmce_mode) + ) return pub def __del__(self): @@ -1356,7 +1930,9 @@ def estimated_strength(self) -> int: return r.value def algo_name(self) -> str: - return _call_fn_returning_str(32, lambda b, bl: _DLL.botan_pubkey_algo_name(self.__obj, b, bl)) + return _call_fn_returning_str( + 32, lambda b, bl: _DLL.botan_pubkey_algo_name(self.__obj, b, bl) + ) def export(self, pem: bool = False) -> str | bytes: if pem: @@ -1365,25 +1941,35 @@ def export(self, pem: bool = False) -> str | bytes: return self.to_der() def to_der(self) -> bytes: - return _call_fn_viewing_vec(lambda vc, vfn: _DLL.botan_pubkey_view_der(self.__obj, vc, vfn)) + return _call_fn_viewing_vec( + lambda vc, vfn: _DLL.botan_pubkey_view_der(self.__obj, vc, vfn) + ) def to_pem(self) -> str: - return _call_fn_viewing_str(lambda vc, vfn: _DLL.botan_pubkey_view_pem(self.__obj, vc, vfn)) + return _call_fn_viewing_str( + lambda vc, vfn: _DLL.botan_pubkey_view_pem(self.__obj, vc, vfn) + ) def to_raw(self) -> bytes: - return _call_fn_viewing_vec(lambda vc, vfn: _DLL.botan_pubkey_view_raw(self.__obj, vc, vfn)) + return _call_fn_viewing_vec( + lambda vc, vfn: _DLL.botan_pubkey_view_raw(self.__obj, vc, vfn) + ) def view_kyber_raw_key(self) -> bytes: """Deprecated: use to_raw() instead""" - return _call_fn_viewing_vec(lambda vc, vfn: _DLL.botan_pubkey_view_kyber_raw_key(self.__obj, vc, vfn)) + return _call_fn_viewing_vec( + lambda vc, vfn: _DLL.botan_pubkey_view_kyber_raw_key(self.__obj, vc, vfn) + ) - def fingerprint(self, hash_algorithm: str = 'SHA-256') -> str: + def fingerprint(self, hash_algorithm: str = "SHA-256") -> str: n = HashFunction(hash_algorithm).output_length() buf = create_string_buffer(n) buf_len = c_size_t(n) - _DLL.botan_pubkey_fingerprint(self.__obj, _ctype_str(hash_algorithm), buf, byref(buf_len)) - return _hex_encode(buf[0:int(buf_len.value)]) + _DLL.botan_pubkey_fingerprint( + self.__obj, _ctype_str(hash_algorithm), buf, byref(buf_len) + ) + return _hex_encode(buf[0 : int(buf_len.value)]) def get_field(self, field_name: str) -> int: v = MPI() @@ -1396,7 +1982,10 @@ def object_identifier(self) -> OID: return oid def get_public_point(self) -> bytes: - return _call_fn_viewing_vec(lambda vc, vfn: _DLL.botan_pubkey_view_ec_public_point(self.__obj, vc, vfn)) + return _call_fn_viewing_vec( + lambda vc, vfn: _DLL.botan_pubkey_view_ec_public_point(self.__obj, vc, vfn) + ) + # # Private Key @@ -1410,40 +1999,56 @@ def __init__(self, obj: c_void_p | None = None): @classmethod def load(cls, val: str | bytes, passphrase: str = "") -> PrivateKey: priv = PrivateKey() - rng_obj = c_void_p(0) # unused in recent versions + rng_obj = c_void_p(0) # unused in recent versions bits = _ctype_bits(val) - _DLL.botan_privkey_load(byref(priv.handle_()), rng_obj, bits, len(bits), _ctype_str(passphrase)) + _DLL.botan_privkey_load( + byref(priv.handle_()), rng_obj, bits, len(bits), _ctype_str(passphrase) + ) return priv @classmethod - def create(cls, algo: str, params: str | int | tuple[int, int], rng_obj: RandomNumberGenerator) -> PrivateKey: - if algo == 'rsa': - algo = 'RSA' + def create( + cls, + algo: str, + params: str | int | tuple[int, int], + rng_obj: RandomNumberGenerator, + ) -> PrivateKey: + if algo == "rsa": + algo = "RSA" params = "%d" % (params) - elif algo == 'ecdsa': - algo = 'ECDSA' - elif algo in ['ecdh', 'ECDH']: - if params in ['x25519', 'curve25519']: - algo = 'X25519' - params = '' - elif params == 'x448': - algo = 'X448' - params = '' + elif algo == "ecdsa": + algo = "ECDSA" + elif algo in ["ecdh", "ECDH"]: + if params in ["x25519", "curve25519"]: + algo = "X25519" + params = "" + elif params == "x448": + algo = "X448" + params = "" else: - algo = 'ECDH' - elif algo in ['mce', 'mceliece']: + algo = "ECDH" + elif algo in ["mce", "mceliece"]: # TODO(Botan4) remove this case - algo = 'McEliece' + algo = "McEliece" params = "%d,%d" % (params[0], params[1]) priv = PrivateKey() - _DLL.botan_privkey_create(byref(priv.handle_()), _ctype_str(algo), _ctype_str(params), rng_obj.handle_()) + _DLL.botan_privkey_create( + byref(priv.handle_()), + _ctype_str(algo), + _ctype_str(params), + rng_obj.handle_(), + ) return priv @classmethod - def create_ec(cls, algo: str, ec_group: ECGroup, rng_obj: RandomNumberGenerator) -> PrivateKey: + def create_ec( + cls, algo: str, ec_group: ECGroup, rng_obj: RandomNumberGenerator + ) -> PrivateKey: obj = c_void_p(0) - _DLL.botan_ec_privkey_create(byref(obj), _ctype_str(algo), ec_group.handle_(), rng_obj.handle_()) + _DLL.botan_ec_privkey_create( + byref(obj), _ctype_str(algo), ec_group.handle_(), rng_obj.handle_() + ) return PrivateKey(obj) @classmethod @@ -1452,7 +2057,9 @@ def load_rsa(cls, p: _MPIArg, q: _MPIArg, e: _MPIArg) -> PrivateKey: p = MPI(p) q = MPI(q) e = MPI(e) - _DLL.botan_privkey_load_rsa(byref(priv.handle_()), p.handle_(), q.handle_(), e.handle_()) + _DLL.botan_privkey_load_rsa( + byref(priv.handle_()), p.handle_(), q.handle_(), e.handle_() + ) return priv @classmethod @@ -1462,7 +2069,9 @@ def load_dsa(cls, p: _MPIArg, q: _MPIArg, g: _MPIArg, x: _MPIArg) -> PrivateKey: q = MPI(q) g = MPI(g) x = MPI(x) - _DLL.botan_privkey_load_dsa(byref(priv.handle_()), p.handle_(), q.handle_(), g.handle_(), x.handle_()) + _DLL.botan_privkey_load_dsa( + byref(priv.handle_()), p.handle_(), q.handle_(), g.handle_(), x.handle_() + ) return priv @classmethod @@ -1471,7 +2080,9 @@ def load_dh(cls, p: _MPIArg, g: _MPIArg, x: _MPIArg) -> PrivateKey: p = MPI(p) g = MPI(g) x = MPI(x) - _DLL.botan_privkey_load_dh(byref(priv.handle_()), p.handle_(), g.handle_(), x.handle_()) + _DLL.botan_privkey_load_dh( + byref(priv.handle_()), p.handle_(), g.handle_(), x.handle_() + ) return priv @classmethod @@ -1481,28 +2092,36 @@ def load_elgamal(cls, p: _MPIArg, q: _MPIArg, g: _MPIArg, x: _MPIArg) -> Private q = MPI(q) g = MPI(g) x = MPI(x) - _DLL.botan_privkey_load_elgamal(byref(priv.handle_()), p.handle_(), q.handle_(), g.handle_(), x.handle_()) + _DLL.botan_privkey_load_elgamal( + byref(priv.handle_()), p.handle_(), q.handle_(), g.handle_(), x.handle_() + ) return priv @classmethod def load_ecdsa(cls, curve: str, x: _MPIArg) -> PrivateKey: priv = PrivateKey() x = MPI(x) - _DLL.botan_privkey_load_ecdsa(byref(priv.handle_()), x.handle_(), _ctype_str(curve)) + _DLL.botan_privkey_load_ecdsa( + byref(priv.handle_()), x.handle_(), _ctype_str(curve) + ) return priv @classmethod def load_ecdh(cls, curve: str, x: _MPIArg) -> PrivateKey: priv = PrivateKey() x = MPI(x) - _DLL.botan_privkey_load_ecdh(byref(priv.handle_()), x.handle_(), _ctype_str(curve)) + _DLL.botan_privkey_load_ecdh( + byref(priv.handle_()), x.handle_(), _ctype_str(curve) + ) return priv @classmethod def load_sm2(cls, curve: str, x: _MPIArg) -> PrivateKey: priv = PrivateKey() x = MPI(x) - _DLL.botan_privkey_load_sm2(byref(priv.handle_()), x.handle_(), _ctype_str(curve)) + _DLL.botan_privkey_load_sm2( + byref(priv.handle_()), x.handle_(), _ctype_str(curve) + ) return priv @classmethod @@ -1514,31 +2133,41 @@ def load_kyber(cls, key: bytes) -> PrivateKey: @classmethod def load_ml_kem(cls, mlkem_mode: str, key: bytes) -> PrivateKey: priv = PrivateKey() - _DLL.botan_privkey_load_ml_kem(byref(priv.handle_()), key, len(key), _ctype_str(mlkem_mode)) + _DLL.botan_privkey_load_ml_kem( + byref(priv.handle_()), key, len(key), _ctype_str(mlkem_mode) + ) return priv @classmethod def load_ml_dsa(cls, mldsa_mode, key): priv = PrivateKey() - _DLL.botan_privkey_load_ml_dsa(byref(priv.handle_()), key, len(key), _ctype_str(mldsa_mode)) + _DLL.botan_privkey_load_ml_dsa( + byref(priv.handle_()), key, len(key), _ctype_str(mldsa_mode) + ) return priv @classmethod def load_slh_dsa(cls, slh_dsa: str, key: bytes) -> PrivateKey: priv = PrivateKey() - _DLL.botan_privkey_load_slh_dsa(byref(priv.handle_()), key, len(key), _ctype_str(slh_dsa)) + _DLL.botan_privkey_load_slh_dsa( + byref(priv.handle_()), key, len(key), _ctype_str(slh_dsa) + ) return priv @classmethod def load_frodokem(cls, frodo_mode: str, key: bytes) -> PrivateKey: priv = PrivateKey() - _DLL.botan_privkey_load_frodokem(byref(priv.handle_()), key, len(key), _ctype_str(frodo_mode)) + _DLL.botan_privkey_load_frodokem( + byref(priv.handle_()), key, len(key), _ctype_str(frodo_mode) + ) return priv @classmethod def load_classic_mceliece(cls, cmce_mode: str, key: bytes) -> PrivateKey: priv = PrivateKey() - _DLL.botan_privkey_load_classic_mceliece(byref(priv.handle_()), key, len(key), _ctype_str(cmce_mode)) + _DLL.botan_privkey_load_classic_mceliece( + byref(priv.handle_()), key, len(key), _ctype_str(cmce_mode) + ) return priv def __del__(self): @@ -1553,7 +2182,9 @@ def check_key(self, rng_obj: RandomNumberGenerator, strong: bool = True) -> bool return rc == 0 def algo_name(self) -> str: - return _call_fn_returning_str(32, lambda b, bl: _DLL.botan_privkey_algo_name(self.__obj, b, bl)) + return _call_fn_returning_str( + 32, lambda b, bl: _DLL.botan_privkey_algo_name(self.__obj, b, bl) + ) def get_public_key(self) -> PublicKey: pub = PublicKey() @@ -1561,17 +2192,25 @@ def get_public_key(self) -> PublicKey: return pub def to_der(self) -> bytes: - return _call_fn_viewing_vec(lambda vc, vfn: _DLL.botan_privkey_view_der(self.__obj, vc, vfn)) + return _call_fn_viewing_vec( + lambda vc, vfn: _DLL.botan_privkey_view_der(self.__obj, vc, vfn) + ) def to_pem(self) -> str: - return _call_fn_viewing_str(lambda vc, vfn: _DLL.botan_privkey_view_pem(self.__obj, vc, vfn)) + return _call_fn_viewing_str( + lambda vc, vfn: _DLL.botan_privkey_view_pem(self.__obj, vc, vfn) + ) def to_raw(self) -> bytes: - return _call_fn_viewing_vec(lambda vc, vfn: _DLL.botan_privkey_view_raw(self.__obj, vc, vfn)) + return _call_fn_viewing_vec( + lambda vc, vfn: _DLL.botan_privkey_view_raw(self.__obj, vc, vfn) + ) def view_kyber_raw_key(self) -> bytes: """Deprecated: use to_raw() instead""" - return _call_fn_viewing_vec(lambda vc, vfn: _DLL.botan_privkey_view_kyber_raw_key(self.__obj, vc, vfn)) + return _call_fn_viewing_vec( + lambda vc, vfn: _DLL.botan_privkey_view_kyber_raw_key(self.__obj, vc, vfn) + ) def export(self, pem: bool = False) -> str | bytes: if pem: @@ -1579,17 +2218,41 @@ def export(self, pem: bool = False) -> str | bytes: else: return self.to_der() - def export_encrypted(self, passphrase: str, rng: RandomNumberGenerator, pem: bool = False, msec: int = 300, cipher: str | None = None, pbkdf: str | None = None): # pylint: disable=redefined-outer-name + def export_encrypted( + self, + passphrase: str, + rng: RandomNumberGenerator, + pem: bool = False, + msec: int = 300, + cipher: str | None = None, + pbkdf: str | None = None, + ): # pylint: disable=redefined-outer-name if pem: return _call_fn_viewing_str( lambda vc, vfn: _DLL.botan_privkey_view_encrypted_pem_timed( - self.__obj, rng.handle_(), _ctype_str(passphrase), - _ctype_str(cipher), _ctype_str(pbkdf), c_size_t(msec), vc, vfn)) + self.__obj, + rng.handle_(), + _ctype_str(passphrase), + _ctype_str(cipher), + _ctype_str(pbkdf), + c_size_t(msec), + vc, + vfn, + ) + ) else: return _call_fn_viewing_vec( lambda vc, vfn: _DLL.botan_privkey_view_encrypted_der_timed( - self.__obj, rng.handle_(), _ctype_str(passphrase), - _ctype_str(cipher), _ctype_str(pbkdf), c_size_t(msec), vc, vfn)) + self.__obj, + rng.handle_(), + _ctype_str(passphrase), + _ctype_str(cipher), + _ctype_str(pbkdf), + c_size_t(msec), + vc, + vfn, + ) + ) def get_field(self, field_name: str) -> int: v = MPI() @@ -1613,11 +2276,14 @@ def remaining_operations(self) -> int: _DLL.botan_privkey_remaining_operations(self.__obj, byref(r)) return r.value + class PKEncrypt: def __init__(self, key: PublicKey, padding: str): self.__obj = c_void_p(0) - flags = c_uint32(0) # always zero in this ABI - _DLL.botan_pk_op_encrypt_create(byref(self.__obj), key.handle_(), _ctype_str(padding), flags) + flags = c_uint32(0) # always zero in this ABI + _DLL.botan_pk_op_encrypt_create( + byref(self.__obj), key.handle_(), _ctype_str(padding), flags + ) def __del__(self): _DLL.botan_pk_op_encrypt_destroy(self.__obj) @@ -1626,15 +2292,19 @@ def encrypt(self, msg: bytes, rng_obj: RandomNumberGenerator) -> bytes: outbuf_sz = c_size_t(0) _DLL.botan_pk_op_encrypt_output_length(self.__obj, len(msg), byref(outbuf_sz)) outbuf = create_string_buffer(outbuf_sz.value) - _DLL.botan_pk_op_encrypt(self.__obj, rng_obj.handle_(), outbuf, byref(outbuf_sz), msg, len(msg)) - return outbuf.raw[0:int(outbuf_sz.value)] + _DLL.botan_pk_op_encrypt( + self.__obj, rng_obj.handle_(), outbuf, byref(outbuf_sz), msg, len(msg) + ) + return outbuf.raw[0 : int(outbuf_sz.value)] class PKDecrypt: def __init__(self, key: PrivateKey, padding: str): self.__obj = c_void_p(0) - flags = c_uint32(0) # always zero in this ABI - _DLL.botan_pk_op_decrypt_create(byref(self.__obj), key.handle_(), _ctype_str(padding), flags) + flags = c_uint32(0) # always zero in this ABI + _DLL.botan_pk_op_decrypt_create( + byref(self.__obj), key.handle_(), _ctype_str(padding), flags + ) def __del__(self): _DLL.botan_pk_op_decrypt_destroy(self.__obj) @@ -1645,13 +2315,16 @@ def decrypt(self, msg: bytes) -> bytes: outbuf = create_string_buffer(outbuf_sz.value) bits = _ctype_bits(msg) _DLL.botan_pk_op_decrypt(self.__obj, outbuf, byref(outbuf_sz), bits, len(bits)) - return outbuf.raw[0:int(outbuf_sz.value)] + return outbuf.raw[0 : int(outbuf_sz.value)] + -class PKSign: # pylint: disable=invalid-name +class PKSign: # pylint: disable=invalid-name def __init__(self, key: PrivateKey, padding: str, der: bool = False): self.__obj = c_void_p(0) flags = c_uint32(1) if der else c_uint32(0) - _DLL.botan_pk_op_sign_create(byref(self.__obj), key.handle_(), _ctype_str(padding), flags) + _DLL.botan_pk_op_sign_create( + byref(self.__obj), key.handle_(), _ctype_str(padding), flags + ) def __del__(self): _DLL.botan_pk_op_sign_destroy(self.__obj) @@ -1663,14 +2336,19 @@ def finish(self, rng_obj: RandomNumberGenerator) -> bytes: outbuf_sz = c_size_t(0) _DLL.botan_pk_op_sign_output_length(self.__obj, byref(outbuf_sz)) outbuf = create_string_buffer(outbuf_sz.value) - _DLL.botan_pk_op_sign_finish(self.__obj, rng_obj.handle_(), outbuf, byref(outbuf_sz)) - return outbuf.raw[0:int(outbuf_sz.value)] + _DLL.botan_pk_op_sign_finish( + self.__obj, rng_obj.handle_(), outbuf, byref(outbuf_sz) + ) + return outbuf.raw[0 : int(outbuf_sz.value)] + class PKVerify: def __init__(self, key: PublicKey, padding: str, der: bool = False): self.__obj = c_void_p(0) flags = c_uint32(1) if der else c_uint32(0) - _DLL.botan_pk_op_verify_create(byref(self.__obj), key.handle_(), _ctype_str(padding), flags) + _DLL.botan_pk_op_verify_create( + byref(self.__obj), key.handle_(), _ctype_str(padding), flags + ) def __del__(self): _DLL.botan_pk_op_verify_destroy(self.__obj) @@ -1686,14 +2364,20 @@ def check_signature(self, signature: str | bytes) -> bool: return True return False + class PKKeyAgreement: def __init__(self, key: PrivateKey, kdf_name: str): self.__obj = c_void_p(0) - flags = c_uint32(0) # always zero in this ABI - _DLL.botan_pk_op_key_agreement_create(byref(self.__obj), key.handle_(), _ctype_str(kdf_name), flags) + flags = c_uint32(0) # always zero in this ABI + _DLL.botan_pk_op_key_agreement_create( + byref(self.__obj), key.handle_(), _ctype_str(kdf_name), flags + ) self.m_public_value = _call_fn_viewing_vec( - lambda vc, vfn: _DLL.botan_pk_op_key_agreement_view_public(key.handle_(), vc, vfn)) + lambda vc, vfn: _DLL.botan_pk_op_key_agreement_view_public( + key.handle_(), vc, vfn + ) + ) def __del__(self): _DLL.botan_pk_op_key_agreement_destroy(self.__obj) @@ -1709,28 +2393,41 @@ def underlying_output_length(self) -> int: def agree(self, other: bytes, key_len: int, salt: bytes) -> bytes: if key_len == 0: key_len = self.underlying_output_length() - return _call_fn_returning_vec(key_len, lambda b, bl: - _DLL.botan_pk_op_key_agreement(self.__obj, b, bl, - other, len(other), - salt, len(salt))) + return _call_fn_returning_vec( + key_len, + lambda b, bl: _DLL.botan_pk_op_key_agreement( + self.__obj, b, bl, other, len(other), salt, len(salt) + ), + ) + class KemEncrypt: def __init__(self, key: PublicKey, params: str): self.__obj = c_void_p(0) - _DLL.botan_pk_op_kem_encrypt_create(byref(self.__obj), key.handle_(), _ctype_str(params)) + _DLL.botan_pk_op_kem_encrypt_create( + byref(self.__obj), key.handle_(), _ctype_str(params) + ) def __del__(self): _DLL.botan_pk_op_kem_encrypt_destroy(self.__obj) def shared_key_length(self, desired_key_len: int) -> int: return _call_fn_returning_sz( - lambda len: _DLL.botan_pk_op_kem_encrypt_shared_key_length(self.__obj, desired_key_len, len)) + lambda len: _DLL.botan_pk_op_kem_encrypt_shared_key_length( + self.__obj, desired_key_len, len + ) + ) def encapsulated_key_length(self) -> int: return _call_fn_returning_sz( - lambda len: _DLL.botan_pk_op_kem_encrypt_encapsulated_key_length(self.__obj, len)) + lambda len: _DLL.botan_pk_op_kem_encrypt_encapsulated_key_length( + self.__obj, len + ) + ) - def create_shared_key(self, rng: RandomNumberGenerator, salt: bytes, desired_key_len: int) -> tuple[bytes, bytes]: + def create_shared_key( + self, rng: RandomNumberGenerator, salt: bytes, desired_key_len: int + ) -> tuple[bytes, bytes]: shared_key_len = self.shared_key_length(desired_key_len) shared_key_buf = create_string_buffer(shared_key_len) @@ -1746,7 +2443,7 @@ def create_shared_key(self, rng: RandomNumberGenerator, salt: bytes, desired_key shared_key_buf, c_size_t(shared_key_len), encapsulated_key_buf, - c_size_t(encapsulated_key_len) + c_size_t(encapsulated_key_len), ) shared_key = shared_key_buf.raw[:] @@ -1754,19 +2451,27 @@ def create_shared_key(self, rng: RandomNumberGenerator, salt: bytes, desired_key return (shared_key, encapsulated_key) + class KemDecrypt: def __init__(self, key: PrivateKey, params: str): self.__obj = c_void_p(0) - _DLL.botan_pk_op_kem_decrypt_create(byref(self.__obj), key.handle_(), _ctype_str(params)) + _DLL.botan_pk_op_kem_decrypt_create( + byref(self.__obj), key.handle_(), _ctype_str(params) + ) def __del__(self): _DLL.botan_pk_op_kem_decrypt_destroy(self.__obj) def shared_key_length(self, desired_key_len: int) -> int: return _call_fn_returning_sz( - lambda len: _DLL.botan_pk_op_kem_decrypt_shared_key_length(self.__obj, desired_key_len, len)) + lambda len: _DLL.botan_pk_op_kem_decrypt_shared_key_length( + self.__obj, desired_key_len, len + ) + ) - def decrypt_shared_key(self, salt: bytes, desired_key_len: int, encapsulated_key: bytes) -> bytes: + def decrypt_shared_key( + self, salt: bytes, desired_key_len: int, encapsulated_key: bytes + ) -> bytes: shared_key_len = self.shared_key_length(desired_key_len) return _call_fn_returning_vec( @@ -1779,9 +2484,11 @@ def decrypt_shared_key(self, salt: bytes, desired_key_len: int, encapsulated_key c_size_t(len(encapsulated_key)), c_size_t(desired_key_len), b, - bl) + bl, + ), ) + def _load_buf_or_file(filename, buf, file_fn, buf_fn): if filename is None and buf is None: raise BotanException("No filename or buf given") @@ -1802,17 +2509,20 @@ def _load_buf_or_file(filename, buf, file_fn, buf_fn): # # X.509 certificates # -class X509Cert: # pylint: disable=invalid-name +class X509Cert: # pylint: disable=invalid-name def __init__(self, filename: str | None = None, buf: bytes | None = None): self.__obj = c_void_p(0) - self.__obj = _load_buf_or_file(filename, buf, _DLL.botan_x509_cert_load_file, _DLL.botan_x509_cert_load) + self.__obj = _load_buf_or_file( + filename, buf, _DLL.botan_x509_cert_load_file, _DLL.botan_x509_cert_load + ) def __del__(self): _DLL.botan_x509_cert_destroy(self.__obj) def time_starts(self) -> datetime: starts = _call_fn_returning_str( - 16, lambda b, bl: _DLL.botan_x509_cert_get_time_starts(self.__obj, b, bl)) + 16, lambda b, bl: _DLL.botan_x509_cert_get_time_starts(self.__obj, b, bl) + ) if len(starts) == 13: # UTC time struct_time = strptime(starts, "%y%m%d%H%M%SZ") @@ -1826,7 +2536,8 @@ def time_starts(self) -> datetime: def time_expires(self) -> datetime: expires = _call_fn_returning_str( - 16, lambda b, bl: _DLL.botan_x509_cert_get_time_expires(self.__obj, b, bl)) + 16, lambda b, bl: _DLL.botan_x509_cert_get_time_expires(self.__obj, b, bl) + ) if len(expires) == 13: # UTC time struct_time = strptime(expires, "%y%m%d%H%M%SZ") @@ -1840,28 +2551,40 @@ def time_expires(self) -> datetime: def to_string(self) -> str: return _call_fn_viewing_str( - lambda vc, vfn: _DLL.botan_x509_cert_view_as_string(self.__obj, vc, vfn)) + lambda vc, vfn: _DLL.botan_x509_cert_view_as_string(self.__obj, vc, vfn) + ) - def fingerprint(self, hash_algo: str = 'SHA-256') -> str: + def fingerprint(self, hash_algo: str = "SHA-256") -> str: n = HashFunction(hash_algo).output_length() * 3 return _call_fn_returning_str( - n, lambda b, bl: _DLL.botan_x509_cert_get_fingerprint(self.__obj, _ctype_str(hash_algo), b, bl)) + n, + lambda b, bl: _DLL.botan_x509_cert_get_fingerprint( + self.__obj, _ctype_str(hash_algo), b, bl + ), + ) def serial_number(self) -> bytes: return _call_fn_returning_vec( - 32, lambda b, bl: _DLL.botan_x509_cert_get_serial_number(self.__obj, b, bl)) + 32, lambda b, bl: _DLL.botan_x509_cert_get_serial_number(self.__obj, b, bl) + ) def authority_key_id(self) -> bytes: return _call_fn_returning_vec( - 32, lambda b, bl: _DLL.botan_x509_cert_get_authority_key_id(self.__obj, b, bl)) + 32, + lambda b, bl: _DLL.botan_x509_cert_get_authority_key_id(self.__obj, b, bl), + ) def subject_key_id(self) -> bytes: return _call_fn_returning_vec( - 32, lambda b, bl: _DLL.botan_x509_cert_get_subject_key_id(self.__obj, b, bl)) + 32, lambda b, bl: _DLL.botan_x509_cert_get_subject_key_id(self.__obj, b, bl) + ) def subject_public_key_bits(self) -> bytes: return _call_fn_viewing_vec( - lambda vc, vfn: _DLL.botan_x509_cert_view_public_key_bits(self.__obj, vc, vfn)) + lambda vc, vfn: _DLL.botan_x509_cert_view_public_key_bits( + self.__obj, vc, vfn + ) + ) def subject_public_key(self) -> PublicKey: pub = c_void_p(0) @@ -1870,11 +2593,19 @@ def subject_public_key(self) -> PublicKey: def subject_dn(self, key: str, index: int) -> str: return _call_fn_returning_str( - 0, lambda b, bl: _DLL.botan_x509_cert_get_subject_dn(self.__obj, _ctype_str(key), index, b, bl)) + 0, + lambda b, bl: _DLL.botan_x509_cert_get_subject_dn( + self.__obj, _ctype_str(key), index, b, bl + ), + ) def issuer_dn(self, key: str, index: int) -> str: return _call_fn_returning_str( - 0, lambda b, bl: _DLL.botan_x509_cert_get_issuer_dn(self.__obj, _ctype_str(key), index, b, bl)) + 0, + lambda b, bl: _DLL.botan_x509_cert_get_issuer_dn( + self.__obj, _ctype_str(key), index, b, bl + ), + ) def hostname_match(self, hostname: str) -> bool: rc = _DLL.botan_x509_cert_hostname_match(self.__obj, _ctype_str(hostname)) @@ -1891,16 +2622,18 @@ def not_after(self) -> int: return time.value def allowed_usage(self, usage_list: List[str]) -> bool: - usage_values = {"NO_CONSTRAINTS": 0, - "DIGITAL_SIGNATURE": 32768, - "NON_REPUDIATION": 16384, - "KEY_ENCIPHERMENT": 8192, - "DATA_ENCIPHERMENT": 4096, - "KEY_AGREEMENT": 2048, - "KEY_CERT_SIGN": 1024, - "CRL_SIGN": 512, - "ENCIPHER_ONLY": 256, - "DECIPHER_ONLY": 128} + usage_values = { + "NO_CONSTRAINTS": 0, + "DIGITAL_SIGNATURE": 32768, + "NON_REPUDIATION": 16384, + "KEY_ENCIPHERMENT": 8192, + "DATA_ENCIPHERMENT": 4096, + "KEY_AGREEMENT": 2048, + "KEY_CERT_SIGN": 1024, + "CRL_SIGN": 512, + "ENCIPHER_ONLY": 256, + "DECIPHER_ONLY": 128, + } usage = 0 for u in usage_list: if u not in usage_values: @@ -1913,15 +2646,16 @@ def allowed_usage(self, usage_list: List[str]) -> bool: def handle_(self): return self.__obj - def verify(self, - intermediates: List[X509Cert] | None = None, - trusted: List[X509Cert] | None = None, - trusted_path: str | None = None, - required_strength: int = 0, - hostname: str | None = None, - reference_time: int = 0, - crls: List[X509CRL] | None = None) -> int: - + def verify( + self, + intermediates: List[X509Cert] | None = None, + trusted: List[X509Cert] | None = None, + trusted_path: str | None = None, + required_strength: int = 0, + hostname: str | None = None, + reference_time: int = 0, + crls: List[X509CRL] | None = None, + ) -> int: if intermediates is not None: c_intermediates = len(intermediates) * c_void_p arr_intermediates = c_intermediates() @@ -1954,18 +2688,20 @@ def verify(self, error_code = c_int(0) - _DLL.botan_x509_cert_verify_with_crl(byref(error_code), - self.__obj, - byref(arr_intermediates), - len_intermediates, - byref(arr_trusted), - len_trusted, - byref(arr_crls), - len_crls, - _ctype_str(trusted_path), - c_size_t(required_strength), - _ctype_str(hostname), - c_uint64(reference_time)) + _DLL.botan_x509_cert_verify_with_crl( + byref(error_code), + self.__obj, + byref(arr_intermediates), + len_intermediates, + byref(arr_trusted), + len_trusted, + byref(arr_crls), + len_crls, + _ctype_str(trusted_path), + c_size_t(required_strength), + _ctype_str(hostname), + c_uint64(reference_time), + ) return error_code.value @@ -1984,7 +2720,9 @@ def is_revoked(self, crl: X509CRL) -> bool: class X509CRL: def __init__(self, filename: str | None = None, buf: bytes | None = None): self.__obj = c_void_p(0) - self.__obj = _load_buf_or_file(filename, buf, _DLL.botan_x509_crl_load_file, _DLL.botan_x509_crl_load) + self.__obj = _load_buf_or_file( + filename, buf, _DLL.botan_x509_crl_load_file, _DLL.botan_x509_crl_load + ) def __del__(self): _DLL.botan_x509_crl_destroy(self.__obj) @@ -1994,18 +2732,18 @@ def handle_(self): class MPI: - def __init__(self, initial_value: _MPIArg = None, radix: int | None = None): - self.__obj = c_void_p(0) _DLL.botan_mp_init(byref(self.__obj)) if initial_value is None: - pass # left as zero + pass # left as zero elif isinstance(initial_value, MPI): _DLL.botan_mp_set_from_mp(self.__obj, initial_value.handle_()) elif radix is not None: - _DLL.botan_mp_set_from_radix_str(self.__obj, _ctype_str(initial_value), c_size_t(radix)) + _DLL.botan_mp_set_from_radix_str( + self.__obj, _ctype_str(initial_value), c_size_t(radix) + ) elif isinstance(initial_value, str): _DLL.botan_mp_set_from_str(self.__obj, _ctype_str(initial_value)) else: @@ -2021,7 +2759,9 @@ def random(cls, rng_obj: RandomNumberGenerator, bits: int) -> MPI: @classmethod def random_range(cls, rng_obj: RandomNumberGenerator, lower: MPI, upper: MPI): bn = MPI() - _DLL.botan_mp_rand_range(bn.handle_(), rng_obj.handle_(), lower.handle_(), upper.handle_()) + _DLL.botan_mp_rand_range( + bn.handle_(), rng_obj.handle_(), lower.handle_(), upper.handle_() + ) return bn def __del__(self): @@ -2031,7 +2771,7 @@ def handle_(self): return self.__obj def __int__(self): - out = create_string_buffer(2*self.byte_count() + 3) + out = create_string_buffer(2 * self.byte_count() + 3) _DLL.botan_mp_to_hex(self.__obj, out) return int(out.value, 16) @@ -2044,7 +2784,7 @@ def __repr__(self): _DLL.botan_mp_to_str(self.__obj, c_uint8(10), out, byref(out_len)) - out = out.raw[0:int(out_len.value - 1)] + out = out.raw[0 : int(out_len.value - 1)] return _ctype_to_str(out) def to_bytes(self) -> Array[c_char]: @@ -2179,7 +2919,9 @@ def __irshift__(self, shift: int): def mod_mul(self, other: MPI, modulus: MPI) -> MPI: r = MPI() - _DLL.botan_mp_mod_mul(r.handle_(), self.__obj, other.handle_(), modulus.handle_()) + _DLL.botan_mp_mod_mul( + r.handle_(), self.__obj, other.handle_(), modulus.handle_() + ) return r def gcd(self, other: MPI) -> MPI: @@ -2189,11 +2931,15 @@ def gcd(self, other: MPI) -> MPI: def pow_mod(self, exponent: MPI, modulus: MPI) -> MPI: r = MPI() - _DLL.botan_mp_powmod(r.handle_(), self.__obj, exponent.handle_(), modulus.handle_()) + _DLL.botan_mp_powmod( + r.handle_(), self.__obj, exponent.handle_(), modulus.handle_() + ) return r def is_prime(self, rng_obj: RandomNumberGenerator, prob: int = 128) -> bool: - return _DLL.botan_mp_is_prime(self.__obj, rng_obj.handle_(), c_size_t(prob)) == 1 + return ( + _DLL.botan_mp_is_prime(self.__obj, rng_obj.handle_(), c_size_t(prob)) == 1 + ) def inverse_mod(self, modulus: MPI) -> MPI: r = MPI() @@ -2239,10 +2985,14 @@ def from_string(cls, value: str) -> OID: return oid def to_string(self) -> str: - return _call_fn_viewing_str(lambda vc, vfn: _DLL.botan_oid_view_string(self.__obj, vc, vfn)) + return _call_fn_viewing_str( + lambda vc, vfn: _DLL.botan_oid_view_string(self.__obj, vc, vfn) + ) def to_name(self) -> str: - return _call_fn_viewing_str(lambda vc, vfn: _DLL.botan_oid_view_name(self.__obj, vc, vfn)) + return _call_fn_viewing_str( + lambda vc, vfn: _DLL.botan_oid_view_name(self.__obj, vc, vfn) + ) def register(self, name: str): _DLL.botan_oid_register(self.__obj, _ctype_str(name)) @@ -2318,7 +3068,9 @@ def supports_named_group(cls, name: str) -> bool: return True @classmethod - def from_params(cls, oid: OID, p: MPI, a: MPI, b: MPI, base_x: MPI, base_y: MPI, order: MPI) -> ECGroup: + def from_params( + cls, oid: OID, p: MPI, a: MPI, b: MPI, base_x: MPI, base_y: MPI, order: MPI + ) -> ECGroup: ec_group = ECGroup() _DLL.botan_ec_group_from_params( byref(ec_group.handle_()), @@ -2328,7 +3080,7 @@ def from_params(cls, oid: OID, p: MPI, a: MPI, b: MPI, base_x: MPI, base_y: MPI, b.handle_(), base_x.handle_(), base_y.handle_(), - order.handle_() + order.handle_(), ) return ec_group @@ -2357,10 +3109,14 @@ def from_name(cls, name: str) -> ECGroup: return ec_group def to_der(self) -> bytes: - return _call_fn_viewing_vec(lambda vc, vfn: _DLL.botan_ec_group_view_der(self.__obj, vc, vfn)) + return _call_fn_viewing_vec( + lambda vc, vfn: _DLL.botan_ec_group_view_der(self.__obj, vc, vfn) + ) def to_pem(self) -> str: - return _call_fn_viewing_str(lambda vc, vfn: _DLL.botan_ec_group_view_pem(self.__obj, vc, vfn)) + return _call_fn_viewing_str( + lambda vc, vfn: _DLL.botan_ec_group_view_pem(self.__obj, vc, vfn) + ) def get_curve_oid(self) -> OID: oid = OID() @@ -2408,11 +3164,14 @@ def __ne__(self, other: ECGroup | object) -> bool: class FormatPreservingEncryptionFE1: - - def __init__(self, modulus: MPI, key: bytes, rounds: int = 5, compat_mode: bool = False): + def __init__( + self, modulus: MPI, key: bytes, rounds: int = 5, compat_mode: bool = False + ): flags = c_uint32(1 if compat_mode else 0) self.__obj = c_void_p(0) - _DLL.botan_fpe_fe1_init(byref(self.__obj), modulus.handle_(), key, len(key), c_size_t(rounds), flags) + _DLL.botan_fpe_fe1_init( + byref(self.__obj), modulus.handle_(), key, len(key), c_size_t(rounds), flags + ) def __del__(self): _DLL.botan_fpe_destroy(self.__obj) @@ -2429,10 +3188,13 @@ def decrypt(self, msg: _MPIArg, tweak: str | bytes) -> MPI: _DLL.botan_fpe_decrypt(self.__obj, r.handle_(), bits, len(bits)) return r + class HOTP: def __init__(self, key: bytes, digest: str = "SHA-1", digits: int = 6): self.__obj = c_void_p(0) - _DLL.botan_hotp_init(byref(self.__obj), key, len(key), _ctype_str(digest), digits) + _DLL.botan_hotp_init( + byref(self.__obj), key, len(key), _ctype_str(digest), digits + ) def __del__(self): _DLL.botan_hotp_destroy(self.__obj) @@ -2444,16 +3206,23 @@ def generate(self, counter: int) -> int: def check(self, code: int, counter: int, resync_range: int = 0) -> tuple[bool, int]: next_ctr = c_uint64(0) - rc = _DLL.botan_hotp_check(self.__obj, byref(next_ctr), code, counter, resync_range) + rc = _DLL.botan_hotp_check( + self.__obj, byref(next_ctr), code, counter, resync_range + ) if rc == 0: return (True, next_ctr.value) else: return (False, counter) + class TOTP: - def __init__(self, key: bytes, digest: str = "SHA-1", digits: int = 6, timestep: int = 30): + def __init__( + self, key: bytes, digest: str = "SHA-1", digits: int = 6, timestep: int = 30 + ): self.__obj = c_void_p(0) - _DLL.botan_totp_init(byref(self.__obj), key, len(key), _ctype_str(digest), digits, timestep) + _DLL.botan_totp_init( + byref(self.__obj), key, len(key), _ctype_str(digest), digits, timestep + ) def __del__(self): _DLL.botan_totp_destroy(self.__obj) @@ -2465,7 +3234,9 @@ def generate(self, timestamp: int | None = None) -> int: _DLL.botan_totp_generate(self.__obj, byref(code), timestamp) return code.value - def check(self, code: int, timestamp: int | None = None, acceptable_drift: int = 0) -> bool: + def check( + self, code: int, timestamp: int | None = None, acceptable_drift: int = 0 + ) -> bool: if timestamp is None: timestamp = int(system_time()) rc = _DLL.botan_totp_check(self.__obj, code, timestamp, acceptable_drift) @@ -2473,27 +3244,42 @@ def check(self, code: int, timestamp: int | None = None, acceptable_drift: int = return True return False + def nist_key_wrap(kek: bytes, key: bytes, cipher: str | None = None) -> bytes: - cipher_algo = "AES-%d" % (8*len(kek)) if cipher is None else cipher + cipher_algo = "AES-%d" % (8 * len(kek)) if cipher is None else cipher padding = 0 output = create_string_buffer(len(key) + 8) out_len = c_size_t(len(output)) - _DLL.botan_nist_kw_enc(_ctype_str(cipher_algo), padding, - key, len(key), - kek, len(kek), - output, byref(out_len)) - return bytes(output[0:int(out_len.value)]) + _DLL.botan_nist_kw_enc( + _ctype_str(cipher_algo), + padding, + key, + len(key), + kek, + len(kek), + output, + byref(out_len), + ) + return bytes(output[0 : int(out_len.value)]) + def nist_key_unwrap(kek: bytes, wrapped: bytes, cipher: str | None = None) -> bytes: - cipher_algo = "AES-%d" % (8*len(kek)) if cipher is None else cipher + cipher_algo = "AES-%d" % (8 * len(kek)) if cipher is None else cipher padding = 0 output = create_string_buffer(len(wrapped)) out_len = c_size_t(len(output)) - _DLL.botan_nist_kw_dec(_ctype_str(cipher_algo), padding, - wrapped, len(wrapped), - kek, len(kek), - output, byref(out_len)) - return bytes(output[0:int(out_len.value)]) + _DLL.botan_nist_kw_dec( + _ctype_str(cipher_algo), + padding, + wrapped, + len(wrapped), + kek, + len(kek), + output, + byref(out_len), + ) + return bytes(output[0 : int(out_len.value)]) + class Srp6ServerSession: __obj = c_void_p(0) @@ -2502,51 +3288,91 @@ def __init__(self, group: str): _DLL.botan_srp6_server_session_init(byref(self.__obj)) self.__group = group self.__group_size = _call_fn_returning_sz( - lambda len: _DLL.botan_srp6_group_size(_ctype_str(group), len)) + lambda len: _DLL.botan_srp6_group_size(_ctype_str(group), len) + ) def __del__(self): _DLL.botan_srp6_server_session_destroy(self.__obj) def step1(self, verifier: bytes, hsh: str, rng: RandomNumberGenerator) -> bytes: - return _call_fn_returning_vec(self.__group_size, - lambda b, bl: - _DLL.botan_srp6_server_session_step1(self.__obj, - verifier, len(verifier), - _ctype_str(self.__group), - _ctype_str(hsh), - rng.handle_(), - b, bl)) + return _call_fn_returning_vec( + self.__group_size, + lambda b, bl: _DLL.botan_srp6_server_session_step1( + self.__obj, + verifier, + len(verifier), + _ctype_str(self.__group), + _ctype_str(hsh), + rng.handle_(), + b, + bl, + ), + ) def step2(self, a: bytes): - return _call_fn_returning_vec(self.__group_size, lambda k, kl: - _DLL.botan_srp6_server_session_step2(self.__obj, - a, len(a), - k, kl)) - -def srp6_generate_verifier(identifier: str, password: str, salt: bytes, group: str, hsh: str) -> bytes: - sz = _call_fn_returning_sz(lambda len: _DLL.botan_srp6_group_size(_ctype_str(group), len)) - - return _call_fn_returning_vec(sz, lambda v, vl: - _DLL.botan_srp6_generate_verifier(_ctype_str(identifier), - _ctype_str(password), - salt, len(salt), - _ctype_str(group), - _ctype_str(hsh), - v, vl)) - -def srp6_client_agree(username: str, password: str, group: str, hsh: str, salt: bytes, b: bytes, rng: RandomNumberGenerator) -> tuple[bytes, bytes]: - sz = _call_fn_returning_sz(lambda len: _DLL.botan_srp6_group_size(_ctype_str(group), len)) - - return _call_fn_returning_vec_pair(sz, sz, lambda a, al, k, kl: - _DLL.botan_srp6_client_agree(_ctype_str(username), - _ctype_str(password), - _ctype_str(group), - _ctype_str(hsh), - salt, len(salt), - b, len(b), - rng.handle_(), - a, al, - k, kl)) + return _call_fn_returning_vec( + self.__group_size, + lambda k, kl: _DLL.botan_srp6_server_session_step2( + self.__obj, a, len(a), k, kl + ), + ) + + +def srp6_generate_verifier( + identifier: str, password: str, salt: bytes, group: str, hsh: str +) -> bytes: + sz = _call_fn_returning_sz( + lambda len: _DLL.botan_srp6_group_size(_ctype_str(group), len) + ) + + return _call_fn_returning_vec( + sz, + lambda v, vl: _DLL.botan_srp6_generate_verifier( + _ctype_str(identifier), + _ctype_str(password), + salt, + len(salt), + _ctype_str(group), + _ctype_str(hsh), + v, + vl, + ), + ) + + +def srp6_client_agree( + username: str, + password: str, + group: str, + hsh: str, + salt: bytes, + b: bytes, + rng: RandomNumberGenerator, +) -> tuple[bytes, bytes]: + sz = _call_fn_returning_sz( + lambda len: _DLL.botan_srp6_group_size(_ctype_str(group), len) + ) + + return _call_fn_returning_vec_pair( + sz, + sz, + lambda a, al, k, kl: _DLL.botan_srp6_client_agree( + _ctype_str(username), + _ctype_str(password), + _ctype_str(group), + _ctype_str(hsh), + salt, + len(salt), + b, + len(b), + rng.handle_(), + a, + al, + k, + kl, + ), + ) + def zfec_encode(k: int, n: int, input_bytes: bytes) -> List[bytes]: """ @@ -2564,15 +3390,9 @@ def zfec_encode(k: int, n: int, input_bytes: bytes) -> List[bytes]: p_p_nbytes = c_char_p * n # allocate memory for the outputs (create_string_buffer makes bytes) - outputs = [ - create_string_buffer(outsize) - for a in range(n) - ] + outputs = [create_string_buffer(outsize) for a in range(n)] - c_outputs = p_p_nbytes(*[ - addressof(output) - for output in outputs - ]) + c_outputs = p_p_nbytes(*[addressof(output) for output in outputs]) # actual C call _DLL.botan_zfec_encode( @@ -2596,7 +3416,7 @@ def zfec_decode(k: int, n: int, indexes: List[int], inputs: List[bytes]) -> List """ if len(inputs) < k: - raise BotanException('Insufficient inputs for zfec decoding') + raise BotanException("Insufficient inputs for zfec decoding") p_size_t = c_size_t * len(indexes) c_indexes = p_size_t(*[c_size_t(index) for index in indexes]) @@ -2606,21 +3426,13 @@ def zfec_decode(k: int, n: int, indexes: List[int], inputs: List[bytes]) -> List share_size = len(inputs[0]) for i in inputs: if len(i) != share_size: - raise ValueError( - "Share size mismatch: {} != {}".format(len(i), share_size) - ) + raise ValueError("Share size mismatch: {} != {}".format(len(i), share_size)) # allocate memory for our outputs (create_string_buffer creates # bytes) - outputs = [ - create_string_buffer(share_size) - for _ in range(k) - ] + outputs = [create_string_buffer(share_size) for _ in range(k)] p_p_kbytes = c_char_p * k - c_outputs = p_p_kbytes(*[ - addressof(output) - for output in outputs - ]) + c_outputs = p_p_kbytes(*[addressof(output) for output in outputs]) # actual C call _DLL.botan_zfec_decode( diff --git a/src/scripts/bench.py b/src/scripts/bench.py index c461b6a8244..86047b0d417 100755 --- a/src/scripts/bench.py +++ b/src/scripts/bench.py @@ -13,7 +13,7 @@ import logging import os import sys -import optparse # pylint: disable=deprecated-module +import optparse # pylint: disable=deprecated-module import subprocess import re import json @@ -24,6 +24,7 @@ import numpy as np import matplotlib.pyplot as plt + def setup_logging(options): if options.verbose: log_level = logging.DEBUG @@ -39,28 +40,31 @@ def emit(self, record): sys.exit(1) lh = LogOnErrorHandler(sys.stdout) - lh.setFormatter(logging.Formatter('%(levelname) 7s: %(message)s')) + lh.setFormatter(logging.Formatter("%(levelname) 7s: %(message)s")) logging.getLogger().addHandler(lh) logging.getLogger().setLevel(log_level) + def run_command(cmd): - logging.debug("Running '%s'", ' '.join(cmd)) + logging.debug("Running '%s'", " ".join(cmd)) - proc = subprocess.Popen(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True) + proc = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True + ) stdout, stderr = proc.communicate() if proc.returncode != 0: - logging.error("Running command %s failed ret %d", ' '.join(cmd), proc.returncode) + logging.error( + "Running command %s failed ret %d", " ".join(cmd), proc.returncode + ) return stdout + stderr + def get_openssl_version(openssl): - output = run_command([openssl, 'version']) + output = run_command([openssl, "version"]) - openssl_version_re = re.compile(r'OpenSSL ([0-9a-z\.]+) .*') + openssl_version_re = re.compile(r"OpenSSL ([0-9a-z\.]+) .*") match = openssl_version_re.match(output) @@ -70,39 +74,35 @@ def get_openssl_version(openssl): logging.warning("Unable to parse OpenSSL version output %s", output) return output + def get_botan_version(botan): - return run_command([botan, 'version']).strip() + return run_command([botan, "version"]).strip() + EVP_MAP = { - 'AES-128/GCM': 'aes-128-gcm', - 'AES-256/GCM': 'aes-256-gcm', - 'ChaCha20': 'chacha20', - 'SHA-1': 'sha1', - 'SHA-256': 'sha256', - 'SHA-384': 'sha384', - 'SHA-512': 'sha512', - 'SHA-3(256)': 'sha3-256', - 'SHA-3(512)': 'sha3-512' - } - -SIGNATURE_EVP_MAP = { - 'RSA': 'rsa', - 'ECDSA': 'ecdsa' - } - -KEY_AGREEMENT_EVP_MAP = { - 'DH': 'ffdh', - 'ECDH': 'ecdh' - } + "AES-128/GCM": "aes-128-gcm", + "AES-256/GCM": "aes-256-gcm", + "ChaCha20": "chacha20", + "SHA-1": "sha1", + "SHA-256": "sha256", + "SHA-384": "sha384", + "SHA-512": "sha512", + "SHA-3(256)": "sha3-256", + "SHA-3(512)": "sha3-512", +} -def run_openssl_bench(openssl, algo): +SIGNATURE_EVP_MAP = {"RSA": "rsa", "ECDSA": "ecdsa"} + +KEY_AGREEMENT_EVP_MAP = {"DH": "ffdh", "ECDH": "ecdh"} - logging.info('Running OpenSSL benchmark for %s', algo) - cmd = [openssl, 'speed', '-seconds', '1', '-mr'] +def run_openssl_bench(openssl, algo): + logging.info("Running OpenSSL benchmark for %s", algo) + + cmd = [openssl, "speed", "-seconds", "1", "-mr"] if algo in EVP_MAP: - cmd += ['-evp', EVP_MAP[algo]] + cmd += ["-evp", EVP_MAP[algo]] elif algo in SIGNATURE_EVP_MAP: cmd += [SIGNATURE_EVP_MAP[algo]] elif algo in KEY_AGREEMENT_EVP_MAP: @@ -114,9 +114,9 @@ def run_openssl_bench(openssl, algo): results = [] if algo in EVP_MAP: - buf_header = re.compile(r'\+DT:([a-zA-Z0-9-]+):([0-9]+):([0-9]+)$') - res_header = re.compile(r'\+R:([0-9]+):[a-zA-Z0-9-]+:([0-9]+\.[0-9]+)$') - ignored = re.compile(r'\+(H|F):.*') + buf_header = re.compile(r"\+DT:([a-zA-Z0-9-]+):([0-9]+):([0-9]+)$") + res_header = re.compile(r"\+R:([0-9]+):[a-zA-Z0-9-]+:([0-9]+\.[0-9]+)$") + ignored = re.compile(r"\+(H|F):.*") result = {} @@ -129,19 +129,19 @@ def run_openssl_bench(openssl, algo): if match is None: logging.error("Unexpected output from OpenSSL %s", line) - result = {'algo': algo, 'buf_size': int(match.group(3))} + result = {"algo": algo, "buf_size": int(match.group(3))} else: match = res_header.match(line) - result['bytes'] = int(match.group(1)) * result['buf_size'] - result['runtime'] = float(match.group(2)) - result['bps'] = int(result['bytes'] / result['runtime']) + result["bytes"] = int(match.group(1)) * result["buf_size"] + result["runtime"] = float(match.group(2)) + result["bps"] = int(result["bytes"] / result["runtime"]) results.append(result) result = {} elif algo in SIGNATURE_EVP_MAP: - signature_ops = re.compile(r'\+(R1|R7|R5):([0-9]+):([0-9]+):([0-9]+\.[0-9]+)$') - verify_ops = re.compile(r'\+(R2|R8|R6):([0-9]+):([0-9]+):([0-9]+\.[0-9]+)$') - ignored = re.compile(r'\+(DTP|F2|R3|R4|F4):.*') + signature_ops = re.compile(r"\+(R1|R7|R5):([0-9]+):([0-9]+):([0-9]+\.[0-9]+)$") + verify_ops = re.compile(r"\+(R2|R8|R6):([0-9]+):([0-9]+):([0-9]+\.[0-9]+)$") + ignored = re.compile(r"\+(DTP|F2|R3|R4|F4):.*") result = {} @@ -150,26 +150,33 @@ def run_openssl_bench(openssl, algo): continue if match := signature_ops.match(line): - results.append({ - 'algo': algo, - 'key_size': int(match.group(3)), - 'op': 'sign', - 'ops': int(match.group(2)), - 'runtime': float(match.group(4))}) + results.append( + { + "algo": algo, + "key_size": int(match.group(3)), + "op": "sign", + "ops": int(match.group(2)), + "runtime": float(match.group(4)), + } + ) elif match := verify_ops.match(line): - results.append({ - 'algo': algo, - 'key_size': int(match.group(3)), - 'op': 'verify', - 'ops': int(match.group(2)), - 'runtime': float(match.group(4)) - }) + results.append( + { + "algo": algo, + "key_size": int(match.group(3)), + "op": "verify", + "ops": int(match.group(2)), + "runtime": float(match.group(4)), + } + ) else: logging.error("Unexpected output from OpenSSL %s", line) elif algo in KEY_AGREEMENT_EVP_MAP: - res_header = re.compile(r'\+(R7|R9|R12|R14):([0-9]+):([0-9]+):([0-9]+\.[0-9]+)$') - ignored = re.compile(r'\+(DTP|F5|F8):.*') + res_header = re.compile( + r"\+(R7|R9|R12|R14):([0-9]+):([0-9]+):([0-9]+\.[0-9]+)$" + ) + ignored = re.compile(r"\+(DTP|F5|F8):.*") result = {} @@ -178,74 +185,100 @@ def run_openssl_bench(openssl, algo): continue if match := res_header.match(line): - results.append({ - 'algo': algo, - 'key_size': int(match.group(3)), - 'ops': int(match.group(2)), - 'runtime': float(match.group(4))}) + results.append( + { + "algo": algo, + "key_size": int(match.group(3)), + "ops": int(match.group(2)), + "runtime": float(match.group(4)), + } + ) else: logging.error("Unexpected output from OpenSSL %s", line) return results -def run_botan_bench(botan, runtime, buf_sizes, algo): - - logging.info('Running Botan benchmark for %s', algo) - cmd = [botan, 'speed', '--format=json', '--msec=%d' % int(runtime * 1000), - '--buf-size=%s' % (','.join([str(i) for i in buf_sizes])), algo] +def run_botan_bench(botan, runtime, buf_sizes, algo): + logging.info("Running Botan benchmark for %s", algo) + + cmd = [ + botan, + "speed", + "--format=json", + "--msec=%d" % int(runtime * 1000), + "--buf-size=%s" % (",".join([str(i) for i in buf_sizes])), + algo, + ] output = run_command(cmd) output = json.loads(output) return output -def run_botan_signature_bench(botan, runtime, algo): - logging.info('Running Botan benchmark for %s', algo) +def run_botan_signature_bench(botan, runtime, algo): + logging.info("Running Botan benchmark for %s", algo) - cmd = [botan, 'speed', '--format=json', '--msec=%d' % int(runtime * 1000), algo] + cmd = [botan, "speed", "--format=json", "--msec=%d" % int(runtime * 1000), algo] output = run_command(cmd) output = json.loads(output) results = [] for verify in output: for sign in output: - if sign['op'] == 'sign' and verify['op'] == 'verify' and verify['algo'] == sign['algo']: - results.append({ - 'algo': algo, - 'key_size': int(re.search(r'[A-Z]+-[a-z]*([0-9]+).*', sign['algo']).group(1)), - 'op': 'sign', - 'ops': sign['events'], - 'runtime': sign['nanos'] / 1000 / 1000 / 1000, - }) - results.append({ - 'algo': algo, - 'key_size': int(re.search(r'[A-Z]+-[a-z]*([0-9]+).*', sign['algo']).group(1)), - 'op': 'verify', - 'ops': verify['events'], - 'runtime': verify['nanos'] / 1000 / 1000 / 1000, - }) + if ( + sign["op"] == "sign" + and verify["op"] == "verify" + and verify["algo"] == sign["algo"] + ): + results.append( + { + "algo": algo, + "key_size": int( + re.search(r"[A-Z]+-[a-z]*([0-9]+).*", sign["algo"]).group(1) + ), + "op": "sign", + "ops": sign["events"], + "runtime": sign["nanos"] / 1000 / 1000 / 1000, + } + ) + results.append( + { + "algo": algo, + "key_size": int( + re.search(r"[A-Z]+-[a-z]*([0-9]+).*", sign["algo"]).group(1) + ), + "op": "verify", + "ops": verify["events"], + "runtime": verify["nanos"] / 1000 / 1000 / 1000, + } + ) return results -def run_botan_key_agreement_bench(botan, runtime, algo): - logging.info('Running Botan benchmark for %s', algo) +def run_botan_key_agreement_bench(botan, runtime, algo): + logging.info("Running Botan benchmark for %s", algo) - cmd = [botan, 'speed', '--format=json', '--msec=%d' % int(runtime * 1000), algo] + cmd = [botan, "speed", "--format=json", "--msec=%d" % int(runtime * 1000), algo] output = run_command(cmd) output = json.loads(output) results = [] for res in output: - if res['op'] == 'key agreements': - results.append({ - 'algo': algo, - 'key_size': int(re.search(r'[A-Z]+-[a-z]*([0-9]+).*', res['algo']).group(1)), - 'ops': res['events'], - 'runtime': res['nanos'] / 1000 / 1000 / 1000, - }) + if res["op"] == "key agreements": + results.append( + { + "algo": algo, + "key_size": int( + re.search(r"[A-Z]+-[a-z]*([0-9]+).*", res["algo"]).group(1) + ), + "ops": res["events"], + "runtime": res["nanos"] / 1000 / 1000 / 1000, + } + ) return results + class BenchmarkResult: def __init__(self, algo, sizes, openssl_results, botan_results): self.algo = algo @@ -253,30 +286,30 @@ def __init__(self, algo, sizes, openssl_results, botan_results): def find_result(results, sz): for r in results: - if 'buf_size' in r and r['buf_size'] == sz: - return r['bps'] + if "buf_size" in r and r["buf_size"] == sz: + return r["bps"] raise Exception("Could not find expected result in data") for size in sizes: self.results[size] = { - 'openssl': find_result(openssl_results, size), - 'botan': find_result(botan_results, size) + "openssl": find_result(openssl_results, size), + "botan": find_result(botan_results, size), } def result_string(self): - out = "" - for (k, v) in self.results.items(): - - if v['openssl'] > v['botan']: - winner = 'openssl' - ratio = float(v['openssl']) / v['botan'] + for k, v in self.results.items(): + if v["openssl"] > v["botan"]: + winner = "openssl" + ratio = float(v["openssl"]) / v["botan"] else: - winner = 'botan' - ratio = float(v['botan']) / v['openssl'] + winner = "botan" + ratio = float(v["botan"]) / v["openssl"] - out += "algo %s buf_size % 6d botan % 12d bps openssl % 12d bps adv %s by %.02f\n" % ( - self.algo, k, v['botan'], v['openssl'], winner, ratio) + out += ( + "algo %s buf_size % 6d botan % 12d bps openssl % 12d bps adv %s by %.02f\n" + % (self.algo, k, v["botan"], v["openssl"], winner, ratio) + ) return out def graphs(self): @@ -285,20 +318,22 @@ def graphs(self): width = 0.4 multiplier = 0 - fig, ax = plt.subplots(layout='constrained') + fig, ax = plt.subplots(layout="constrained") - for lib in ['Botan', 'OpenSSL']: + for lib in ["Botan", "OpenSSL"]: offset = width * multiplier - results = [math.ceil(x[lib.lower()]/(1024*1024)) for x in self.results.values()] + results = [ + math.ceil(x[lib.lower()] / (1024 * 1024)) for x in self.results.values() + ] rects = ax.bar(x + offset, results, width, label=lib) ax.bar_label(rects, padding=5) multiplier += 1 - ax.set_xlabel('Buffer size (bytes)') - ax.set_ylabel('MiB/sec') + ax.set_xlabel("Buffer size (bytes)") + ax.set_ylabel("MiB/sec") ax.set_title(self.algo) ax.set_xticks(x + width, buf_sizes) - ax.legend(loc='upper left', ncols=2) + ax.legend(loc="upper left", ncols=2) return {self.algo: fig} @@ -311,33 +346,38 @@ def __init__(self, algo, sizes, openssl_results, botan_results): def find_result(results, sz): for verify in results: for sign in results: - if sign['op'] == 'sign' and verify['op'] == 'verify' and verify['key_size'] == sz: - return {'sign': sign['ops'], 'verify': verify['ops']} + if ( + sign["op"] == "sign" + and verify["op"] == "verify" + and verify["key_size"] == sz + ): + return {"sign": sign["ops"], "verify": verify["ops"]} raise Exception("Could not find expected result in data") for size in sizes: self.results[size] = { - 'openssl': find_result(openssl_results, size), - 'botan': find_result(botan_results, size) + "openssl": find_result(openssl_results, size), + "botan": find_result(botan_results, size), } def result_string(self): - out = "" - for (k, v) in self.results.items(): - openssl = v['openssl'] - botan = v['botan'] + for k, v in self.results.items(): + openssl = v["openssl"] + botan = v["botan"] for op in openssl.keys(): if openssl[op] > botan[op]: - winner = 'openssl' + winner = "openssl" ratio = float(openssl[op]) / botan[op] else: - winner = 'botan' + winner = "botan" ratio = float(botan[op]) / openssl[op] - out += "algo %s key_size % 5d % 8s botan % 10d openssl % 10d adv %s by %.02f\n" % ( - self.algo, k, op, botan[op], openssl[op], winner, ratio) + out += ( + "algo %s key_size % 5d % 8s botan % 10d openssl % 10d adv %s by %.02f\n" + % (self.algo, k, op, botan[op], openssl[op], winner, ratio) + ) return out @@ -348,11 +388,11 @@ def graphs(self): graphs = {} - for op in ['sign', 'verify']: - fig, ax = plt.subplots(layout='constrained') + for op in ["sign", "verify"]: + fig, ax = plt.subplots(layout="constrained") multiplier = 0 - for lib in ['Botan', 'OpenSSL']: + for lib in ["Botan", "OpenSSL"]: offset = width * multiplier results = [x[lib.lower()][op] for x in self.results.values()] rects = ax.bar(x + offset, results, width, label=lib) @@ -360,13 +400,13 @@ def graphs(self): multiplier += 1 # Add some text for labels, title and custom x-axis tick labels, etc. - ax.set_xlabel('Key size (bits)') - ax.set_ylabel('Ops/sec') - ax.set_title(self.algo + ' ' + op) + ax.set_xlabel("Key size (bits)") + ax.set_ylabel("Ops/sec") + ax.set_title(self.algo + " " + op) ax.set_xticks(x + width, key_sizes) - ax.legend(loc='upper right', ncols=2) + ax.legend(loc="upper right", ncols=2) - graphs.update({self.algo + '_' + op: fig}) + graphs.update({self.algo + "_" + op: fig}) return graphs @@ -378,30 +418,30 @@ def __init__(self, algo, sizes, openssl_results, botan_results): def find_result(results, sz): for r in results: - if 'key_size' in r and r['key_size'] == sz: - return r['ops'] + if "key_size" in r and r["key_size"] == sz: + return r["ops"] raise Exception("Could not find expected result in data") for size in sizes: self.results[size] = { - 'openssl': find_result(openssl_results, size), - 'botan': find_result(botan_results, size) + "openssl": find_result(openssl_results, size), + "botan": find_result(botan_results, size), } def result_string(self): - out = "" - for (k, v) in self.results.items(): - - if v['openssl'] > v['botan']: - winner = 'openssl' - ratio = float(v['openssl']) / v['botan'] + for k, v in self.results.items(): + if v["openssl"] > v["botan"]: + winner = "openssl" + ratio = float(v["openssl"]) / v["botan"] else: - winner = 'botan' - ratio = float(v['botan']) / v['openssl'] + winner = "botan" + ratio = float(v["botan"]) / v["openssl"] - out += "algo %s key_size % 6d botan % 12d key agreements openssl % 12d key agreements adv %s by %.02f\n" % ( - self.algo, k, v['botan'], v['openssl'], winner, ratio) + out += ( + "algo %s key_size % 6d botan % 12d key agreements openssl % 12d key agreements adv %s by %.02f\n" + % (self.algo, k, v["botan"], v["openssl"], winner, ratio) + ) return out def graphs(self): @@ -410,20 +450,20 @@ def graphs(self): width = 0.4 multiplier = 0 - fig, ax = plt.subplots(layout='constrained') + fig, ax = plt.subplots(layout="constrained") - for lib in ['Botan', 'OpenSSL']: + for lib in ["Botan", "OpenSSL"]: offset = width * multiplier results = [x[lib.lower()] for x in self.results.values()] rects = ax.bar(x + offset, results, width, label=lib) ax.bar_label(rects, padding=5) multiplier += 1 - ax.set_xlabel('Key size (bits)') - ax.set_ylabel('Key agreements/sec') + ax.set_xlabel("Key size (bits)") + ax.set_ylabel("Key agreements/sec") ax.set_title(self.algo) ax.set_xticks(x + width, key_sizes) - ax.legend(loc='upper right', ncols=2) + ax.legend(loc="upper right", ncols=2) return {self.algo: fig} @@ -431,34 +471,44 @@ def graphs(self): def bench_algo(openssl, botan, algo): openssl_results = run_openssl_bench(openssl, algo) - buf_sizes = sorted([x['buf_size'] for x in openssl_results]) - runtime = sum(x['runtime'] for x in openssl_results) / len(openssl_results) / len(buf_sizes) + buf_sizes = sorted([x["buf_size"] for x in openssl_results]) + runtime = ( + sum(x["runtime"] for x in openssl_results) + / len(openssl_results) + / len(buf_sizes) + ) botan_results = run_botan_bench(botan, runtime, buf_sizes, algo) return BenchmarkResult(algo, buf_sizes, openssl_results, botan_results) + def bench_signature_algo(openssl, botan, algo): openssl_results = run_openssl_bench(openssl, algo) - runtime = sum(x['runtime'] for x in openssl_results) / len(openssl_results) + runtime = sum(x["runtime"] for x in openssl_results) / len(openssl_results) botan_results = run_botan_signature_bench(botan, runtime, algo) - kszs_ossl = {x['key_size'] for x in openssl_results} - kszs_botan = {x['key_size'] for x in botan_results} + kszs_ossl = {x["key_size"] for x in openssl_results} + kszs_botan = {x["key_size"] for x in botan_results} + + return SignatureBenchmarkResult( + algo, sorted(kszs_ossl.intersection(kszs_botan)), openssl_results, botan_results + ) - return SignatureBenchmarkResult(algo, sorted(kszs_ossl.intersection(kszs_botan)), openssl_results, botan_results) def bench_key_agreement_algo(openssl, botan, algo): openssl_results = run_openssl_bench(openssl, algo) - runtime = sum(x['runtime'] for x in openssl_results) / len(openssl_results) + runtime = sum(x["runtime"] for x in openssl_results) / len(openssl_results) botan_results = run_botan_key_agreement_bench(botan, runtime, algo) - kszs_ossl = {x['key_size'] for x in openssl_results} - kszs_botan = {x['key_size'] for x in botan_results} + kszs_ossl = {x["key_size"] for x in openssl_results} + kszs_botan = {x["key_size"] for x in botan_results} - return KeyAgreementBenchmarkResult(algo, sorted(kszs_ossl.intersection(kszs_botan)), openssl_results, botan_results) + return KeyAgreementBenchmarkResult( + algo, sorted(kszs_ossl.intersection(kszs_botan)), openssl_results, botan_results + ) def gen_html_report(botan_version, openssl_version, cmdline): @@ -484,7 +534,7 @@ def gen_html_report(botan_version, openssl_version, cmdline):

Results

""" - for img in sorted(os.listdir('bench_html/img')): + for img in sorted(os.listdir("bench_html/img")): html += '' % img html += """ @@ -493,27 +543,35 @@ def gen_html_report(botan_version, openssl_version, cmdline): """ os_info = platform.system() - os_info += ' ' + platform.release() - os_info += ' (' + os_info += " " + platform.release() + os_info += " (" os_info += platform.machine() - os_info += ')' + os_info += ")" - include_path = re.search(r'-I(.*)', run_command(['botan', 'config', 'cflags']).strip()).group(1) - build_h = 'N/A' + include_path = re.search( + r"-I(.*)", run_command(["botan", "config", "cflags"]).strip() + ).group(1) + build_h = "N/A" try: - with open(os.path.join(include_path, 'botan/build.h'), 'r', encoding='utf-8') as f: + with open( + os.path.join(include_path, "botan/build.h"), "r", encoding="utf-8" + ) as f: build_h = f.read() except Exception as e: logging.warning("Unable to read build.h: %s", e) - with open('bench_html/index.html', 'w', encoding='utf-8') as f: - f.write(html.format(date=datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3], - os_info=os_info, - botan_version=botan_version, - build_h=build_h, - openssl_version=openssl_version, - cmdline=cmdline)) + with open("bench_html/index.html", "w", encoding="utf-8") as f: + f.write( + html.format( + date=datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3], + os_info=os_info, + botan_version=botan_version, + build_h=build_h, + openssl_version=openssl_version, + cmdline=cmdline, + ) + ) def main(args=None): @@ -523,18 +581,28 @@ def main(args=None): usage = "usage: %prog [options] [algo1] [algo2] [...]" parser = optparse.OptionParser(usage=usage) - parser.add_option('--verbose', action='store_true', default=False, help="be noisy") - parser.add_option('--quiet', action='store_true', default=False, help="be very quiet") - - parser.add_option('--html-report', action='store_true', default=False, help="create HTML report") - - parser.add_option('--openssl-cli', metavar='PATH', - default='/usr/bin/openssl', - help='Path to openssl binary (default %default)') - - parser.add_option('--botan-cli', metavar='PATH', - default='/usr/bin/botan', - help='Path to botan binary (default %default)') + parser.add_option("--verbose", action="store_true", default=False, help="be noisy") + parser.add_option( + "--quiet", action="store_true", default=False, help="be very quiet" + ) + + parser.add_option( + "--html-report", action="store_true", default=False, help="create HTML report" + ) + + parser.add_option( + "--openssl-cli", + metavar="PATH", + default="/usr/bin/openssl", + help="Path to openssl binary (default %default)", + ) + + parser.add_option( + "--botan-cli", + metavar="PATH", + default="/usr/bin/botan", + help="Path to botan binary (default %default)", + ) (options, args) = parser.parse_args(args) @@ -555,11 +623,11 @@ def main(args=None): logging.info("Comparing Botan %s with OpenSSL %s", botan_version, openssl_version) if options.html_report: - if os.path.exists('bench_html'): - shutil.rmtree('bench_html') + if os.path.exists("bench_html"): + shutil.rmtree("bench_html") - os.mkdir('bench_html') - os.mkdir('bench_html/img') + os.mkdir("bench_html") + os.mkdir("bench_html/img") graphs = {} @@ -598,11 +666,12 @@ def main(args=None): if options.html_report: for title, graph in graphs.items(): - graph.savefig('bench_html/img/' + title.replace('/', '--') + ".png") + graph.savefig("bench_html/img/" + title.replace("/", "--") + ".png") - gen_html_report(botan_version, openssl_version, ' '.join(sys.argv)) + gen_html_report(botan_version, openssl_version, " ".join(sys.argv)) return 0 -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/build_docs.py b/src/scripts/build_docs.py index 2c7eafd0597..7dc4f222a57 100755 --- a/src/scripts/build_docs.py +++ b/src/scripts/build_docs.py @@ -9,7 +9,7 @@ """ import sys -import optparse # pylint: disable=deprecated-module +import optparse # pylint: disable=deprecated-module import subprocess import shutil import logging @@ -19,6 +19,7 @@ import stat import multiprocessing + def get_concurrency(): """ Get default concurrency level of build @@ -30,32 +31,35 @@ def get_concurrency(): except ImportError: return def_concurrency + def have_prog(prog): """ Check if some named program exists in the path """ - for path in os.environ['PATH'].split(os.pathsep): + for path in os.environ["PATH"].split(os.pathsep): exe_file = os.path.join(path, prog) if os.path.exists(exe_file) and os.access(exe_file, os.X_OK): return True return False + def find_rst2man(): - possible_names = ['rst2man', 'rst2man.py', 'rst2man-%d.%d' % sys.version_info[:2]] + possible_names = ["rst2man", "rst2man.py", "rst2man-%d.%d" % sys.version_info[:2]] for name in possible_names: if have_prog(name): return name raise Exception("Was configured with rst2man but could not be located in PATH") + def touch(fname): try: os.utime(fname, None) except OSError: - open(fname, 'a', encoding='utf8').close() + open(fname, "a", encoding="utf8").close() -def copy_files(src_path, dest_dir): +def copy_files(src_path, dest_dir): logging.debug("Copying %s to %s", src_path, dest_dir) file_mode = os.stat(src_path).st_mode @@ -78,9 +82,9 @@ def copy_files(src_path, dest_dir): elif stat.S_ISDIR(file_mode): copy_files(os.path.join(src_path, f), os.path.join(dest_dir, f)) -def run_and_check(cmd_line, cwd=None): - logging.info("Starting %s", ' '.join(cmd_line)) +def run_and_check(cmd_line, cwd=None): + logging.info("Starting %s", " ".join(cmd_line)) rc = None @@ -90,25 +94,38 @@ def run_and_check(cmd_line, cwd=None): proc.communicate() rc = proc.returncode except OSError as e: - logging.error("Executing %s failed (%s)", ' '.join(cmd_line), e) + logging.error("Executing %s failed (%s)", " ".join(cmd_line), e) if rc != 0: - logging.error("Error running %s", ' '.join(cmd_line)) + logging.error("Error running %s", " ".join(cmd_line)) sys.exit(1) def parse_options(args): parser = optparse.OptionParser() - parser.add_option('--verbose', action='store_true', default=False, - help='Show debug messages') - parser.add_option('--quiet', action='store_true', default=False, - help='Show only warnings and errors') - - parser.add_option('--build-dir', metavar='DIR', default='build', - help='Location of build output (default \'%default\')') - parser.add_option('--dry-run', default=False, action='store_true', - help='Just display what would be done') + parser.add_option( + "--verbose", action="store_true", default=False, help="Show debug messages" + ) + parser.add_option( + "--quiet", + action="store_true", + default=False, + help="Show only warnings and errors", + ) + + parser.add_option( + "--build-dir", + metavar="DIR", + default="build", + help="Location of build output (default '%default')", + ) + parser.add_option( + "--dry-run", + default=False, + action="store_true", + help="Just display what would be done", + ) (options, args) = parser.parse_args(args) @@ -127,85 +144,103 @@ def log_level(): return options + def read_config(config): try: - f = open(config, encoding='utf8') + f = open(config, encoding="utf8") cfg = json.load(f) f.close() except OSError as ex: - raise Exception('Failed to load build config %s - is build dir correct?' % (config)) from ex + raise Exception( + "Failed to load build config %s - is build dir correct?" % (config) + ) from ex return cfg -def main(args=None): +def main(args=None): if args is None: args = sys.argv - logging.basicConfig(stream=sys.stdout, - format='%(levelname) 7s: %(message)s') + logging.basicConfig(stream=sys.stdout, format="%(levelname) 7s: %(message)s") options = parse_options(args) if options is None: return 1 - cfg = read_config(os.path.join(options.build_dir, 'build_config.json')) + cfg = read_config(os.path.join(options.build_dir, "build_config.json")) - with_docs = bool(cfg['with_documentation']) - with_sphinx = bool(cfg['with_sphinx']) - with_pdf = bool(cfg['with_pdf']) - with_rst2man = bool(cfg['with_rst2man']) - with_doxygen = bool(cfg['with_doxygen']) + with_docs = bool(cfg["with_documentation"]) + with_sphinx = bool(cfg["with_sphinx"]) + with_pdf = bool(cfg["with_pdf"]) + with_rst2man = bool(cfg["with_rst2man"]) + with_doxygen = bool(cfg["with_doxygen"]) - doc_stamp_file = cfg['doc_stamp_file'] + doc_stamp_file = cfg["doc_stamp_file"] - handbook_src = cfg['doc_dir'] - handbook_output = cfg['handbook_output_dir'] + handbook_src = cfg["doc_dir"] + handbook_output = cfg["handbook_output_dir"] if with_docs is False: - logging.debug('Documentation build disabled') + logging.debug("Documentation build disabled") return 1 cmds = [] if with_doxygen: - cmds.append(['doxygen', os.path.join(cfg['build_dir'], 'botan.doxy')]) + cmds.append(["doxygen", os.path.join(cfg["build_dir"], "botan.doxy")]) if with_sphinx: - sphinx_build = ['sphinx-build', '-q', '-c', cfg['sphinx_config_dir'], '-j', 'auto', '-W', '--keep-going'] - - cmds.append(sphinx_build + ['-b', 'html', handbook_src, handbook_output]) + sphinx_build = [ + "sphinx-build", + "-q", + "-c", + cfg["sphinx_config_dir"], + "-j", + "auto", + "-W", + "--keep-going", + ] + + cmds.append(sphinx_build + ["-b", "html", handbook_src, handbook_output]) if with_pdf: - latex_output = tempfile.mkdtemp(prefix='botan_latex_') - cmds.append(sphinx_build + ['-b', 'latex', handbook_src, latex_output]) - cmds.append(['make', '-C', latex_output]) - cmds.append(['cp', os.path.join(latex_output, 'botan.pdf'), handbook_output]) + latex_output = tempfile.mkdtemp(prefix="botan_latex_") + cmds.append(sphinx_build + ["-b", "latex", handbook_src, latex_output]) + cmds.append(["make", "-C", latex_output]) + cmds.append( + ["cp", os.path.join(latex_output, "botan.pdf"), handbook_output] + ) else: # otherwise just copy it - cmds.append(['cp', handbook_src, handbook_output]) + cmds.append(["cp", handbook_src, handbook_output]) if with_rst2man: - cmds.append([find_rst2man(), - os.path.join(cfg['build_dir'], 'botan.rst'), - os.path.join(cfg['build_dir'], 'botan.1')]) + cmds.append( + [ + find_rst2man(), + os.path.join(cfg["build_dir"], "botan.rst"), + os.path.join(cfg["build_dir"], "botan.1"), + ] + ) - cmds.append(['touch', doc_stamp_file]) + cmds.append(["touch", doc_stamp_file]) for cmd in cmds: if options.dry_run: - print(' '.join(cmd)) + print(" ".join(cmd)) else: - if cmd[0] == 'cp': + if cmd[0] == "cp": assert len(cmd) == 3 copy_files(cmd[1], cmd[2]) - elif cmd[0] == 'touch': + elif cmd[0] == "touch": assert len(cmd) == 2 touch(cmd[1]) else: run_and_check(cmd) return 0 -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/check.py b/src/scripts/check.py index cc75f3a2357..0087efe961f 100755 --- a/src/scripts/check.py +++ b/src/scripts/check.py @@ -10,23 +10,23 @@ import json import logging -import optparse # pylint: disable=deprecated-module +import optparse # pylint: disable=deprecated-module import os import subprocess import sys -def run_and_check(cmd_line, env=None, cwd=None): - logging.info("Starting %s", ' '.join(cmd_line)) +def run_and_check(cmd_line, env=None, cwd=None): + logging.info("Starting %s", " ".join(cmd_line)) try: proc = subprocess.Popen(cmd_line, cwd=cwd, env=env) proc.communicate() except OSError as e: - logging.error("Executing %s failed (%s)", ' '.join(cmd_line), e) + logging.error("Executing %s failed (%s)", " ".join(cmd_line), e) if proc.returncode != 0: - logging.error("Error running %s", ' '.join(cmd_line)) + logging.error("Error running %s", " ".join(cmd_line)) sys.exit(1) @@ -47,8 +47,12 @@ def extend_colon_list(k, n): def parse_options(args): parser = optparse.OptionParser() - parser.add_option('--build-dir', default='build', metavar='DIR', - help='specify the botan build directory (default %default)') + parser.add_option( + "--build-dir", + default="build", + metavar="DIR", + help="specify the botan build directory (default %default)", + ) (options, args) = parser.parse_args(args) @@ -60,10 +64,12 @@ def parse_options(args): def read_config(config): try: - with open(config, encoding='utf8') as f: + with open(config, encoding="utf8") as f: return json.load(f) except OSError as ex: - raise Exception('Failed to load build config %s - is build dir correct?' % (config)) from ex + raise Exception( + "Failed to load build config %s - is build dir correct?" % (config) + ) from ex def main(args=None): @@ -72,18 +78,21 @@ def main(args=None): options = parse_options(args) - cfg = read_config(os.path.join(options.build_dir, 'build_config.json')) + cfg = read_config(os.path.join(options.build_dir, "build_config.json")) - test_exe = cfg.get('test_exe') - build_shared_lib = cfg.get('build_shared_lib') + test_exe = cfg.get("test_exe") + build_shared_lib = cfg.get("build_shared_lib") if not os.path.isfile(test_exe) or not os.access(test_exe, os.X_OK): raise Exception("Test binary not built") - run_and_check([test_exe, "--data-dir=%s" % cfg.get('test_data_dir')], - make_environment(build_shared_lib)) + run_and_check( + [test_exe, "--data-dir=%s" % cfg.get("test_data_dir")], + make_environment(build_shared_lib), + ) return 0 -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/ci/ci_tlsanvil_check.py b/src/scripts/ci/ci_tlsanvil_check.py index b81b0310564..7f6737fd74d 100644 --- a/src/scripts/ci/ci_tlsanvil_check.py +++ b/src/scripts/ci/ci_tlsanvil_check.py @@ -20,7 +20,7 @@ def expected_result_for(method_id: str): - """ Get the expected result for a given test id """ + """Get the expected result for a given test id""" allowed_to_conceptually_succeed = { # Okay: RFC does not specifically define an alert. Bogo Test expects an DecodeError Alert # while TLS-Anvil expects an IllegalParameter Alert. We use the DecodeError Alert. @@ -30,7 +30,7 @@ def expected_result_for(method_id: str): # TODO: Analyze partially failing tests and document if/why they are allowed to fail allowed_to_partially_fail = { "server.tls12.statemachine.StateMachine.earlyChangeCipherSpec", - "server.tls12.rfc7568.DoNotUseSSLVersion30.sendClientHelloVersion0300RecordVersion" + "server.tls12.rfc7568.DoNotUseSSLVersion30.sendClientHelloVersion0300RecordVersion", } # TODO: Analyze failing tests and document if/why they are allowed to fail @@ -43,7 +43,7 @@ def expected_result_for(method_id: str): "server.tls12.rfc8422.TLSExtensionForECC.rejectsInvalidCurvePoints", "server.tls12.rfc5246.ClientHello.leaveOutExtensions", "server.tls12.rfc5246.E1CompatibilityWithTLS10_11andSSL30.acceptAnyRecordVersionNumber", - "both.tls13.rfc8446.KeyUpdate.appDataUnderNewKeysSucceeds" + "both.tls13.rfc8446.KeyUpdate.appDataUnderNewKeysSucceeds", } if method_id in allowed_to_fully_fail: @@ -67,39 +67,63 @@ def test_result_valid(method_id: str, result: str): expected_res = expected_result_for(method_id) if result_level[result] < expected_res: - logging.warning("Warning: Test result better than expected for '%s'. Consider tighten the expectation.", method_id) + logging.warning( + "Warning: Test result better than expected for '%s'. Consider tighten the expectation.", + method_id, + ) return result_level[result] <= expected_result_for(method_id) def failing_test_info(json_data, method_id) -> str: - """ Print debug information about a failing test """ + """Print debug information about a failing test""" info_str = "" try: - method_class, method_name = method_id.rsplit('.', 1) + method_class, method_name = method_id.rsplit(".", 1) info = [f"Error: {method_id} - Unexpected result '{json_data['Result']}'"] info += [""] info += [f"Class Name: 'de.rub.nds.tlstest.suite.tests.{method_class}'"] info += [f"Method Name: '{method_name}'"] info += [""] - if json_data['TestMethod']['RFC'] is not None: - info += [ f"RFC {json_data['TestMethod']['RFC']['number']}, Section {json_data['TestMethod']['RFC']['Section']}:"] + if json_data["TestMethod"]["RFC"] is not None: + info += [ + f"RFC {json_data['TestMethod']['RFC']['number']}, Section {json_data['TestMethod']['RFC']['Section']}:" + ] else: info += ["Custom Test Case:"] info += [f"{json_data['TestMethod']['Description']}"] info += [""] - info += [f"Result: {json_data['Result']} (expected {list(result_level.keys())[list(result_level.values()).index(expected_result_for(method_id))]})"] - if json_data['DisabledReason'] is not None: + info += [ + f"Result: {json_data['Result']} (expected {list(result_level.keys())[list(result_level.values()).index(expected_result_for(method_id))]})" + ] + if json_data["DisabledReason"] is not None: info += [f"Disabled Reason: {json_data['DisabledReason']}"] - - additional_res_info = list({state["AdditionalResultInformation"] for state in json_data['States'] if state["AdditionalResultInformation"] != ""}) - additional_test_info = list({state["AdditionalTestInformation"] for state in json_data['States'] if state["AdditionalTestInformation"] != ""}) - state_result = [{state["Result"] for state in json_data['States']}] - - if len(state_result) > 1 or len(additional_res_info) > 1 or len(additional_test_info) > 1: - info += ["Different results for different states. See test results artifact for more information."] + additional_res_info = list( + { + state["AdditionalResultInformation"] + for state in json_data["States"] + if state["AdditionalResultInformation"] != "" + } + ) + additional_test_info = list( + { + state["AdditionalTestInformation"] + for state in json_data["States"] + if state["AdditionalTestInformation"] != "" + } + ) + state_result = [{state["Result"] for state in json_data["States"]}] + + if ( + len(state_result) > 1 + or len(additional_res_info) > 1 + or len(additional_test_info) > 1 + ): + info += [ + "Different results for different states. See test results artifact for more information." + ] if len(additional_res_info) == 1: info += ["", f"Additional Result Info: {additional_res_info[0]}"] @@ -111,7 +135,9 @@ def failing_test_info(json_data, method_id) -> str: info_str = "\n".join(info) # Color in red - info_str = "\n".join([f"\033[0;31m{line}\033[0m" for line in info_str.split("\n")]) + info_str = "\n".join( + [f"\033[0;31m{line}\033[0m" for line in info_str.split("\n")] + ) # In GitHub Actions logging group info_str = f"::group::{info_str}\n::endgroup::" diff --git a/src/scripts/ci/ci_tlsanvil_test.py b/src/scripts/ci/ci_tlsanvil_test.py index 8f5120cac93..7207e024756 100644 --- a/src/scripts/ci/ci_tlsanvil_test.py +++ b/src/scripts/ci/ci_tlsanvil_test.py @@ -12,9 +12,10 @@ class Config: - """ Hardcoded configurations for this CI script """ + """Hardcoded configurations for this CI script""" + tls_anvil_docker_image = "ghcr.io/tls-attacker/tlsanvil" - tls_anvil_version_tag = "@sha256:e9abe034e6b1dac7fe204d524db338f379087a16aca71e94dc7b51ac835bb53f" # v1.2.2 + HelloRetry test fix + tls_anvil_version_tag = "@sha256:e9abe034e6b1dac7fe204d524db338f379087a16aca71e94dc7b51ac835bb53f" # v1.2.2 + HelloRetry test fix key_and_cert_storage_path = "/tmp/" test_suite_results_dest = "." test_suite_results_dir_name = "TestSuiteResults" @@ -23,8 +24,13 @@ class Config: server_dest_ip = "127.0.0.1" server_dest_port = 4433 botan_server_log = "./logs/botan_server.log" - botan_config_args = ["--compiler-cache=ccache", "--build-targets=static,cli", - "--without-documentation", "--with-boost"] + botan_config_args = [ + "--compiler-cache=ccache", + "--build-targets=static,cli", + "--without-documentation", + "--with-boost", + ] + def group_output(group_title: str, func): """ @@ -33,11 +39,13 @@ def group_output(group_title: str, func): Returns the wrapped function """ + def wrapped_func(*args, **kwargs): print(f"::group::{group_title}", flush=True) ret = func(*args, **kwargs) print("\n::endgroup::", flush=True) return ret + return wrapped_func @@ -50,13 +58,23 @@ def create_cert_and_key(botan_dir_path): """ key_path = os.path.join(Config.key_and_cert_storage_path, Config.tmp_key_file_name) - cert_path = os.path.join(Config.key_and_cert_storage_path, Config.tmp_cert_file_name) - - with open(key_path, 'w', encoding='utf-8') as keyfile: - subprocess.run([botan_dir_path, "keygen", "--algo=RSA", "--params=2048"], stdout=keyfile, check=True) - - with open(cert_path, 'w', encoding='utf-8') as certfile: - subprocess.run([botan_dir_path, "gen_self_signed", key_path, "localhost"], stdout=certfile, check=True) + cert_path = os.path.join( + Config.key_and_cert_storage_path, Config.tmp_cert_file_name + ) + + with open(key_path, "w", encoding="utf-8") as keyfile: + subprocess.run( + [botan_dir_path, "keygen", "--algo=RSA", "--params=2048"], + stdout=keyfile, + check=True, + ) + + with open(cert_path, "w", encoding="utf-8") as certfile: + subprocess.run( + [botan_dir_path, "gen_self_signed", key_path, "localhost"], + stdout=certfile, + check=True, + ) return (cert_path, key_path) @@ -67,47 +85,72 @@ def build_botan(botan_dir: str, parallel_jobs: int) -> str: Returns the botan executable path """ - group_output("Configure Botan", subprocess.run)(["python3", "./configure.py"] + Config.botan_config_args, check=True, cwd=botan_dir) - group_output("Build Botan with Make", subprocess.run)(["make", f"-j{parallel_jobs}"], check=True, cwd=botan_dir) + group_output("Configure Botan", subprocess.run)( + ["python3", "./configure.py"] + Config.botan_config_args, + check=True, + cwd=botan_dir, + ) + group_output("Build Botan with Make", subprocess.run)( + ["make", f"-j{parallel_jobs}"], check=True, cwd=botan_dir + ) return os.path.join(botan_dir, "botan") def server_test(botan_dir_path: str, parallel: int): - """ Test the Botan TLS server """ + """Test the Botan TLS server""" cert_path, key_path = create_cert_and_key(botan_dir_path) docker_img = f"{Config.tls_anvil_docker_image}{Config.tls_anvil_version_tag}" - group_output("Pull TLS-Anvil image", subprocess.run)(["docker", "pull", docker_img], check=True) + group_output("Pull TLS-Anvil image", subprocess.run)( + ["docker", "pull", docker_img], check=True + ) tls_anvil_cmd = [ - "docker", "run", - "--network", "host", - "-v", f"{Config.test_suite_results_dest}:/output", + "docker", + "run", + "--network", + "host", + "-v", + f"{Config.test_suite_results_dest}:/output", docker_img, - "-strength", "1", - "-parallelHandshakes", str(parallel), + "-strength", + "1", + "-parallelHandshakes", + str(parallel), "-disableTcpDump", - "-outputFolder", os.path.join(Config.test_suite_results_dest, Config.test_suite_results_dir_name), - "-connectionTimeout", "5000", - "server", "-connect", f"{Config.server_dest_ip}:{Config.server_dest_port}" + "-outputFolder", + os.path.join( + Config.test_suite_results_dest, Config.test_suite_results_dir_name + ), + "-connectionTimeout", + "5000", + "server", + "-connect", + f"{Config.server_dest_ip}:{Config.server_dest_port}", ] botan_server_cmd = [ - botan_dir_path, "tls_http_server", cert_path, key_path, f"--port={Config.server_dest_port}" + botan_dir_path, + "tls_http_server", + cert_path, + key_path, + f"--port={Config.server_dest_port}", ] os.makedirs(os.path.dirname(Config.botan_server_log), exist_ok=True) # Run Botan and test is with TLS-Anvil - with open(Config.botan_server_log, 'w', encoding='utf-8') as server_log_file: - botan_server_process = subprocess.Popen(botan_server_cmd, stdout=server_log_file, stderr=server_log_file) + with open(Config.botan_server_log, "w", encoding="utf-8") as server_log_file: + botan_server_process = subprocess.Popen( + botan_server_cmd, stdout=server_log_file, stderr=server_log_file + ) subprocess.run(tls_anvil_cmd, check=True) botan_server_process.kill() def client_test(botan_dir_path: str, parallel: int): - """ Test the Botan TLS server """ + """Test the Botan TLS server""" raise NotImplementedError("Client tests not yet implemented") @@ -117,8 +160,15 @@ def main(args=None): parser = argparse.ArgumentParser() parser.add_argument("--botan-dir", help="Botan base directory", required=True) - parser.add_argument("--test-target", help="The TLS side to test", choices=['client', 'server'], required=True) - parser.add_argument("--parallel", help="The number of parallel handshakes", type=int, default=1) + parser.add_argument( + "--test-target", + help="The TLS side to test", + choices=["client", "server"], + required=True, + ) + parser.add_argument( + "--parallel", help="The number of parallel handshakes", type=int, default=1 + ) args = vars(parser.parse_args(args)) diff --git a/src/scripts/ci/gh_clang_tidy_fixes_in_pr.py b/src/scripts/ci/gh_clang_tidy_fixes_in_pr.py index ad756546dc9..91d57356308 100755 --- a/src/scripts/ci/gh_clang_tidy_fixes_in_pr.py +++ b/src/scripts/ci/gh_clang_tidy_fixes_in_pr.py @@ -14,8 +14,15 @@ import yaml + class FileLocation: - def __init__(self, line : int, column : int, endline : int | None = None, endcolumn : int | None = None): + def __init__( + self, + line: int, + column: int, + endline: int | None = None, + endcolumn: int | None = None, + ): self.line = line self.column = column self.endline = endline @@ -27,24 +34,22 @@ def has_range(self): class Diagnostic: @staticmethod - def read_diagnostics_from(yaml_file : str): + def read_diagnostics_from(yaml_file: str): with open(yaml_file, encoding="utf-8") as yml: fixes = yaml.load(yml, Loader=yaml.FullLoader) if "Diagnostics" not in fixes: raise RuntimeError(f"No Diagnostics found in {yaml_file}") return [Diagnostic(diag) for diag in fixes["Diagnostics"]] - - def __map_file_path(self, file_path, base_path): # pylint: disable=unused-argument + def __map_file_path(self, file_path, base_path): # pylint: disable=unused-argument if file_path.endswith(".h"): # This only works for symlink builds, which is sufficient for CI reporting return os.path.realpath(file_path) return file_path - - def __map_file_offset(self, offset : int) -> tuple[int, int]: - """ For self.file determine the (line, column) given a byte offset """ + def __map_file_offset(self, offset: int) -> tuple[int, int]: + """For self.file determine the (line, column) given a byte offset""" with open(self.file, encoding="utf-8") as srcfile: readoffset = 0 lineoffset = 0 @@ -56,18 +61,21 @@ def __map_file_offset(self, offset : int) -> tuple[int, int]: return (lineoffset, coloffset) raise RuntimeError(f"FileOffset {offset} out of range for {self.file}") - def __map_file_location(self, msg): - """ For self.file determine the specified error range of the message """ + """For self.file determine the specified error range of the message""" location = self.__map_file_offset(msg["FileOffset"]) if "Ranges" in msg and len(msg["Ranges"]) == 1: - the_range = msg ["Ranges"][0] - if the_range["FilePath"] == msg["FilePath"] and the_range["FileOffset"] == msg["FileOffset"]: - endlocation = self.__map_file_offset(the_range["FileOffset"] + the_range["Length"]) + the_range = msg["Ranges"][0] + if ( + the_range["FilePath"] == msg["FilePath"] + and the_range["FileOffset"] == msg["FileOffset"] + ): + endlocation = self.__map_file_offset( + the_range["FileOffset"] + the_range["Length"] + ) return FileLocation(*location, *endlocation) return FileLocation(*location) - def __init__(self, yaml_diag): self.name = yaml_diag["DiagnosticName"] msg = yaml_diag["DiagnosticMessage"] @@ -77,22 +85,22 @@ def __init__(self, yaml_diag): self.location = self.__map_file_location(msg) -def render_as_github_annotations(diagnostics : list[Diagnostic]): - def map_level(level : str) -> str: +def render_as_github_annotations(diagnostics: list[Diagnostic]): + def map_level(level: str) -> str: if level == "Error": return "error" elif level == "Warning": return "warning" else: - return "notice" # fallback: likely never used + return "notice" # fallback: likely never used - def render_location(location : FileLocation) -> str: + def render_location(location: FileLocation) -> str: linemarkers = [f"line={location.line}"] colmarkers = [f"col={location.column}"] if location.has_range(): linemarkers += [f"endLine={location.endline}"] colmarkers += [f"endColumn={location.endcolumn}"] - return ','.join(linemarkers + colmarkers) + return ",".join(linemarkers + colmarkers) def render_message(msg: str) -> str: return msg.replace("\n", " - ") @@ -105,9 +113,11 @@ def render_message(msg: str) -> str: def main(): - parser = argparse.ArgumentParser(prog="ClangTidy Decoder", - description="Parses ClangTidy YAML output and emits GitHub Workflow commands") - parser.add_argument('directory') + parser = argparse.ArgumentParser( + prog="ClangTidy Decoder", + description="Parses ClangTidy YAML output and emits GitHub Workflow commands", + ) + parser.add_argument("directory") args = parser.parse_args() diagnostics = [] @@ -116,5 +126,6 @@ def main(): render_as_github_annotations(diagnostics) -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/ci/gha_linux_packages.py b/src/scripts/ci/gha_linux_packages.py index 874210c17dc..31a8e9c0d24 100755 --- a/src/scripts/ci/gha_linux_packages.py +++ b/src/scripts/ci/gha_linux_packages.py @@ -8,141 +8,154 @@ import sys + def gha_linux_packages(target, compiler): packages = [ - 'ccache', - 'libbz2-dev', - 'liblzma-dev', - 'libsqlite3-dev', + "ccache", + "libbz2-dev", + "liblzma-dev", + "libsqlite3-dev", ] - if compiler in ['gcc-14']: - packages.append('gcc-14') - - if target.startswith('valgrind'): - packages.append('valgrind') + if compiler in ["gcc-14"]: + packages.append("gcc-14") - if target in ['shared', 'coverage', 'amalgamation', 'sanitizer', 'tlsanvil', 'examples', 'clang-tidy']: - packages.append('libboost-dev') + if target.startswith("valgrind"): + packages.append("valgrind") - if target in ['clang']: - packages.append('clang') + if target in [ + "shared", + "coverage", + "amalgamation", + "sanitizer", + "tlsanvil", + "examples", + "clang-tidy", + ]: + packages.append("libboost-dev") - if target in ['cross-i386']: - packages.append('g++-multilib') - packages.append('linux-libc-dev') - packages.append('libc6-dev-i386') + if target in ["clang"]: + packages.append("clang") - if target in ['cross-win64']: - packages.append('wine-development') - packages.append('g++-mingw-w64-x86-64') + if target in ["cross-i386"]: + packages.append("g++-multilib") + packages.append("linux-libc-dev") + packages.append("libc6-dev-i386") - if target in ['cross-arm32']: - packages.append('qemu-user') - packages.append('g++-arm-linux-gnueabihf') + if target in ["cross-win64"]: + packages.append("wine-development") + packages.append("g++-mingw-w64-x86-64") - if target in ['cross-arm64', 'cross-arm64-amalgamation']: - packages.append('qemu-user') - packages.append('g++-aarch64-linux-gnu') + if target in ["cross-arm32"]: + packages.append("qemu-user") + packages.append("g++-arm-linux-gnueabihf") - if target in ['cross-ppc32']: - packages.append('qemu-user') - packages.append('g++-powerpc-linux-gnu') + if target in ["cross-arm64", "cross-arm64-amalgamation"]: + packages.append("qemu-user") + packages.append("g++-aarch64-linux-gnu") - if target in ['cross-ppc64']: - packages.append('qemu-user') - packages.append('g++-powerpc64le-linux-gnu') + if target in ["cross-ppc32"]: + packages.append("qemu-user") + packages.append("g++-powerpc-linux-gnu") - if target in ['cross-sh4']: - packages.append('qemu-user') - packages.append('g++-sh4-linux-gnu') + if target in ["cross-ppc64"]: + packages.append("qemu-user") + packages.append("g++-powerpc64le-linux-gnu") - if target in ['cross-sparc64']: - packages.append('qemu-user') - packages.append('g++-sparc64-linux-gnu') + if target in ["cross-sh4"]: + packages.append("qemu-user") + packages.append("g++-sh4-linux-gnu") - if target in ['cross-m68k']: - packages.append('qemu-user') - packages.append('g++-m68k-linux-gnu') + if target in ["cross-sparc64"]: + packages.append("qemu-user") + packages.append("g++-sparc64-linux-gnu") - if target in ['cross-riscv64']: - packages.append('qemu-user') - packages.append('g++-riscv64-linux-gnu') + if target in ["cross-m68k"]: + packages.append("qemu-user") + packages.append("g++-m68k-linux-gnu") - if target in ['cross-alpha']: - packages.append('qemu-user') - packages.append('g++-alpha-linux-gnu') + if target in ["cross-riscv64"]: + packages.append("qemu-user") + packages.append("g++-riscv64-linux-gnu") - if target in ['cross-arc']: - packages.append('qemu-user') - packages.append('g++-arc-linux-gnu') + if target in ["cross-alpha"]: + packages.append("qemu-user") + packages.append("g++-alpha-linux-gnu") - if target in ['cross-hppa64']: - packages.append('qemu-user') - packages.append('g++-hppa-linux-gnu') + if target in ["cross-arc"]: + packages.append("qemu-user") + packages.append("g++-arc-linux-gnu") - if target in ['cross-loongarch64']: - packages.append('qemu-user') - packages.append('g++-14-loongarch64-linux-gnu') + if target in ["cross-hppa64"]: + packages.append("qemu-user") + packages.append("g++-hppa-linux-gnu") - if target in ['cross-mips']: - packages.append('qemu-user') - packages.append('g++-mips-linux-gnu') + if target in ["cross-loongarch64"]: + packages.append("qemu-user") + packages.append("g++-14-loongarch64-linux-gnu") - if target in ['cross-mips64']: - packages.append('qemu-user') - packages.append('g++-mips64-linux-gnuabi64') + if target in ["cross-mips"]: + packages.append("qemu-user") + packages.append("g++-mips-linux-gnu") - if target in ['cross-s390x']: - packages.append('qemu-user') - packages.append('g++-s390x-linux-gnu') + if target in ["cross-mips64"]: + packages.append("qemu-user") + packages.append("g++-mips64-linux-gnuabi64") - if target in ['cross-arm32-baremetal']: - packages.append('gcc-arm-none-eabi') - packages.append('libstdc++-arm-none-eabi-newlib') + if target in ["cross-s390x"]: + packages.append("qemu-user") + packages.append("g++-s390x-linux-gnu") - if target in ['emscripten']: - packages.append('emscripten') + if target in ["cross-arm32-baremetal"]: + packages.append("gcc-arm-none-eabi") + packages.append("libstdc++-arm-none-eabi-newlib") - if target in ['lint']: - packages.append('pylint') - packages.append('python3-matplotlib') + if target in ["emscripten"]: + packages.append("emscripten") - if target in ['limbo']: - packages.append('python3-dateutil') + if target in ["lint"]: + packages.append("pylint") + packages.append("python3-matplotlib") - if target in ['coverage']: - packages.append('lcov') - packages.append('python3-coverage') + if target in ["limbo"]: + packages.append("python3-dateutil") - if target in ['strubbing']: - packages.append('gdb') + if target in ["coverage"]: + packages.append("lcov") + packages.append("python3-coverage") + + if target in ["strubbing"]: + packages.append("gdb") - if target in ['coverage', 'sanitizer', 'clang-tidy']: - packages.append('libtspi-dev') # TPM 1 development library [TODO(Botan4) remove this] - packages.append('libtss2-dev') # TPM 2 development library + if target in ["coverage", "sanitizer", "clang-tidy"]: + packages.append( + "libtspi-dev" + ) # TPM 1 development library [TODO(Botan4) remove this] + packages.append("libtss2-dev") # TPM 2 development library - if target in ['coverage', 'sanitizer']: - packages.append('softhsm2') + if target in ["coverage", "sanitizer"]: + packages.append("softhsm2") # Following are only available on Ubuntu 24.04 # If we wanted to test building of TPM2 on 22.04 we'd need to restrict these - packages.append('tpm2-tools') # CLI tools to interact with the TPM - packages.append('swtpm') # TPM 2.0 simulator - packages.append('swtpm-tools') # CLI tools to set up the TPM simulator - packages.append('tpm2-abrmd') # user-space resource manager for TPM 2.0 - packages.append('libtss2-tcti-tabrmd0') # TCTI (TPM Command Transmission Interface) for the user-space resource manager + packages.append("tpm2-tools") # CLI tools to interact with the TPM + packages.append("swtpm") # TPM 2.0 simulator + packages.append("swtpm-tools") # CLI tools to set up the TPM simulator + packages.append("tpm2-abrmd") # user-space resource manager for TPM 2.0 + packages.append( + "libtss2-tcti-tabrmd0" + ) # TCTI (TPM Command Transmission Interface) for the user-space resource manager - if target in ['docs']: - packages.append('doxygen') - packages.append('python3-docutils') - packages.append('python3-sphinx') + if target in ["docs"]: + packages.append("doxygen") + packages.append("python3-docutils") + packages.append("python3-sphinx") return packages -def main(args = None): +def main(args=None): if args is None: args = sys.argv @@ -157,5 +170,6 @@ def main(args = None): return 0 -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/ci_build.py b/src/scripts/ci_build.py index c8000900a09..8391803f1ac 100755 --- a/src/scripts/ci_build.py +++ b/src/scripts/ci_build.py @@ -15,9 +15,10 @@ import sys import time import tempfile -import optparse # pylint: disable=deprecated-module +import optparse # pylint: disable=deprecated-module import multiprocessing + def get_concurrency(): def_concurrency = 2 max_concurrency = 16 @@ -27,62 +28,65 @@ def get_concurrency(): except ImportError: return def_concurrency + def known_targets(): return [ - 'amalgamation', - 'codeql', - 'coverage', - 'cross-alpha', - 'cross-android-arm32', - 'cross-android-arm64', - 'cross-android-arm64-amalgamation', - 'cross-arm32', - 'cross-arm32-baremetal', - 'cross-arm64', - 'cross-arm64-amalgamation', - 'cross-hppa64', - 'cross-i386', - 'cross-ios-arm64', - 'cross-loongarch64', - 'cross-m68k', - 'cross-mips', - 'cross-mips64', - 'cross-ppc32', - 'cross-ppc64', - 'cross-riscv64', - 'cross-s390x', - 'cross-sh4', - 'cross-sparc64', - 'cross-win64', - 'docs', - 'emscripten', - 'examples', - 'format', - 'fuzzers', - 'hybrid-tls13-interop-test', - 'lint', - 'limbo', - 'minimized', - 'nist', - 'no_pcurves', - 'policy-bsi', - 'policy-fips140', - 'policy-modern', - 'sanitizer', - 'sde', - 'shared', - 'static', - 'strubbing', - 'typos', - 'valgrind', - 'valgrind-full', - 'valgrind-ct', - 'valgrind-ct-full', + "amalgamation", + "codeql", + "coverage", + "cross-alpha", + "cross-android-arm32", + "cross-android-arm64", + "cross-android-arm64-amalgamation", + "cross-arm32", + "cross-arm32-baremetal", + "cross-arm64", + "cross-arm64-amalgamation", + "cross-hppa64", + "cross-i386", + "cross-ios-arm64", + "cross-loongarch64", + "cross-m68k", + "cross-mips", + "cross-mips64", + "cross-ppc32", + "cross-ppc64", + "cross-riscv64", + "cross-s390x", + "cross-sh4", + "cross-sparc64", + "cross-win64", + "docs", + "emscripten", + "examples", + "format", + "fuzzers", + "hybrid-tls13-interop-test", + "lint", + "limbo", + "minimized", + "nist", + "no_pcurves", + "policy-bsi", + "policy-fips140", + "policy-modern", + "sanitizer", + "sde", + "shared", + "static", + "strubbing", + "typos", + "valgrind", + "valgrind-full", + "valgrind-ct", + "valgrind-ct-full", ] + def is_running_in_github_actions(): return os.environ.get("GITHUB_ACTIONS", "false") == "true" + class LoggingGroup: """ Context Manager that opportunistically uses GitHub Actions workflow commands @@ -110,79 +114,112 @@ def __exit__(self, exc_type, exc_value, exc_tb): if time_taken > 10: print("> Running '%s' took %d seconds" % (self.group_title, time_taken)) + def build_targets(target, target_os): - if target in ['shared', 'minimized', 'examples', 'limbo'] or target.startswith('policy-'): - yield 'shared' - elif target in ['static', 'fuzzers', 'cross-arm32-baremetal', 'emscripten', 'strubbing']: - yield 'static' - elif target_os in ['windows']: - yield 'shared' - elif target_os in ['ios', 'mingw']: - yield 'static' + if target in ["shared", "minimized", "examples", "limbo"] or target.startswith( + "policy-" + ): + yield "shared" + elif target in [ + "static", + "fuzzers", + "cross-arm32-baremetal", + "emscripten", + "strubbing", + ]: + yield "static" + elif target_os in ["windows"]: + yield "shared" + elif target_os in ["ios", "mingw"]: + yield "static" else: - yield 'shared' - yield 'static' + yield "shared" + yield "static" + + if target not in ["examples", "limbo"]: + yield "cli" - if target not in ['examples', 'limbo']: - yield 'cli' + if target not in ["examples", "limbo", "hybrid-tls13-interop-test", "strubbing"]: + yield "tests" - if target not in ['examples', 'limbo', 'hybrid-tls13-interop-test', 'strubbing']: - yield 'tests' + if target in ["coverage"]: + yield "bogo_shim" + if target in ["sanitizer"] and target_os not in ["windows"]: + yield "bogo_shim" + if target in ["examples", "amalgamation"]: + yield "examples" + if target in ["valgrind", "valgrind-full", "valgrind-ct", "valgrind-ct-full"]: + yield "ct_selftest" - if target in ['coverage']: - yield 'bogo_shim' - if target in ['sanitizer'] and target_os not in ['windows']: - yield 'bogo_shim' - if target in ['examples', 'amalgamation']: - yield 'examples' - if target in ['valgrind', 'valgrind-full', 'valgrind-ct', 'valgrind-ct-full']: - yield 'ct_selftest' def make_targets(target, target_os): # The result of build_targets() is for ./configure.py --build-targets='...'. # make_targets() go into `make/ninja ...` and they are mostly equal. Except # for 'libs' which is an umbrella target for 'static' and 'shared'. - tgts = [tgt if tgt not in ['static', 'shared'] else 'libs' for tgt in build_targets(target, target_os)] - if target in ['coverage', 'fuzzers']: + tgts = [ + tgt if tgt not in ["static", "shared"] else "libs" + for tgt in build_targets(target, target_os) + ] + if target in ["coverage", "fuzzers"]: # These are special targets not found in ./configure.py --build-targets= - tgts += ['fuzzers', 'fuzzer_corpus_zip'] + tgts += ["fuzzers", "fuzzer_corpus_zip"] return list(set(tgts)) -def determine_flags(target, target_os, target_cpu, target_cc, cc_bin, ccache, - root_dir, build_dir, test_results_dir, pkcs11_lib, use_gdb, - disable_werror, extra_cxxflags, custom_optimization_flags, disabled_tests): +def determine_flags( + target, + target_os, + target_cpu, + target_cc, + cc_bin, + ccache, + root_dir, + build_dir, + test_results_dir, + pkcs11_lib, + use_gdb, + disable_werror, + extra_cxxflags, + custom_optimization_flags, + disabled_tests, +): """ Return the configure.py flags as well as make/test running prefixes """ - is_cross_target = target.startswith('cross-') + is_cross_target = target.startswith("cross-") - if target_os not in ['linux', 'osx', 'windows', 'freebsd']: - print('Error unknown OS %s' % (target_os)) + if target_os not in ["linux", "osx", "windows", "freebsd"]: + print("Error unknown OS %s" % (target_os)) return (None, None, None) if is_cross_target: - if target_os == 'osx': - target_os = 'ios' - elif target == 'cross-win64': - target_os = 'mingw' - elif target in ['cross-android-arm32', 'cross-android-arm64', 'cross-android-arm64-amalgamation']: - target_os = 'android' - - if target_os == 'windows' and target_cc == 'gcc': - target_os = 'mingw' - - if target == 'cross-arm32-baremetal': - target_os = 'none' - - if target == 'emscripten': - target_os = 'emscripten' + if target_os == "osx": + target_os = "ios" + elif target == "cross-win64": + target_os = "mingw" + elif target in [ + "cross-android-arm32", + "cross-android-arm64", + "cross-android-arm64-amalgamation", + ]: + target_os = "android" + + if target_os == "windows" and target_cc == "gcc": + target_os = "mingw" + + if target == "cross-arm32-baremetal": + target_os = "none" + + if target == "emscripten": + target_os = "emscripten" make_prefix = [] test_prefix = [] - test_cmd = [os.path.join(build_dir, 'botan-test'), - '--data-dir=%s' % os.path.join(root_dir, 'src', 'tests', 'data'), - '--run-memory-intensive-tests'] + test_cmd = [ + os.path.join(build_dir, "botan-test"), + "--data-dir=%s" % os.path.join(root_dir, "src", "tests", "data"), + "--run-memory-intensive-tests", + ] # generate JUnit test report if test_results_dir: @@ -190,297 +227,360 @@ def determine_flags(target, target_os, target_cpu, target_cc, cc_bin, ccache, raise Exception("Test results directory does not exist") def sanitize_kv(some_string): - return some_string.replace(':', '').replace(',', '') + return some_string.replace(":", "").replace(",", "") report_props = {"ci_target": target, "os": target_os} - test_cmd += ['--test-results-dir=%s' % test_results_dir] - test_cmd += ['--report-properties=%s' % - ','.join(['%s:%s' % (sanitize_kv(k), sanitize_kv(v)) for k, v in report_props.items()])] - - - flags = ['--cc=%s' % (target_cc), - '--os=%s' % (target_os), - '--build-targets=%s' % ','.join(build_targets(target, target_os)), - '--with-build-dir=%s' % build_dir, - '--link-method=symlink', - '--enable-experimental-features'] + test_cmd += ["--test-results-dir=%s" % test_results_dir] + test_cmd += [ + "--report-properties=%s" + % ",".join( + [ + "%s:%s" % (sanitize_kv(k), sanitize_kv(v)) + for k, v in report_props.items() + ] + ) + ] + + flags = [ + "--cc=%s" % (target_cc), + "--os=%s" % (target_os), + "--build-targets=%s" % ",".join(build_targets(target, target_os)), + "--with-build-dir=%s" % build_dir, + "--link-method=symlink", + "--enable-experimental-features", + ] - if target in ['shared', 'static']: - install_prefix = tempfile.mkdtemp(prefix='botan-install-') - flags += ['--prefix=%s' % (install_prefix)] + if target in ["shared", "static"]: + install_prefix = tempfile.mkdtemp(prefix="botan-install-") + flags += ["--prefix=%s" % (install_prefix)] if ccache is not None: - flags += ['--compiler-cache=%s' % (ccache)] + flags += ["--compiler-cache=%s" % (ccache)] if not disable_werror: - flags += ['--werror-mode'] + flags += ["--werror-mode"] if target_cpu is not None: - flags += ['--cpu=%s' % (target_cpu)] + flags += ["--cpu=%s" % (target_cpu)] if custom_optimization_flags and any(flag for flag in custom_optimization_flags): - flags += ['--no-optimizations'] + flags += ["--no-optimizations"] for flag in custom_optimization_flags: - flags += ['--extra-cxxflags=%s' % (flag)] + flags += ["--extra-cxxflags=%s" % (flag)] for flag in extra_cxxflags: - flags += ['--extra-cxxflags=%s' % (flag)] + flags += ["--extra-cxxflags=%s" % (flag)] - if target_os == 'windows': + if target_os == "windows": # Workaround for https://github.com/actions/runner-images/issues/10004 - flags += ['--extra-cxxflags=/D_DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR'] + flags += ["--extra-cxxflags=/D_DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR"] - if target in ['minimized']: - flags += ['--minimized-build', '--enable-modules=system_rng,sha2*,aes'] + if target in ["minimized"]: + flags += ["--minimized-build", "--enable-modules=system_rng,sha2*,aes"] - if target in ['no_pcurves']: - flags += ['--disable-modules=pcurves_impl'] + if target in ["no_pcurves"]: + flags += ["--disable-modules=pcurves_impl"] - if target in ['amalgamation', 'cross-arm64-amalgamation', 'cross-android-arm64-amalgamation']: - flags += ['--amalgamation'] + if target in [ + "amalgamation", + "cross-arm64-amalgamation", + "cross-android-arm64-amalgamation", + ]: + flags += ["--amalgamation"] - if target.startswith('policy-'): + if target.startswith("policy-"): # tls is optional for bsi/fips140 but add it so verify tests work with these minimized configs - flags += ['--module-policy=%s' % (target.replace('policy-', '')), '--enable-modules=tls12,tls13', '--disable-deprecated-features'] - - if target in ['docs']: - flags += ['--with-doxygen', '--with-sphinx', '--with-rst2man'] + flags += [ + "--module-policy=%s" % (target.replace("policy-", "")), + "--enable-modules=tls12,tls13", + "--disable-deprecated-features", + ] + + if target in ["docs"]: + flags += ["--with-doxygen", "--with-sphinx", "--with-rst2man"] else: - flags += ['--without-doc'] + flags += ["--without-doc"] - if target in ['docs', 'codeql', 'hybrid-tls13-interop-test', 'limbo']: + if target in ["docs", "codeql", "hybrid-tls13-interop-test", "limbo"]: test_cmd = None - if target in ['codeql']: - flags += ['--no-optimizations'] + if target in ["codeql"]: + flags += ["--no-optimizations"] - if target == 'cross-win64': + if target == "cross-win64": # this test compiles under MinGW but fails when run under Wine - disabled_tests.append('certstor_system') + disabled_tests.append("certstor_system") - if target_os == 'mingw': + if target_os == "mingw": # make sure to link against static versions of libstdc++, libgcc* and winpthread - flags += ['--ldflags=-static'] + flags += ["--ldflags=-static"] - if target == 'coverage': - flags += ['--with-coverage-info'] + if target == "coverage": + flags += ["--with-coverage-info"] - if target in ['coverage', 'valgrind', 'valgrind-full', 'valgrind-ct', 'valgrind-ct-full']: - flags += ['--with-debug-info'] + if target in [ + "coverage", + "valgrind", + "valgrind-full", + "valgrind-ct", + "valgrind-ct-full", + ]: + flags += ["--with-debug-info"] - if target in ['strubbing']: + if target in ["strubbing"]: # Stack scrubbing tests are based on scripted gdb runs and won't work on # an optimized build unfortunately. - flags += ['--debug-mode'] + flags += ["--debug-mode"] test_cmd = None - if target in ['coverage', 'sanitizer', 'fuzzers']: - flags += ['--unsafe-terminate-on-asserts', '--enable-modules=tls_null'] + if target in ["coverage", "sanitizer", "fuzzers"]: + flags += ["--unsafe-terminate-on-asserts", "--enable-modules=tls_null"] - if target in ['sde']: - test_prefix = ['sde', '-future', '--'] + if target in ["sde"]: + test_prefix = ["sde", "-future", "--"] - if target in ['valgrind', 'valgrind-full', 'valgrind-ct', 'valgrind-ct-full']: - flags += ['--with-valgrind'] + if target in ["valgrind", "valgrind-full", "valgrind-ct", "valgrind-ct-full"]: + flags += ["--with-valgrind"] # valgrind is run via a script setup later test_cmd = None - if target == 'examples': - flags += ['--with-boost'] + if target == "examples": + flags += ["--with-boost"] test_cmd = None - if target == 'fuzzers': - flags += ['--unsafe-fuzzer-mode'] + if target == "fuzzers": + flags += ["--unsafe-fuzzer-mode"] - if target in ['fuzzers', 'coverage']: - flags += ['--build-fuzzers=test'] + if target in ["fuzzers", "coverage"]: + flags += ["--build-fuzzers=test"] - if target in ['fuzzers', 'sanitizer']: - flags += ['--with-debug-asserts'] + if target in ["fuzzers", "sanitizer"]: + flags += ["--with-debug-asserts"] - if target_cc in ['clang', 'gcc', 'xcode']: - flags += ['--enable-sanitizers=address,undefined,iterator'] + if target_cc in ["clang", "gcc", "xcode"]: + flags += ["--enable-sanitizers=address,undefined,iterator"] else: - flags += ['--enable-sanitizers=address,iterator'] - - if target in ['valgrind', 'valgrind-full', 'valgrind-ct', 'valgrind-ct-full', 'sanitizer', 'fuzzers']: - flags += ['--disable-modules=locking_allocator'] - - if target == 'emscripten': - flags += ['--cpu=wasm'] + flags += ["--enable-sanitizers=address,iterator"] + + if target in [ + "valgrind", + "valgrind-full", + "valgrind-ct", + "valgrind-ct-full", + "sanitizer", + "fuzzers", + ]: + flags += ["--disable-modules=locking_allocator"] + + if target == "emscripten": + flags += ["--cpu=wasm"] # need to find a way to run the wasm-compiled tests w/o a browser test_cmd = None - if target in ['sanitizer', 'strubbing'] and target_cc in ['gcc']: + if target in ["sanitizer", "strubbing"] and target_cc in ["gcc"]: # Stack scrubbing is supported on GCC 14 and newer, only. This is newer # than the current default compiler on GHA's Linux image (ubuntu 24.04). # The CI setup has to ensure that we are configured to use a recent # compiler for these targets, otherwise `./configure.py` will fail. - flags += ['--enable-stack-scrubbing'] + flags += ["--enable-stack-scrubbing"] if is_cross_target: - if target_os == 'ios': - make_prefix = ['xcrun', '--sdk', 'iphoneos'] + if target_os == "ios": + make_prefix = ["xcrun", "--sdk", "iphoneos"] test_cmd = None - if target == 'cross-ios-arm64': - flags += ['--cpu=arm64', '--cc-abi-flags=-arch arm64 -stdlib=libc++'] + if target == "cross-ios-arm64": + flags += ["--cpu=arm64", "--cc-abi-flags=-arch arm64 -stdlib=libc++"] else: raise Exception("Unknown cross target '%s' for iOS" % (target)) - elif target_os == 'android': - - ndk = os.getenv('ANDROID_NDK') + elif target_os == "android": + ndk = os.getenv("ANDROID_NDK") if ndk is None: - raise Exception('Android CI build requires ANDROID_NDK env variable be set') + raise Exception( + "Android CI build requires ANDROID_NDK env variable be set" + ) - api_lvl = int(os.getenv('ANDROID_API_LEVEL', '0')) + api_lvl = int(os.getenv("ANDROID_API_LEVEL", "0")) if api_lvl == 0: # If not set arbitrarily choose API 21 (Android 5.0) for ARMv7 and 31 (Android 12) for AArch64 - api_lvl = 21 if target == 'cross-android-arm32' else 31 + api_lvl = 21 if target == "cross-android-arm32" else 31 - toolchain_dir = os.path.join(ndk, 'toolchains/llvm/prebuilt/linux-x86_64/bin') + toolchain_dir = os.path.join( + ndk, "toolchains/llvm/prebuilt/linux-x86_64/bin" + ) test_cmd = None - if target == 'cross-android-arm32': - cc_bin = os.path.join(toolchain_dir, 'armv7a-linux-androideabi%d-clang++' % (api_lvl)) - flags += ['--cpu=armv7', - '--ar-command=%s' % (os.path.join(toolchain_dir, 'llvm-ar'))] - elif target in ['cross-android-arm64', 'cross-android-arm64-amalgamation']: - cc_bin = os.path.join(toolchain_dir, 'aarch64-linux-android%d-clang++' % (api_lvl)) - flags += ['--cpu=arm64', - '--ar-command=%s' % (os.path.join(toolchain_dir, 'llvm-ar'))] + if target == "cross-android-arm32": + cc_bin = os.path.join( + toolchain_dir, "armv7a-linux-androideabi%d-clang++" % (api_lvl) + ) + flags += [ + "--cpu=armv7", + "--ar-command=%s" % (os.path.join(toolchain_dir, "llvm-ar")), + ] + elif target in ["cross-android-arm64", "cross-android-arm64-amalgamation"]: + cc_bin = os.path.join( + toolchain_dir, "aarch64-linux-android%d-clang++" % (api_lvl) + ) + flags += [ + "--cpu=arm64", + "--ar-command=%s" % (os.path.join(toolchain_dir, "llvm-ar")), + ] if api_lvl < 18: - flags += ['--without-os-features=getauxval'] + flags += ["--without-os-features=getauxval"] if api_lvl >= 28: - flags += ['--with-os-features=getentropy'] + flags += ["--with-os-features=getentropy"] - elif target == 'cross-i386': - flags += ['--cpu=x86_32'] + elif target == "cross-i386": + flags += ["--cpu=x86_32"] - elif target == 'cross-win64': + elif target == "cross-win64": # MinGW in 16.04 is lacking std::mutex for unknown reason - cc_bin = 'x86_64-w64-mingw32-g++' - flags += ['--cpu=x86_64', '--cc-abi-flags=-static', - '--ar-command=x86_64-w64-mingw32-ar', '--without-os-feature=threads'] - test_cmd = [os.path.join(root_dir, 'botan-test.exe')] + test_cmd[1:] - test_prefix = ['wine'] + cc_bin = "x86_64-w64-mingw32-g++" + flags += [ + "--cpu=x86_64", + "--cc-abi-flags=-static", + "--ar-command=x86_64-w64-mingw32-ar", + "--without-os-feature=threads", + ] + test_cmd = [os.path.join(root_dir, "botan-test.exe")] + test_cmd[1:] + test_prefix = ["wine"] else: - if target == 'cross-arm32': - flags += ['--cpu=armv7', '--extra-cxxflags=-D_FILE_OFFSET_BITS=64'] - cc_bin = 'arm-linux-gnueabihf-g++' - test_prefix = ['qemu-arm', '-L', '/usr/arm-linux-gnueabihf/'] - elif target in ['cross-arm64', 'cross-arm64-amalgamation']: - flags += ['--cpu=aarch64'] - cc_bin = 'aarch64-linux-gnu-g++' - test_prefix = ['qemu-aarch64', '-L', '/usr/aarch64-linux-gnu/'] - elif target == 'cross-alpha': - flags += ['--cpu=alpha'] - cc_bin = 'alpha-linux-gnu-g++' - test_prefix = ['qemu-alpha', '-L', '/usr/alpha-linux-gnu/'] - flags += ['--without-stack-protector'] # not supported - elif target == 'cross-sh4': - flags += ['--cpu=sh4'] - cc_bin = 'sh4-linux-gnu-g++' - test_prefix = ['qemu-sh4', '-L', '/usr/sh4-linux-gnu/'] - elif target == 'cross-m68k': - flags += ['--cpu=m68k'] - cc_bin = 'm68k-linux-gnu-g++' - test_prefix = ['qemu-m68k', '-L', '/usr/m68k-linux-gnu/'] - elif target == 'cross-hppa64': - flags += ['--cpu=hppa'] - cc_bin = 'hppa-linux-gnu-g++' - test_prefix = ['qemu-hppa', '-L', '/usr/hppa-linux-gnu/'] - elif target == 'cross-sparc64': - flags += ['--cpu=sparc64'] - cc_bin = 'sparc64-linux-gnu-g++' - test_prefix = ['qemu-sparc64', '-L', '/usr/sparc64-linux-gnu/'] - elif target == 'cross-ppc32': - flags += ['--cpu=ppc32'] - cc_bin = 'powerpc-linux-gnu-g++' - test_prefix = ['qemu-ppc', '-L', '/usr/powerpc-linux-gnu/'] - test_cmd = None # qemu crashes ... - elif target == 'cross-ppc64': - flags += ['--cpu=ppc64'] - cc_bin = 'powerpc64le-linux-gnu-g++' - test_prefix = ['qemu-ppc64le', '-cpu', 'power10', '-L', '/usr/powerpc64le-linux-gnu/'] - elif target == 'cross-riscv64': - flags += ['--cpu=riscv64'] - cc_bin = 'riscv64-linux-gnu-g++' - test_prefix = ['qemu-riscv64', '-L', '/usr/riscv64-linux-gnu/'] - elif target == 'cross-s390x': - flags += ['--cpu=s390x'] - cc_bin = 's390x-linux-gnu-g++' - test_prefix = ['qemu-s390x', '-L', '/usr/s390x-linux-gnu/'] - elif target == 'cross-loongarch64': - flags += ['--cpu=loongarch64'] - cc_bin = 'loongarch64-linux-gnu-g++-14' - test_prefix = ['qemu-loongarch64', '-L', '/usr/loongarch64-linux-gnu/'] - elif target == 'cross-mips': - flags += ['--cpu=mips32'] - cc_bin = 'mips-linux-gnu-g++' - test_prefix = ['qemu-mips', '-L', '/usr/mips-linux-gnu/'] - elif target == 'cross-mips64': - flags += ['--cpu=mips64'] - cc_bin = 'mips64-linux-gnuabi64-g++' - test_prefix = ['qemu-mips64', '-L', '/usr/mips64-linux-gnuabi64/'] - elif target in ['cross-arm32-baremetal']: - flags += ['--cpu=arm32', '--disable-neon', '--without-stack-protector', '--ldflags=-specs=nosys.specs'] - cc_bin = 'arm-none-eabi-c++' + if target == "cross-arm32": + flags += ["--cpu=armv7", "--extra-cxxflags=-D_FILE_OFFSET_BITS=64"] + cc_bin = "arm-linux-gnueabihf-g++" + test_prefix = ["qemu-arm", "-L", "/usr/arm-linux-gnueabihf/"] + elif target in ["cross-arm64", "cross-arm64-amalgamation"]: + flags += ["--cpu=aarch64"] + cc_bin = "aarch64-linux-gnu-g++" + test_prefix = ["qemu-aarch64", "-L", "/usr/aarch64-linux-gnu/"] + elif target == "cross-alpha": + flags += ["--cpu=alpha"] + cc_bin = "alpha-linux-gnu-g++" + test_prefix = ["qemu-alpha", "-L", "/usr/alpha-linux-gnu/"] + flags += ["--without-stack-protector"] # not supported + elif target == "cross-sh4": + flags += ["--cpu=sh4"] + cc_bin = "sh4-linux-gnu-g++" + test_prefix = ["qemu-sh4", "-L", "/usr/sh4-linux-gnu/"] + elif target == "cross-m68k": + flags += ["--cpu=m68k"] + cc_bin = "m68k-linux-gnu-g++" + test_prefix = ["qemu-m68k", "-L", "/usr/m68k-linux-gnu/"] + elif target == "cross-hppa64": + flags += ["--cpu=hppa"] + cc_bin = "hppa-linux-gnu-g++" + test_prefix = ["qemu-hppa", "-L", "/usr/hppa-linux-gnu/"] + elif target == "cross-sparc64": + flags += ["--cpu=sparc64"] + cc_bin = "sparc64-linux-gnu-g++" + test_prefix = ["qemu-sparc64", "-L", "/usr/sparc64-linux-gnu/"] + elif target == "cross-ppc32": + flags += ["--cpu=ppc32"] + cc_bin = "powerpc-linux-gnu-g++" + test_prefix = ["qemu-ppc", "-L", "/usr/powerpc-linux-gnu/"] + test_cmd = None # qemu crashes ... + elif target == "cross-ppc64": + flags += ["--cpu=ppc64"] + cc_bin = "powerpc64le-linux-gnu-g++" + test_prefix = [ + "qemu-ppc64le", + "-cpu", + "power10", + "-L", + "/usr/powerpc64le-linux-gnu/", + ] + elif target == "cross-riscv64": + flags += ["--cpu=riscv64"] + cc_bin = "riscv64-linux-gnu-g++" + test_prefix = ["qemu-riscv64", "-L", "/usr/riscv64-linux-gnu/"] + elif target == "cross-s390x": + flags += ["--cpu=s390x"] + cc_bin = "s390x-linux-gnu-g++" + test_prefix = ["qemu-s390x", "-L", "/usr/s390x-linux-gnu/"] + elif target == "cross-loongarch64": + flags += ["--cpu=loongarch64"] + cc_bin = "loongarch64-linux-gnu-g++-14" + test_prefix = ["qemu-loongarch64", "-L", "/usr/loongarch64-linux-gnu/"] + elif target == "cross-mips": + flags += ["--cpu=mips32"] + cc_bin = "mips-linux-gnu-g++" + test_prefix = ["qemu-mips", "-L", "/usr/mips-linux-gnu/"] + elif target == "cross-mips64": + flags += ["--cpu=mips64"] + cc_bin = "mips64-linux-gnuabi64-g++" + test_prefix = ["qemu-mips64", "-L", "/usr/mips64-linux-gnuabi64/"] + elif target in ["cross-arm32-baremetal"]: + flags += [ + "--cpu=arm32", + "--disable-neon", + "--without-stack-protector", + "--ldflags=-specs=nosys.specs", + ] + cc_bin = "arm-none-eabi-c++" test_cmd = None else: raise Exception("Unknown cross target '%s' for Linux" % (target)) else: # Flags specific to native targets - if target_os in ['osx', 'linux']: - flags += ['--with-bzip2', '--with-sqlite', '--with-zlib'] + if target_os in ["osx", "linux"]: + flags += ["--with-bzip2", "--with-sqlite", "--with-zlib"] - if target_os in ['osx', 'ios']: - flags += ['--with-commoncrypto'] + if target_os in ["osx", "ios"]: + flags += ["--with-commoncrypto"] def add_boost_support(target, target_os): - if target in ['coverage', 'amalgamation']: + if target in ["coverage", "amalgamation"]: return True - if target == 'sanitizer' and target_os == 'linux': + if target == "sanitizer" and target_os == "linux": return True - if target == 'shared' and target_os != 'windows': + if target == "shared" and target_os != "windows": return True return False if add_boost_support(target, target_os): - flags += ['--with-boost'] - if target_cc in ['clang', 'xcode']: + flags += ["--with-boost"] + if target_cc in ["clang", "xcode"]: # make sure clang ignores warnings in boost headers flags += ["--extra-cxxflags=--system-header-prefix=boost/"] - if 'BOOST_INCLUDEDIR' in os.environ: + if "BOOST_INCLUDEDIR" in os.environ: # ./configure.py needs boost's location on some platforms # BOOST_INCLUDEDIR is set by the setup_gh_actions.* script - flags += ['--with-external-includedir', os.environ.get('BOOST_INCLUDEDIR')] + flags += [ + "--with-external-includedir", + os.environ.get("BOOST_INCLUDEDIR"), + ] - if target_os == 'mingw': + if target_os == "mingw": # apparently mingw needs this legacy socket library version for reasons # as per: https://stackoverflow.com/questions/38770895/how-to-fix-undefined-reference-to-getacceptexsockaddrs-boost-asio-in-clion#comment105791579_38771260 - flags += ['--ldflags=-static -lwsock32'] + flags += ["--ldflags=-static -lwsock32"] - if target_os == 'linux': - flags += ['--with-lzma'] + if target_os == "linux": + flags += ["--with-lzma"] - if is_running_in_github_actions() and 'BOTAN_TPM2_ENABLED' in os.environ: - flags += ['--with-tpm2'] + if is_running_in_github_actions() and "BOTAN_TPM2_ENABLED" in os.environ: + flags += ["--with-tpm2"] - if os.environ.get('BOTAN_TPM2_ENABLED') == 'test': + if os.environ.get("BOTAN_TPM2_ENABLED") == "test": # run the TPM2 tests - test_cmd += ["--tpm2-tcti-name=%s" % os.getenv('BOTAN_TPM2_TCTI_NAME'), - "--tpm2-tcti-conf=%s" % os.getenv('BOTAN_TPM2_TCTI_CONF'), - "--tpm2-persistent-rsa-handle=%s" % os.getenv('BOTAN_TPM2_PERSISTENT_RSA_KEY_HANDLE'), - "--tpm2-persistent-ecc-handle=%s" % os.getenv('BOTAN_TPM2_PERSISTENT_ECC_KEY_HANDLE'), - "--tpm2-persistent-auth-value=%s" % os.getenv('BOTAN_TPM2_PERSISTENT_KEY_AUTH_VALUE')] - elif os.environ.get('BOTAN_TPM2_ENABLED') == 'build': + test_cmd += [ + "--tpm2-tcti-name=%s" % os.getenv("BOTAN_TPM2_TCTI_NAME"), + "--tpm2-tcti-conf=%s" % os.getenv("BOTAN_TPM2_TCTI_CONF"), + "--tpm2-persistent-rsa-handle=%s" + % os.getenv("BOTAN_TPM2_PERSISTENT_RSA_KEY_HANDLE"), + "--tpm2-persistent-ecc-handle=%s" + % os.getenv("BOTAN_TPM2_PERSISTENT_ECC_KEY_HANDLE"), + "--tpm2-persistent-auth-value=%s" + % os.getenv("BOTAN_TPM2_PERSISTENT_KEY_AUTH_VALUE"), + ] + elif os.environ.get("BOTAN_TPM2_ENABLED") == "build": # build the TPM2 module but don't run the tests # TCTI name 'disabled' is a special value that disables the TPM2 tests # @@ -490,31 +590,31 @@ def add_boost_support(target, target_os): test_cmd += ["--tpm2-tcti-name=disabled"] if is_running_in_github_actions(): - if 'BOTAN_BUILD_WITH_JITTERENTROPY' in os.environ: - flags += ['--enable-modules=jitter_rng'] - if 'BOTAN_BUILD_WITH_ESDM' in os.environ: - flags += ['--with-esdm_rng'] - - if target in ['coverage']: - flags += ['--with-tpm'] - test_cmd += ['--run-online-tests'] + if "BOTAN_BUILD_WITH_JITTERENTROPY" in os.environ: + flags += ["--enable-modules=jitter_rng"] + if "BOTAN_BUILD_WITH_ESDM" in os.environ: + flags += ["--with-esdm_rng"] + + if target in ["coverage"]: + flags += ["--with-tpm"] + test_cmd += ["--run-online-tests"] if pkcs11_lib and os.access(pkcs11_lib, os.R_OK): - test_cmd += ['--pkcs11-lib=%s' % (pkcs11_lib)] + test_cmd += ["--pkcs11-lib=%s" % (pkcs11_lib)] - if target in ['coverage', 'sanitizer']: - test_cmd += ['--run-long-tests'] + if target in ["coverage", "sanitizer"]: + test_cmd += ["--run-long-tests"] - if target_os == 'windows' and target == 'sanitizer': + if target_os == "windows" and target == "sanitizer": # GitHub Actions worker intermittently ran out of memory when # asked to allocate multi-gigabyte buffers under MSVC's ASan. - test_cmd.remove('--run-memory-intensive-tests') + test_cmd.remove("--run-memory-intensive-tests") # MSVC sanitizer produces very slow code causing some of the # slower tests to take as long as 5 minutes - test_cmd.remove('--run-long-tests') + test_cmd.remove("--run-long-tests") - if os.getenv('CXX') is None: - flags += ['--cc-bin=%s' % (cc_bin)] + if os.getenv("CXX") is None: + flags += ["--cc-bin=%s" % (cc_bin)] if test_cmd is None: run_test_command = None @@ -524,74 +624,92 @@ def add_boost_support(target, target_os): # render 'disabled_tests' array into test_cmd if disabled_tests: - test_cmd += ['--skip-tests=%s' % (','.join(disabled_tests))] + test_cmd += ["--skip-tests=%s" % (",".join(disabled_tests))] if use_gdb: (cmd, args) = test_cmd[0], test_cmd[1:] - run_test_command = test_prefix + ['gdb', cmd, - '-ex', 'run %s' % (' '.join(args)), - '-ex', 'bt', - '-ex', 'quit'] + run_test_command = test_prefix + [ + "gdb", + cmd, + "-ex", + "run %s" % (" ".join(args)), + "-ex", + "bt", + "-ex", + "quit", + ] else: run_test_command = test_prefix + test_cmd return flags, run_test_command, make_prefix + def run_cmd(cmd, root_dir, build_dir): """ Execute a command, die if it failed """ - with LoggingGroup(' '.join(cmd)): + with LoggingGroup(" ".join(cmd)): cmd = [os.path.expandvars(elem) for elem in cmd] sub_env = os.environ.copy() - sub_env['LD_LIBRARY_PATH'] = os.path.abspath(build_dir) - sub_env['DYLD_LIBRARY_PATH'] = os.path.abspath(build_dir) - sub_env['PYTHONPATH'] = os.path.abspath(os.path.join(root_dir, 'src/python')) + sub_env["LD_LIBRARY_PATH"] = os.path.abspath(build_dir) + sub_env["DYLD_LIBRARY_PATH"] = os.path.abspath(build_dir) + sub_env["PYTHONPATH"] = os.path.abspath(os.path.join(root_dir, "src/python")) cwd = None redirect_stdout_fd = None redirect_stdout_fsname = None - if len(cmd) >= 3 and cmd[-2] == '>': + if len(cmd) >= 3 and cmd[-2] == ">": redirect_stdout_fsname = cmd[-1] - redirect_stdout_fd = open(redirect_stdout_fsname, 'w', encoding='utf8') + redirect_stdout_fd = open(redirect_stdout_fsname, "w", encoding="utf8") cmd = cmd[:-2] - if len(cmd) > 1 and cmd[0].startswith('indir:'): + if len(cmd) > 1 and cmd[0].startswith("indir:"): cwd = cmd[0][6:] cmd = cmd[1:] - while len(cmd) > 1 and cmd[0].startswith('env:') and cmd[0].find('=') > 0: - env_key, env_val = cmd[0][4:].split('=') + while len(cmd) > 1 and cmd[0].startswith("env:") and cmd[0].find("=") > 0: + env_key, env_val = cmd[0][4:].split("=") sub_env[env_key] = env_val cmd = cmd[1:] - proc = subprocess.Popen(cmd, cwd=cwd, close_fds=True, env=sub_env, stdout=redirect_stdout_fd) + proc = subprocess.Popen( + cmd, cwd=cwd, close_fds=True, env=sub_env, stdout=redirect_stdout_fd + ) proc.communicate() if proc.returncode != 0: - print("Command '%s' failed with error code %d" % (' '.join(cmd), proc.returncode)) + print( + "Command '%s' failed with error code %d" + % (" ".join(cmd), proc.returncode) + ) if redirect_stdout_fd is not None: redirect_stdout_fd.close() - last_lines = open(redirect_stdout_fsname, encoding='utf8').readlines()[-100:] - print("%s", ''.join(last_lines)) + last_lines = open(redirect_stdout_fsname, encoding="utf8").readlines()[ + -100: + ] + print("%s", "".join(last_lines)) - if cmd[0] not in ['lcov', 'codecov']: + if cmd[0] not in ["lcov", "codecov"]: sys.exit(proc.returncode) + def default_os(): platform_os = platform.system().lower() - if platform_os == 'darwin': - return 'osx' + if platform_os == "darwin": + return "osx" return platform_os + def default_cc(): platform_os = platform.system().lower() - return 'msvc' if platform_os == 'windows' else 'gcc' + return "msvc" if platform_os == "windows" else "gcc" + def default_make_tool(): platform_os = platform.system().lower() - return 'nmake' if platform_os == 'windows' else 'make' + return "nmake" if platform_os == "windows" else "make" + def parse_args(args): """ @@ -599,96 +717,160 @@ def parse_args(args): """ parser = optparse.OptionParser() - parser.add_option('--os', default=default_os(), - help='Set the target os (default %default)') - parser.add_option('--cpu', default=None, - help='Specify a target CPU platform') - parser.add_option('--cc', default=default_cc(), - help='Set the target compiler type (default %default)') - parser.add_option('--cc-bin', default=None, - help='Set path to compiler') - parser.add_option('--root-dir', metavar='D', default='.', - help='Set directory to execute from (default %default)') - parser.add_option('--build-dir', metavar='D', default='.', - help='Set directory to place build artifacts into (default %default)') - parser.add_option('--boringssl-dir', metavar='D', default='boringssl', - help='Set directory of BoringSSL checkout to use for BoGo tests') - - parser.add_option('--make-tool', metavar='TOOL', default=default_make_tool(), - help='Specify tool to run to build source (default %default)') - - parser.add_option('--extra-cxxflags', metavar='FLAGS', default=[], action='append', - help='Specify extra build flags') - parser.add_option('--custom-optimization-flags', metavar='FLAGS', default=[], action='append', - help='Specify custom optimization flags (disables all default optimizations)') - - parser.add_option('--disabled-tests', metavar='DISABLED_TESTS', default=[], action='append', - help='Comma separated list of tests that should not be run') - - parser.add_option('--dry-run', action='store_true', default=False, - help='Just show commands to be executed') - parser.add_option('--build-jobs', metavar='J', default=get_concurrency(), - help='Set number of jobs to run in parallel (default %default)') - - parser.add_option('--compiler-cache', default=None, metavar='CC', - help='Set a compiler cache to use (ccache, sccache, none)') - - parser.add_option('--pkcs11-lib', default=os.getenv('PKCS11_LIB'), metavar='LIB', - help='Set PKCS11 lib to use for testing') - - parser.add_option('--disable-werror', action='store_true', default=False, - help='Allow warnings to compile') - - parser.add_option('--run-under-gdb', dest='use_gdb', action='store_true', default=False, - help='Run test suite under gdb and capture backtrace') - - parser.add_option('--test-results-dir', default=None, - help='Directory to store JUnit XML test reports') + parser.add_option( + "--os", default=default_os(), help="Set the target os (default %default)" + ) + parser.add_option("--cpu", default=None, help="Specify a target CPU platform") + parser.add_option( + "--cc", + default=default_cc(), + help="Set the target compiler type (default %default)", + ) + parser.add_option("--cc-bin", default=None, help="Set path to compiler") + parser.add_option( + "--root-dir", + metavar="D", + default=".", + help="Set directory to execute from (default %default)", + ) + parser.add_option( + "--build-dir", + metavar="D", + default=".", + help="Set directory to place build artifacts into (default %default)", + ) + parser.add_option( + "--boringssl-dir", + metavar="D", + default="boringssl", + help="Set directory of BoringSSL checkout to use for BoGo tests", + ) + + parser.add_option( + "--make-tool", + metavar="TOOL", + default=default_make_tool(), + help="Specify tool to run to build source (default %default)", + ) + + parser.add_option( + "--extra-cxxflags", + metavar="FLAGS", + default=[], + action="append", + help="Specify extra build flags", + ) + parser.add_option( + "--custom-optimization-flags", + metavar="FLAGS", + default=[], + action="append", + help="Specify custom optimization flags (disables all default optimizations)", + ) + + parser.add_option( + "--disabled-tests", + metavar="DISABLED_TESTS", + default=[], + action="append", + help="Comma separated list of tests that should not be run", + ) + + parser.add_option( + "--dry-run", + action="store_true", + default=False, + help="Just show commands to be executed", + ) + parser.add_option( + "--build-jobs", + metavar="J", + default=get_concurrency(), + help="Set number of jobs to run in parallel (default %default)", + ) + + parser.add_option( + "--compiler-cache", + default=None, + metavar="CC", + help="Set a compiler cache to use (ccache, sccache, none)", + ) + + parser.add_option( + "--pkcs11-lib", + default=os.getenv("PKCS11_LIB"), + metavar="LIB", + help="Set PKCS11 lib to use for testing", + ) + + parser.add_option( + "--disable-werror", + action="store_true", + default=False, + help="Allow warnings to compile", + ) + + parser.add_option( + "--run-under-gdb", + dest="use_gdb", + action="store_true", + default=False, + help="Run test suite under gdb and capture backtrace", + ) + + parser.add_option( + "--test-results-dir", + default=None, + help="Directory to store JUnit XML test reports", + ) return parser.parse_args(args) + def have_prog(prog): """ Check if some named program exists in the path """ - for path in os.environ['PATH'].split(os.pathsep): + for path in os.environ["PATH"].split(os.pathsep): exe_file = os.path.join(path, prog) for ef in [exe_file, exe_file + ".exe"]: if os.path.exists(ef) and os.access(ef, os.X_OK): return True return False + def validate_make_tool(make_tool, build_jobs): - if make_tool == '': - return validate_make_tool('make', build_jobs) + if make_tool == "": + return validate_make_tool("make", build_jobs) - if make_tool not in ['ninja', 'nmake', 'make']: + if make_tool not in ["ninja", "nmake", "make"]: raise Exception("Don't know about %s as a make tool" % (make_tool)) - if make_tool in ['make']: - return make_tool, ['-j%d' % (build_jobs), '-k'] - elif make_tool in ['ninja']: - return make_tool, ['-k', '0'] + if make_tool in ["make"]: + return make_tool, ["-j%d" % (build_jobs), "-k"] + elif make_tool in ["ninja"]: + return make_tool, ["-k", "0"] else: return make_tool, [] + def main(args=None): """ Parse options, do the things """ - if os.getenv('COVERITY_SCAN_BRANCH') == '1': - print('Skipping build COVERITY_SCAN_BRANCH set in environment') + if os.getenv("COVERITY_SCAN_BRANCH") == "1": + print("Skipping build COVERITY_SCAN_BRANCH set in environment") return 0 if args is None: args = sys.argv - print("Invoked as '%s'" % (' '.join(args))) + print("Invoked as '%s'" % (" ".join(args))) (options, args) = parse_args(args) if len(args) != 2: - print('Usage: %s [options] target' % (args[0])) + print("Usage: %s [options] target" % (args[0])) return 1 target = args[1] @@ -697,299 +879,502 @@ def main(args=None): print("Unknown target '%s'" % (target)) return 2 - py_interp = 'python3' + py_interp = "python3" if options.cc_bin is None: - if options.cc == 'gcc': - options.cc_bin = 'g++' - elif options.cc == 'gcc-14': - options.cc = 'gcc' # Hack: 'gcc-14' is not a valid compiler identifier for ``./configure.py --cc`` - options.cc_bin = 'g++-14' - elif options.cc == 'clang': - options.cc_bin = 'clang++' - elif options.cc == 'xcode': - options.cc_bin = 'clang++' - elif options.cc == 'msvc': - options.cc_bin = 'cl' - elif options.cc == 'clangcl': - options.cc_bin = 'clang-cl' + if options.cc == "gcc": + options.cc_bin = "g++" + elif options.cc == "gcc-14": + options.cc = "gcc" # Hack: 'gcc-14' is not a valid compiler identifier for ``./configure.py --cc`` + options.cc_bin = "g++-14" + elif options.cc == "clang": + options.cc_bin = "clang++" + elif options.cc == "xcode": + options.cc_bin = "clang++" + elif options.cc == "msvc": + options.cc_bin = "cl" + elif options.cc == "clangcl": + options.cc_bin = "clang-cl" elif options.cc == "emcc": options.cc_bin = "em++" else: - print('Error unknown compiler %s' % (options.cc)) + print("Error unknown compiler %s" % (options.cc)) return 1 if options.compiler_cache is None: # Autodetect compiler cache - if have_prog('sccache'): - options.compiler_cache = 'sccache' - elif have_prog('ccache'): - options.compiler_cache = 'ccache' + if have_prog("sccache"): + options.compiler_cache = "sccache" + elif have_prog("ccache"): + options.compiler_cache = "ccache" if options.compiler_cache: print("Found '%s' installed, will use it..." % (options.compiler_cache)) - if options.compiler_cache == 'none': + if options.compiler_cache == "none": options.compiler_cache = None - if options.compiler_cache not in [None, 'ccache', 'sccache']: - raise Exception("Don't know about %s as a compiler cache" % (options.compiler_cache)) + if options.compiler_cache not in [None, "ccache", "sccache"]: + raise Exception( + "Don't know about %s as a compiler cache" % (options.compiler_cache) + ) root_dir = options.root_dir build_dir = options.build_dir if not os.access(root_dir, os.R_OK): - raise Exception('Bad root dir setting, dir %s not readable' % (root_dir)) + raise Exception("Bad root dir setting, dir %s not readable" % (root_dir)) if not os.path.exists(build_dir): os.makedirs(build_dir) elif not os.path.isdir(build_dir) or not os.access(build_dir, os.R_OK | os.W_OK): - raise Exception("Bad build dir setting %s is not a directory or not accessible" % (build_dir)) + raise Exception( + "Bad build dir setting %s is not a directory or not accessible" + % (build_dir) + ) cmds = [] - if target == 'lint': - - pylint_rc = '--rcfile=%s' % (os.path.join(root_dir, 'src/configs/pylint.rc')) - pylint_flags = [pylint_rc, '--reports=no'] - pylint_flags += ['--ignored-modules=gdb'] # 'import gdb' is not available outside gdb... + if target == "lint": + pylint_rc = "--rcfile=%s" % (os.path.join(root_dir, "src/configs/pylint.rc")) + pylint_flags = [pylint_rc, "--reports=no"] + pylint_flags += [ + "--ignored-modules=gdb" + ] # 'import gdb' is not available outside gdb... if is_running_in_github_actions(): - pylint_flags += ["--msg-template='::warning file={path},line={line},endLine={end_line}::Pylint ({category}): {msg_id} {msg} ({symbol})'"] + pylint_flags += [ + "--msg-template='::warning file={path},line={line},endLine={end_line}::Pylint ({category}): {msg_id} {msg} ({symbol})'" + ] py_scripts = [ - 'configure.py', - 'src/python/botan3.py', - 'src/scripts/ci_build.py', - 'src/scripts/install.py', - 'src/scripts/ci_check_headers.py', - 'src/scripts/ci_check_install.py', - 'src/scripts/dist.py', - 'src/scripts/cleanup.py', - 'src/scripts/check.py', - 'src/scripts/compare_perf.py', - 'src/scripts/build_docs.py', - 'src/scripts/website.py', - 'src/scripts/bench.py', - 'src/scripts/test_python.py', - 'src/scripts/test_strubbed_symbols.py', - 'src/scripts/test_fuzzers.py', - 'src/scripts/test_cli.py', - 'src/scripts/repo_config.py', - 'src/scripts/python_unittests.py', - 'src/scripts/python_unittests_unix.py', - 'src/scripts/dev_tools/run_clang_format.py', - 'src/scripts/dev_tools/run_clang_tidy.py', - 'src/scripts/gdb/strubtest.py', - 'src/editors/vscode/scripts/bogo.py', - 'src/editors/vscode/scripts/common.py', - 'src/editors/vscode/scripts/test.py', - 'src/ct_selftest/ct_selftest.py'] + "configure.py", + "src/python/botan3.py", + "src/scripts/ci_build.py", + "src/scripts/install.py", + "src/scripts/ci_check_headers.py", + "src/scripts/ci_check_install.py", + "src/scripts/dist.py", + "src/scripts/cleanup.py", + "src/scripts/check.py", + "src/scripts/compare_perf.py", + "src/scripts/build_docs.py", + "src/scripts/website.py", + "src/scripts/bench.py", + "src/scripts/test_python.py", + "src/scripts/test_strubbed_symbols.py", + "src/scripts/test_fuzzers.py", + "src/scripts/test_cli.py", + "src/scripts/repo_config.py", + "src/scripts/python_unittests.py", + "src/scripts/python_unittests_unix.py", + "src/scripts/dev_tools/run_clang_format.py", + "src/scripts/dev_tools/run_clang_tidy.py", + "src/scripts/gdb/strubtest.py", + "src/editors/vscode/scripts/bogo.py", + "src/editors/vscode/scripts/common.py", + "src/editors/vscode/scripts/test.py", + "src/ct_selftest/ct_selftest.py", + ] # This has to run in the repository root to generate the correct # relative paths in the output. Otherwise GitHub Actions will not # be able to annotate the correct files. - cmds.append(["indir:%s" % root_dir, py_interp, '-m', 'pylint'] + pylint_flags + py_scripts) - - elif target == 'typos': - cmds.append(['indir:%s' % (root_dir), 'typos', '-c', 'src/configs/typos.toml', '.']) - elif target == 'format': - cmds.append([py_interp, - os.path.join(root_dir, 'src/scripts/dev_tools/run_clang_format.py'), - '--clang-format=clang-format-17', - '--src-dir=%s' % (os.path.join(root_dir, 'src')), - '--check']) + cmds.append( + ["indir:%s" % root_dir, py_interp, "-m", "pylint"] + + pylint_flags + + py_scripts + ) + + elif target == "typos": + cmds.append( + ["indir:%s" % (root_dir), "typos", "-c", "src/configs/typos.toml", "."] + ) + elif target == "format": + cmds.append( + [ + py_interp, + os.path.join(root_dir, "src/scripts/dev_tools/run_clang_format.py"), + "--clang-format=clang-format-17", + "--src-dir=%s" % (os.path.join(root_dir, "src")), + "--check", + ] + ) else: if options.test_results_dir: os.makedirs(options.test_results_dir) config_flags, run_test_command, make_prefix = determine_flags( - target, options.os, options.cpu, options.cc, options.cc_bin, - options.compiler_cache, root_dir, build_dir, options.test_results_dir, - options.pkcs11_lib, options.use_gdb, options.disable_werror, - options.extra_cxxflags, options.custom_optimization_flags, options.disabled_tests) + target, + options.os, + options.cpu, + options.cc, + options.cc_bin, + options.compiler_cache, + root_dir, + build_dir, + options.test_results_dir, + options.pkcs11_lib, + options.use_gdb, + options.disable_werror, + options.extra_cxxflags, + options.custom_optimization_flags, + options.disabled_tests, + ) make_tool, make_opts = validate_make_tool(options.make_tool, options.build_jobs) - cmds.append([py_interp, - os.path.join(root_dir, 'configure.py')] + - ['--build-tool=' + make_tool] + - config_flags) + cmds.append( + [py_interp, os.path.join(root_dir, "configure.py")] + + ["--build-tool=" + make_tool] + + config_flags + ) make_cmd = [make_tool] + make_opts - if build_dir != '.': - make_cmd = ['indir:%s' % build_dir] + [make_tool] + make_opts + if build_dir != ".": + make_cmd = ["indir:%s" % build_dir] + [make_tool] + make_opts - if target == 'docs': - cmds.append(make_cmd + ['docs']) + if target == "docs": + cmds.append(make_cmd + ["docs"]) else: if options.compiler_cache is not None: - cmds.append([options.compiler_cache, '--show-stats']) + cmds.append([options.compiler_cache, "--show-stats"]) cmds.append(make_prefix + make_cmd + make_targets(target, options.os)) - if target in ['examples'] and options.cc in ['clang', 'gcc']: - cmds.append([options.cc, '-Wall', '-Wextra', '-std=c89', - '-I%s' % (os.path.join(build_dir, 'build/include/public')), - os.path.join(root_dir, 'src/examples/ffi.c'), - '-L%s' % (build_dir), '-lbotan-3', '-o', - os.path.join(build_dir, 'build/examples/ffi')]) + if target in ["examples"] and options.cc in ["clang", "gcc"]: + cmds.append( + [ + options.cc, + "-Wall", + "-Wextra", + "-std=c89", + "-I%s" % (os.path.join(build_dir, "build/include/public")), + os.path.join(root_dir, "src/examples/ffi.c"), + "-L%s" % (build_dir), + "-lbotan-3", + "-o", + os.path.join(build_dir, "build/examples/ffi"), + ] + ) if options.compiler_cache is not None: - cmds.append([options.compiler_cache, '--show-stats']) + cmds.append([options.compiler_cache, "--show-stats"]) if run_test_command is not None: cmds.append(run_test_command) - if target in ['valgrind', 'valgrind-full', 'valgrind-ct', 'valgrind-ct-full']: - - build_config = os.path.join(build_dir, 'build', 'build_config.json') - cmds.append([os.path.join(root_dir, 'src', 'ct_selftest', 'ct_selftest.py'), - "--build-config-path=%s" % build_config, - os.path.join(build_dir, 'botan_ct_selftest')]) + if target in ["valgrind", "valgrind-full", "valgrind-ct", "valgrind-ct-full"]: + build_config = os.path.join(build_dir, "build", "build_config.json") + cmds.append( + [ + os.path.join(root_dir, "src", "ct_selftest", "ct_selftest.py"), + "--build-config-path=%s" % build_config, + os.path.join(build_dir, "botan_ct_selftest"), + ] + ) - valgrind_script_options = ['--test-binary=%s' % (os.path.join(build_dir, 'botan-test')), - '--verbose', - '--bunch', - '--track-origins'] + valgrind_script_options = [ + "--test-binary=%s" % (os.path.join(build_dir, "botan-test")), + "--verbose", + "--bunch", + "--track-origins", + ] # For finding memory bugs, we're enabling more features that add runtime # overhead which we don't need for the secret-dependent execution checks # that 'valgrind-ct' and 'valgrind-ct-full' are aiming for. - if target not in ['valgrind-ct', 'valgrind-ct-full']: - valgrind_script_options.append('--with-leak-check') + if target not in ["valgrind-ct", "valgrind-ct-full"]: + valgrind_script_options.append("--with-leak-check") - if target not in ['valgrind-full', 'valgrind-ct-full']: + if target not in ["valgrind-full", "valgrind-ct-full"]: # valgrind is slow, so some tests only run in the nightly check slow_tests = [ - 'argon2', 'bcrypt', 'bcrypt_pbkdf', 'compression_tests', 'cryptobox', - 'dh_invalid', 'dh_kat', 'dh_keygen', 'dl_group_gen', 'dlies', - 'dsa_kat_verify', 'dsa_param', 'ecc_basemul', 'ecdsa_verify_wycheproof', - 'ed25519_sign', 'elgamal_decrypt', 'elgamal_encrypt', 'elgamal_keygen', - 'ffi_dh', 'ffi_dsa', 'ffi_elgamal', 'frodo_kat_tests', 'hash_nist_mc', - 'hss_lms_keygen', 'hss_lms_sign', 'mce_keygen', 'passhash9', 'pbkdf', - 'pcurves_arith', 'pwdhash', 'rsa_encrypt', 'rsa_pss', 'rsa_pss_raw', 'scrypt', - 'sphincsplus', 'sphincsplus_fors', 'slh_dsa_keygen', 'slh_dsa', 'srp6_kat', - 'srp6_rt', 'unit_tls', 'x509_path_bsi', 'x509_path_rsa_pss', - 'xmss_keygen', 'xmss_keygen_reference', 'xmss_sign', 'xmss_unit_tests', - 'xmss_verify', 'xmss_verify_invalid', + "argon2", + "bcrypt", + "bcrypt_pbkdf", + "compression_tests", + "cryptobox", + "dh_invalid", + "dh_kat", + "dh_keygen", + "dl_group_gen", + "dlies", + "dsa_kat_verify", + "dsa_param", + "ecc_basemul", + "ecdsa_verify_wycheproof", + "ed25519_sign", + "elgamal_decrypt", + "elgamal_encrypt", + "elgamal_keygen", + "ffi_dh", + "ffi_dsa", + "ffi_elgamal", + "frodo_kat_tests", + "hash_nist_mc", + "hss_lms_keygen", + "hss_lms_sign", + "mce_keygen", + "passhash9", + "pbkdf", + "pcurves_arith", + "pwdhash", + "rsa_encrypt", + "rsa_pss", + "rsa_pss_raw", + "scrypt", + "sphincsplus", + "sphincsplus_fors", + "slh_dsa_keygen", + "slh_dsa", + "srp6_kat", + "srp6_rt", + "unit_tls", + "x509_path_bsi", + "x509_path_rsa_pss", + "xmss_keygen", + "xmss_keygen_reference", + "xmss_sign", + "xmss_unit_tests", + "xmss_verify", + "xmss_verify_invalid", + ] + slow_tests += [ + f"dilithium_kat_{mode}_{rand}" + for mode in ("6x5", "8x7", "6x5_AES", "8x7_AES") + for rand in ("Deterministic", "Randomized") + ] + slow_tests += [ + f"ml_dsa_kat_{mode}_{rand}" + for mode in ("6x5", "8x7") + for rand in ("Deterministic", "Randomized") ] - slow_tests += [f"dilithium_kat_{mode}_{rand}" for mode in ('6x5', '8x7', '6x5_AES', '8x7_AES') for rand in ('Deterministic', 'Randomized')] - slow_tests += [f"ml_dsa_kat_{mode}_{rand}" for mode in ('6x5', '8x7') for rand in ('Deterministic', 'Randomized')] - valgrind_script_options.append('--skip-tests=%s' % (','.join(slow_tests))) - elif target == 'valgrind-ct-full' and options.cc == 'clang' and '-Os' in options.custom_optimization_flags: + valgrind_script_options.append( + "--skip-tests=%s" % (",".join(slow_tests)) + ) + elif ( + target == "valgrind-ct-full" + and options.cc == "clang" + and "-Os" in options.custom_optimization_flags + ): # Clang 18 (only) with -Os seems to have a problem with std::optional which flags certain # uses as touching an uninitialized stack variable. This affects the x509_rpki tests # TODO(26.04) We can remove this once we have a new version of Clang to use - valgrind_script_options.append('--skip-tests=x509_rpki') + valgrind_script_options.append("--skip-tests=x509_rpki") - cmds.append(['indir:%s' % (root_dir), - 'src/scripts/run_tests_under_valgrind.py'] + - valgrind_script_options) + cmds.append( + ["indir:%s" % (root_dir), "src/scripts/run_tests_under_valgrind.py"] + + valgrind_script_options + ) - if target in ['coverage', 'sanitizer'] and options.os != 'windows': + if target in ["coverage", "sanitizer"] and options.os != "windows": if not options.boringssl_dir: - raise Exception('coverage build needs --boringssl-dir') - - runner_dir = os.path.abspath(os.path.join(options.boringssl_dir, 'ssl', 'test', 'runner')) - - cmds.append(['indir:%s' % (runner_dir), - 'go', 'test', '-pipe', - '-num-workers', str(4*get_concurrency()), - '-shim-path', os.path.abspath(os.path.join(build_dir, 'botan_bogo_shim')), - '-shim-config', os.path.abspath(os.path.join(root_dir, 'src', 'bogo_shim', 'config.json'))]) - - if target in ['limbo']: - cmds.append([py_interp, os.path.join(root_dir, 'src/scripts/run_limbo_tests.py'), - os.path.join(root_dir, 'limbo.json')]) - - if target in ['coverage', 'fuzzers']: - cmds.append([py_interp, os.path.join(root_dir, 'src/scripts/test_fuzzers.py'), - os.path.join(build_dir, 'fuzzer_corpus'), - os.path.join(build_dir, 'build/fuzzer')]) - - if target in ['shared', 'coverage', 'sanitizer']: - botan_exe = os.path.join(build_dir, 'botan-cli.exe' if options.os == 'windows' else 'botan') - - args = ['--threads=%d' % (options.build_jobs)] - if target in ['coverage']: - args.append('--run-slow-tests') - if root_dir != '.': - args.append('--test-data-dir=%s' % root_dir) - test_scripts = ['test_cli.py', 'test_cli_crypt.py'] + raise Exception("coverage build needs --boringssl-dir") + + runner_dir = os.path.abspath( + os.path.join(options.boringssl_dir, "ssl", "test", "runner") + ) + + cmds.append( + [ + "indir:%s" % (runner_dir), + "go", + "test", + "-pipe", + "-num-workers", + str(4 * get_concurrency()), + "-shim-path", + os.path.abspath(os.path.join(build_dir, "botan_bogo_shim")), + "-shim-config", + os.path.abspath( + os.path.join(root_dir, "src", "bogo_shim", "config.json") + ), + ] + ) + + if target in ["limbo"]: + cmds.append( + [ + py_interp, + os.path.join(root_dir, "src/scripts/run_limbo_tests.py"), + os.path.join(root_dir, "limbo.json"), + ] + ) + + if target in ["coverage", "fuzzers"]: + cmds.append( + [ + py_interp, + os.path.join(root_dir, "src/scripts/test_fuzzers.py"), + os.path.join(build_dir, "fuzzer_corpus"), + os.path.join(build_dir, "build/fuzzer"), + ] + ) + + if target in ["shared", "coverage", "sanitizer"]: + botan_exe = os.path.join( + build_dir, "botan-cli.exe" if options.os == "windows" else "botan" + ) + + args = ["--threads=%d" % (options.build_jobs)] + if target in ["coverage"]: + args.append("--run-slow-tests") + if root_dir != ".": + args.append("--test-data-dir=%s" % root_dir) + test_scripts = ["test_cli.py", "test_cli_crypt.py"] for script in test_scripts: test_data_arg = [] - cmds.append([py_interp, os.path.join(root_dir, 'src/scripts', script)] + - args + test_data_arg + [botan_exe]) - - if target in ['strubbing']: - cmds.append([py_interp, os.path.join(root_dir, 'src/scripts/test_strubbed_symbols.py'), - '--botan-cli', os.path.join(build_dir, 'botan')]) - - if target in ['hybrid-tls13-interop-test']: - cmds.append([py_interp, os.path.join(root_dir, 'src/scripts/test_cli.py'), - '--run-online-tests', os.path.join(build_dir, 'botan'), 'pqc_hybrid_tests']) - - python_tests = [os.path.join(root_dir, 'src/scripts/test_python.py')] - if root_dir != '.': - python_tests.append('--test-data-dir=%s' % root_dir) - - if is_running_in_github_actions() and os.environ.get('BOTAN_TPM2_ENABLED', 'no') == 'test': - python_tests.extend(["--tpm2-tcti-name=%s" % os.getenv('BOTAN_TPM2_TCTI_NAME'), - "--tpm2-tcti-conf=%s" % os.getenv('BOTAN_TPM2_TCTI_CONF')]) - - if target in ['shared', 'coverage'] and not (options.os == 'windows' and options.cpu == 'x86'): - cmds.append([py_interp, '-b'] + python_tests) - - if target in ['shared', 'static']: - cmds.append(make_cmd + ['install']) - build_config = os.path.join(build_dir, 'build', 'build_config.json') - cmds.append([py_interp, os.path.join(root_dir, 'src/scripts/ci_check_install.py'), build_config]) - cmds.append([py_interp, os.path.join(root_dir, 'src/scripts/ci_check_headers.py'), build_config]) - cmds.append([py_interp, os.path.join(root_dir, 'src/scripts/ci_report_sizes.py'), build_config]) - - if target in ['coverage']: - if have_prog('coverage'): - cmds.append(['coverage', 'run', '--branch', - '--rcfile', os.path.join(root_dir, 'src/configs/coverage.rc')] + - python_tests) - - cov_file = os.path.join(build_dir, 'coverage.lcov') - raw_cov_file = os.path.join(build_dir, 'coverage.raw.lcov') - - cmds.append(['lcov', '--capture', '--directory', build_dir, - '--output-file', raw_cov_file]) - cmds.append(['lcov', '--remove', raw_cov_file, '/usr/*', '--output-file', cov_file]) - cmds.append(['lcov', '--list', cov_file]) - cmds.append([os.path.join(root_dir, 'src/scripts/rewrite_lcov.py'), cov_file]) - - if have_prog('coveralls'): + cmds.append( + [py_interp, os.path.join(root_dir, "src/scripts", script)] + + args + + test_data_arg + + [botan_exe] + ) + + if target in ["strubbing"]: + cmds.append( + [ + py_interp, + os.path.join(root_dir, "src/scripts/test_strubbed_symbols.py"), + "--botan-cli", + os.path.join(build_dir, "botan"), + ] + ) + + if target in ["hybrid-tls13-interop-test"]: + cmds.append( + [ + py_interp, + os.path.join(root_dir, "src/scripts/test_cli.py"), + "--run-online-tests", + os.path.join(build_dir, "botan"), + "pqc_hybrid_tests", + ] + ) + + python_tests = [os.path.join(root_dir, "src/scripts/test_python.py")] + if root_dir != ".": + python_tests.append("--test-data-dir=%s" % root_dir) + + if ( + is_running_in_github_actions() + and os.environ.get("BOTAN_TPM2_ENABLED", "no") == "test" + ): + python_tests.extend( + [ + "--tpm2-tcti-name=%s" % os.getenv("BOTAN_TPM2_TCTI_NAME"), + "--tpm2-tcti-conf=%s" % os.getenv("BOTAN_TPM2_TCTI_CONF"), + ] + ) + + if target in ["shared", "coverage"] and not ( + options.os == "windows" and options.cpu == "x86" + ): + cmds.append([py_interp, "-b"] + python_tests) + + if target in ["shared", "static"]: + cmds.append(make_cmd + ["install"]) + build_config = os.path.join(build_dir, "build", "build_config.json") + cmds.append( + [ + py_interp, + os.path.join(root_dir, "src/scripts/ci_check_install.py"), + build_config, + ] + ) + cmds.append( + [ + py_interp, + os.path.join(root_dir, "src/scripts/ci_check_headers.py"), + build_config, + ] + ) + cmds.append( + [ + py_interp, + os.path.join(root_dir, "src/scripts/ci_report_sizes.py"), + build_config, + ] + ) + + if target in ["coverage"]: + if have_prog("coverage"): + cmds.append( + [ + "coverage", + "run", + "--branch", + "--rcfile", + os.path.join(root_dir, "src/configs/coverage.rc"), + ] + + python_tests + ) + + cov_file = os.path.join(build_dir, "coverage.lcov") + raw_cov_file = os.path.join(build_dir, "coverage.raw.lcov") + + cmds.append( + [ + "lcov", + "--capture", + "--directory", + build_dir, + "--output-file", + raw_cov_file, + ] + ) + cmds.append( + ["lcov", "--remove", raw_cov_file, "/usr/*", "--output-file", cov_file] + ) + cmds.append(["lcov", "--list", cov_file]) + cmds.append( + [os.path.join(root_dir, "src/scripts/rewrite_lcov.py"), cov_file] + ) + + if have_prog("coveralls"): # If coveralls command exists, assume we are in CI and report to coveralls.io - cmds.append(['coveralls', '--no-fail', '--format=lcov', '--file=%s' % (cov_file)]) + cmds.append( + [ + "coveralls", + "--no-fail", + "--format=lcov", + "--file=%s" % (cov_file), + ] + ) else: # Otherwise generate a local HTML report - cmds.append(['genhtml', cov_file, '--output-directory', os.path.join(build_dir, 'lcov-out')]) - - cmds.append(make_cmd + ['clean']) - cmds.append(make_cmd + ['distclean']) + cmds.append( + [ + "genhtml", + cov_file, + "--output-directory", + os.path.join(build_dir, "lcov-out"), + ] + ) + + cmds.append(make_cmd + ["clean"]) + cmds.append(make_cmd + ["distclean"]) esdm_process = None # start ESDM in background, if on Linux - if is_running_in_github_actions() and 'BOTAN_BUILD_WITH_ESDM' in os.environ: - print('Starting esdm-server for this target') - esdm_process = subprocess.Popen('sudo /usr/bin/esdm-server -f', shell=True) - assert esdm_process.poll() is None, f"esdm-server did not start for target {target}" + if is_running_in_github_actions() and "BOTAN_BUILD_WITH_ESDM" in os.environ: + print("Starting esdm-server for this target") + esdm_process = subprocess.Popen("sudo /usr/bin/esdm-server -f", shell=True) + esdm_is_ok = esdm_process.poll() is None + assert esdm_is_ok, f"esdm-server did not start for target {target}" for cmd in cmds: if options.dry_run: - print('$ ' + ' '.join(cmd)) + print("$ " + " ".join(cmd)) else: run_cmd(cmd, root_dir, build_dir) if esdm_process: - print('Stopping esdm-server') + print("Stopping esdm-server") esdm_process.kill() return 0 -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/ci_check_headers.py b/src/scripts/ci_check_headers.py index 70cdb7acc6a..91440d87c55 100755 --- a/src/scripts/ci_check_headers.py +++ b/src/scripts/ci_check_headers.py @@ -14,6 +14,7 @@ import json import re + def main(args=None): if args is None: args = sys.argv @@ -22,49 +23,59 @@ def main(args=None): print("Usage: %s " % args[0]) return 1 - with open(os.path.join(args[1]), encoding='utf8') as f: + with open(os.path.join(args[1]), encoding="utf8") as f: build_config = json.load(f) - public_include_dir = build_config['public_include_path'] - internal_include_dir = build_config['internal_include_path'] + public_include_dir = build_config["public_include_path"] + internal_include_dir = build_config["internal_include_path"] - internal_inc = re.compile(r'#include ') + internal_inc = re.compile(r"#include ") - for header in build_config['public_headers']: - contents = open(os.path.join(public_include_dir, header), encoding='utf8').read() + for header in build_config["public_headers"]: + contents = open( + os.path.join(public_include_dir, header), encoding="utf8" + ).read() match = internal_inc.search(contents) if match: - print("ERROR: Public header '%s' includes an internal header '%s'" % (header, match.group(1))) + print( + "ERROR: Public header '%s' includes an internal header '%s'" + % (header, match.group(1)) + ) return 1 - - all_headers = build_config['public_headers'] + build_config['internal_headers'] - header_guard = re.compile(r'\n#define BOTAN_([A-Z0-9_]{3,})_H_\n') + all_headers = build_config["public_headers"] + build_config["internal_headers"] + header_guard = re.compile(r"\n#define BOTAN_([A-Z0-9_]{3,})_H_\n") header_guards = {} for header in all_headers: - - if header in build_config['public_headers']: + if header in build_config["public_headers"]: path = os.path.join(public_include_dir, header) else: path = os.path.join(internal_include_dir, header) - contents = open(path, encoding='utf8').read() + contents = open(path, encoding="utf8").read() match = header_guard.search(contents) if not match: - print("ERROR: Header '%s' is missing an appropriate header guard" % (header)) + print( + "ERROR: Header '%s' is missing an appropriate header guard" + % (header) + ) return 1 guard = match.group(1) if guard in header_guards: - print("ERROR: Duplicate header guard (%s) in '%s' and '%s'" % (guard, header, header_guards[guard])) + print( + "ERROR: Duplicate header guard (%s) in '%s' and '%s'" + % (guard, header, header_guards[guard]) + ) return 1 else: header_guards[guard] = header return 0 -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/ci_check_install.py b/src/scripts/ci_check_install.py index f6a69511522..b4b8e6ef7fb 100755 --- a/src/scripts/ci_check_install.py +++ b/src/scripts/ci_check_install.py @@ -17,8 +17,9 @@ import re import subprocess + def verify_library(build_config): - lib_dir = build_config['libdir'] + lib_dir = build_config["libdir"] if not os.path.isdir(lib_dir): print('Error: libdir "%s" is not a directory' % lib_dir) return False @@ -27,17 +28,17 @@ def verify_library(build_config): major_version = int(build_config["version_major"]) - if build_config['compiler'] in ['msvc', 'clangcl']: - expected_lib_format = r'^botan-%d\.(dll|lib)$' % (major_version) - elif build_config['os'] == 'macos': - expected_lib_format = r'^libbotan-%d\.(a|dylib)$' % (major_version) + if build_config["compiler"] in ["msvc", "clangcl"]: + expected_lib_format = r"^botan-%d\.(dll|lib)$" % (major_version) + elif build_config["os"] == "macos": + expected_lib_format = r"^libbotan-%d\.(a|dylib)$" % (major_version) else: - expected_lib_format = r'^libbotan-%d\.(a|so)$' % (major_version) + expected_lib_format = r"^libbotan-%d\.(a|so)$" % (major_version) lib_re = re.compile(expected_lib_format) # Unlike the include dir this may have other random libs in it - for (_, _, filenames) in os.walk(lib_dir): + for _, _, filenames in os.walk(lib_dir): for filename in filenames: if lib_re.match(filename) is not None: found_libs.add(filename) @@ -51,16 +52,19 @@ def verify_library(build_config): return True + def verify_includes(build_config): - include_dir = build_config['installed_include_dir'] + include_dir = build_config["installed_include_dir"] if not os.path.isdir(include_dir): print('Error: installed_include_dir "%s" is not a directory' % include_dir) return False - expected_headers = set(build_config['public_headers'] + build_config['external_headers']) + expected_headers = set( + build_config["public_headers"] + build_config["external_headers"] + ) found_headers = set([]) - for (_, _, filenames) in os.walk(include_dir): + for _, _, filenames in os.walk(include_dir): for filename in filenames: found_headers.add(filename) @@ -77,40 +81,76 @@ def verify_includes(build_config): return True + def verify_cmake_package(build_config): - if build_config['os'] not in ['windows', 'linux', 'macos']: - return True # skip (e.g. for mingw) + if build_config["os"] not in ["windows", "linux", "macos"]: + return True # skip (e.g. for mingw) - cmake_build_dir = os.path.join(build_config['abs_root_dir'], build_config['build_dir'], 'cmake_test') - cmake_test_dir = os.path.join(build_config['abs_root_dir'], "src", "scripts", "ci", "cmake_tests") + cmake_build_dir = os.path.join( + build_config["abs_root_dir"], build_config["build_dir"], "cmake_test" + ) + cmake_test_dir = os.path.join( + build_config["abs_root_dir"], "src", "scripts", "ci", "cmake_tests" + ) def cmake_preset(): - if build_config['os'] == 'windows': - return 'windows_x86_64' if build_config['arch'] == 'x86_64' else 'windows_x86' - return 'unix' + if build_config["os"] == "windows": + return ( + "windows_x86_64" if build_config["arch"] == "x86_64" else "windows_x86" + ) + return "unix" def test_target(): - return 'test' if build_config['os'] != 'windows' else 'RUN_TESTS' + return "test" if build_config["os"] != "windows" else "RUN_TESTS" - disabled_module = build_config['disabled_mod_list'][0] if build_config['disabled_mod_list'] else None + disabled_module = ( + build_config["disabled_mod_list"][0] + if build_config["disabled_mod_list"] + else None + ) if not disabled_module: - print("Not a single disabled module in this build to use for testing.") # just for good measure + print( + "Not a single disabled module in this build to use for testing." + ) # just for good measure return False try: - subprocess.run(["cmake", "--preset", cmake_preset(), - "-B", cmake_build_dir, - "-S", cmake_test_dir, - "-DCMAKE_PREFIX_PATH=%s" % build_config['prefix'], - "-DBOTAN_DISABLED_MODULE=%s" % disabled_module], check=True) - subprocess.run(["cmake", "--build", cmake_build_dir, "--config", "Release"], check=True) - subprocess.run(["cmake", "--build", cmake_build_dir, "--config", "Release", "--target", test_target()], check=True) + subprocess.run( + [ + "cmake", + "--preset", + cmake_preset(), + "-B", + cmake_build_dir, + "-S", + cmake_test_dir, + "-DCMAKE_PREFIX_PATH=%s" % build_config["prefix"], + "-DBOTAN_DISABLED_MODULE=%s" % disabled_module, + ], + check=True, + ) + subprocess.run( + ["cmake", "--build", cmake_build_dir, "--config", "Release"], check=True + ) + subprocess.run( + [ + "cmake", + "--build", + cmake_build_dir, + "--config", + "Release", + "--target", + test_target(), + ], + check=True, + ) except RuntimeError as e: print("Using the CMake package failed: %s" % str(e)) return False return True + def main(args=None): if args is None: args = sys.argv @@ -119,10 +159,10 @@ def main(args=None): print("Usage: %s " % args[0]) return 1 - with open(os.path.join(args[1]), encoding='utf8') as f: + with open(os.path.join(args[1]), encoding="utf8") as f: build_config = json.load(f) - install_prefix = build_config['prefix'] + install_prefix = build_config["prefix"] if not os.path.isdir(install_prefix): print('Error: install_prefix "%s" is not a directory' % install_prefix) @@ -134,11 +174,15 @@ def main(args=None): if not verify_library(build_config): return 1 - has_cmake = 'botan_cmake_config' in build_config and 'botan_cmake_version_config' in build_config + has_cmake = ( + "botan_cmake_config" in build_config + and "botan_cmake_version_config" in build_config + ) if has_cmake and not verify_cmake_package(build_config): return 1 return 0 -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/ci_report_sizes.py b/src/scripts/ci_report_sizes.py index 54e85f4e5e8..128b9e8f96f 100755 --- a/src/scripts/ci_report_sizes.py +++ b/src/scripts/ci_report_sizes.py @@ -13,6 +13,7 @@ import sys import json + def format_size(bytes): kB = 1024 @@ -21,6 +22,7 @@ def format_size(bytes): else: return "%d bytes" % (bytes) + def report_size(fsname): if not os.access(fsname, os.R_OK): print("ERROR: Could not find %s" % (fsname)) @@ -28,7 +30,8 @@ def report_size(fsname): bytes = os.stat(fsname).st_size print("File '%s' is %s" % (fsname, format_size(bytes))) -def main(args = None): + +def main(args=None): if args is None: args = sys.argv @@ -38,14 +41,15 @@ def main(args = None): build_config = json.loads(open(args[1]).read()) - for lib in build_config['library_targets'].split(' '): + for lib in build_config["library_targets"].split(" "): report_size(lib) - if 'text_exe' in build_config: - report_size(build_config['test_exe']) + if "text_exe" in build_config: + report_size(build_config["test_exe"]) + + if "cli_exe" in build_config: + report_size(build_config["cli_exe"]) - if 'cli_exe' in build_config: - report_size(build_config['cli_exe']) -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/cleanup.py b/src/scripts/cleanup.py index d020576a4f3..3ec150d50ba 100755 --- a/src/scripts/cleanup.py +++ b/src/scripts/cleanup.py @@ -18,16 +18,18 @@ import shutil import errno + def remove_dir(d): try: if os.access(d, os.X_OK): logging.debug('Removing directory "%s"', d) shutil.rmtree(d) else: - logging.debug('Directory %s was missing', d) + logging.debug("Directory %s was missing", d) except Exception as e: logging.error('Failed removing directory "%s": %s', d, e) + def remove_file(f): try: logging.debug('Removing file "%s"', f) @@ -36,6 +38,7 @@ def remove_file(f): if e.errno != errno.ENOENT: logging.error('Failed removing file "%s": %s', f, e) + def remove_all_in_dir(d): if os.access(d, os.X_OK): logging.debug('Removing all files in directory "%s"', d) @@ -49,15 +52,22 @@ def remove_all_in_dir(d): else: remove_file(full_path) + def parse_options(args): parser = optparse.OptionParser() - parser.add_option('--build-dir', default='build', metavar='DIR', - help='specify build dir to clean (default %default)') - - parser.add_option('--distclean', action='store_true', default=False, - help='clean everything') - parser.add_option('--verbose', action='store_true', default=False, - help='noisy logging') + parser.add_option( + "--build-dir", + default="build", + metavar="DIR", + help="specify build dir to clean (default %default)", + ) + + parser.add_option( + "--distclean", action="store_true", default=False, help="clean everything" + ) + parser.add_option( + "--verbose", action="store_true", default=False, help="noisy logging" + ) (options, args) = parser.parse_args(args) @@ -66,28 +76,31 @@ def parse_options(args): return options + def main(args=None): if args is None: args = sys.argv options = parse_options(args) - logging.basicConfig(stream=sys.stderr, - format='%(levelname) 7s: %(message)s', - level=logging.DEBUG if options.verbose else logging.INFO) + logging.basicConfig( + stream=sys.stderr, + format="%(levelname) 7s: %(message)s", + level=logging.DEBUG if options.verbose else logging.INFO, + ) build_dir = options.build_dir if not os.access(build_dir, os.X_OK): - logging.debug('No build directory found') + logging.debug("No build directory found") # No build dir: clean enough! return 0 - build_config_path = os.path.join(build_dir, 'build_config.json') + build_config_path = os.path.join(build_dir, "build_config.json") build_config_str = None try: - build_config_file = open(build_config_path, encoding='utf8') + build_config_file = open(build_config_path, encoding="utf8") build_config_str = build_config_file.read() build_config_file.close() except Exception: @@ -98,36 +111,43 @@ def main(args=None): build_config = json.loads(build_config_str) if options.distclean: - build_dir = build_config['build_dir'] - remove_file(build_config['makefile_path']) + build_dir = build_config["build_dir"] + remove_file(build_config["makefile_path"]) remove_dir(build_dir) else: - for dir_type in ['libobj_dir', 'cliobj_dir', 'testobj_dir', 'handbook_output_dir', 'doc_output_dir_doxygen']: + for dir_type in [ + "libobj_dir", + "cliobj_dir", + "testobj_dir", + "handbook_output_dir", + "doc_output_dir_doxygen", + ]: dir_path = build_config[dir_type] if dir_path: remove_all_in_dir(dir_path) - remove_file(build_config['doc_stamp_file']) + remove_file(build_config["doc_stamp_file"]) - remove_file(build_config['cli_exe']) - remove_file(build_config['test_exe']) + remove_file(build_config["cli_exe"]) + remove_file(build_config["test_exe"]) - lib_basename = build_config['lib_prefix'] + build_config['libname'] - matches_libname = re.compile('^' + lib_basename + '.([a-z]+)((\\.[0-9\\.]+)|$)') + lib_basename = build_config["lib_prefix"] + build_config["libname"] + matches_libname = re.compile("^" + lib_basename + ".([a-z]+)((\\.[0-9\\.]+)|$)") - known_suffix = ['a', 'so', 'dll', 'manifest', 'exp'] + known_suffix = ["a", "so", "dll", "manifest", "exp"] - for f in os.listdir(build_config['out_dir']): + for f in os.listdir(build_config["out_dir"]): match = matches_libname.match(f) if match and match.group(1) in known_suffix: - remove_file(os.path.join(build_config['out_dir'], f)) + remove_file(os.path.join(build_config["out_dir"], f)) if options.distclean: - if 'generated_files' in build_config: - for f in build_config['generated_files'].split(' '): + if "generated_files" in build_config: + for f in build_config["generated_files"].split(" "): remove_file(f) return 0 -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/compare_perf.py b/src/scripts/compare_perf.py index dab63aeaad3..e187a786d81 100755 --- a/src/scripts/compare_perf.py +++ b/src/scripts/compare_perf.py @@ -9,55 +9,58 @@ """ import json -import optparse # pylint: disable=deprecated-module +import optparse # pylint: disable=deprecated-module import sys import re + def ops_per_second(events, nanos): return (events * 1000000000) / nanos + def format_pct(r): - #assert r > 1 + # assert r > 1 return "%.01f%%" % (abs(r - 1) * 100) + def parse_perf_report(report): if len(report) == 0: print("No report data") return None - version = {'version': 'unknown', 'git': 'unknown'} - if 'version' in report[0]: + version = {"version": "unknown", "git": "unknown"} + if "version" in report[0]: version = report[0] - if 'git' in version and version['git'] != 'unknown': - version['git'] = version['git'][:12] + if "git" in version and version["git"] != "unknown": + version["git"] = version["git"][:12] report = report[1:] results = [] for t in report: - if 'algo' in t and 'op' in t and 'events' in t and 'nanos' in t: + if "algo" in t and "op" in t and "events" in t and "nanos" in t: + op = t["op"] + if "buf_size" in t: + op += " " + str(t["buf_size"]) + " buffer" - op = t['op'] - if 'buf_size' in t: - op += ' ' + str(t['buf_size']) + ' buffer' - - results.append(((t['algo'], op), ops_per_second(t['events'], t['nanos']))) + results.append(((t["algo"], op), ops_per_second(t["events"], t["nanos"]))) else: print("Unexpected record", t) results = sorted(results, key=lambda r: r[0]) return (version, results) -def main(args = None): + +def main(args=None): if args is None: args = sys.argv usage = "usage: %prog [options] base.json compare.json" parser = optparse.OptionParser(usage=usage) - parser.add_option('--limit', default=3, help="set reporting limit (as percent)") - parser.add_option('--filter', default='.*', help="filter results by regex") + parser.add_option("--limit", default=3, help="set reporting limit (as percent)") + parser.add_option("--filter", default=".*", help="filter results by regex") (options, args) = parser.parse_args(args) @@ -65,20 +68,25 @@ def main(args = None): print("Usage: compare_perf.py orig.json new.json") return 1 - (ver0, rep0) = parse_perf_report(json.loads(open(args[1], encoding='utf8').read())) - (ver1, rep1) = parse_perf_report(json.loads(open(args[2], encoding='utf8').read())) + (ver0, rep0) = parse_perf_report(json.loads(open(args[1], encoding="utf8").read())) + (ver1, rep1) = parse_perf_report(json.loads(open(args[2], encoding="utf8").read())) reportable = float(options.limit) / 100 filter_re = re.compile(options.filter) def diff(k, a, b): - return k in a and k in b and a[k] != b[k] and (a[k] != "unknown" or b[k] != "unknown") + return ( + k in a + and k in b + and a[k] != b[k] + and (a[k] != "unknown" or b[k] != "unknown") + ) if ver0 and ver1: s = "Diff between " - diff_version = diff('version', ver0, ver1) - diff_git = diff('git', ver0, ver1) + diff_version = diff("version", ver0, ver1) + diff_git = diff("git", ver0, ver1) # TODO check/diff the compiler flags @@ -87,11 +95,11 @@ def diff(k, a, b): if diff_version or diff_git: s += " (" if diff_version and diff_git: - s += ver0['version'] + " " + ver0['git'] + s += ver0["version"] + " " + ver0["git"] elif diff_version: - s += ver0['version'] + s += ver0["version"] elif diff_git: - s += ver0['git'] + s += ver0["git"] s += ")" s += " and " + args[2] @@ -99,11 +107,11 @@ def diff(k, a, b): if diff_version or diff_git: s += " (" if diff_version and diff_git: - s += ver1['version'] + " " + ver1['git'] + s += ver1["version"] + " " + ver1["git"] elif diff_version: - s += ver1['version'] + s += ver1["version"] elif diff_git: - s += ver1['git'] + s += ver1["git"] s += ")" s += "\n" @@ -125,7 +133,7 @@ def diff(k, a, b): if rep0 != [] and rep1 != []: assert rep0[0][0] == rep1[0][0] - algo = ' '.join(rep0[0][0]) + algo = " ".join(rep0[0][0]) if filter_re.search(algo) is not None: orig = rep0[0][1] @@ -144,19 +152,25 @@ def diff(k, a, b): rep0 = rep0[1:] rep1 = rep1[1:] - for (pct, algo) in sorted(speedups, key=lambda v: v[0], reverse=True): + for pct, algo in sorted(speedups, key=lambda v: v[0], reverse=True): print("+ %s improvement in %s" % (format_pct(pct), algo)) - for (pct, algo) in sorted(slowdowns, key=lambda v: v[0]): + for pct, algo in sorted(slowdowns, key=lambda v: v[0]): print("- %s regression in %s" % (format_pct(pct), algo)) if data_points > 0: - print("\nSummary: over %d tests saw %d speedups and %d slowdowns" % (data_points, len(speedups), len(slowdowns))) + print( + "\nSummary: over %d tests saw %d speedups and %d slowdowns" + % (data_points, len(speedups), len(slowdowns)) + ) else: print("\nNo data points") if missing > 0: - print("NOTE: there were %d data points in one set but not the other" % (missing)) + print( + "NOTE: there were %d data points in one set but not the other" % (missing) + ) + -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/config_for_oss_fuzz.py b/src/scripts/config_for_oss_fuzz.py index e9967a166df..9c060ed0292 100755 --- a/src/scripts/config_for_oss_fuzz.py +++ b/src/scripts/config_for_oss_fuzz.py @@ -34,8 +34,10 @@ ] if cxxflags.find("-fsanitize=memory") > 0: - args.append('--disable-asm') + args.append("--disable-asm") -configure = os.path.realpath(os.path.join(os.path.realpath(__file__), '..', '..', '..', 'configure.py')) +configure = os.path.realpath( + os.path.join(os.path.realpath(__file__), "..", "..", "..", "configure.py") +) subprocess.run([configure] + args) diff --git a/src/scripts/create_corpus_zip.py b/src/scripts/create_corpus_zip.py index 96971d31613..9b6259231d0 100755 --- a/src/scripts/create_corpus_zip.py +++ b/src/scripts/create_corpus_zip.py @@ -13,6 +13,7 @@ import zipfile import stat + def main(args=None): if args is None: args = sys.argv @@ -26,26 +27,27 @@ def main(args=None): if len(args) == 3: output_dir = args[2] else: - output_dir = '' + output_dir = "" if not os.access(root_dir, os.R_OK): print("Error could not access directory '%s'" % (root_dir)) return 1 for corpus_dir in os.listdir(root_dir): - if corpus_dir == '.git': + if corpus_dir == ".git": continue subdir = os.path.join(root_dir, corpus_dir) if not stat.S_ISDIR(os.stat(subdir).st_mode): continue - zipfile_path = os.path.join(output_dir, '%s.zip' % (corpus_dir)) - zf = zipfile.ZipFile(zipfile_path, 'w', zipfile.ZIP_DEFLATED) + zipfile_path = os.path.join(output_dir, "%s.zip" % (corpus_dir)) + zf = zipfile.ZipFile(zipfile_path, "w", zipfile.ZIP_DEFLATED) for f in os.listdir(subdir): zf.write(os.path.join(subdir, f), f) zf.close() return 0 -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/dev_tools/addchain.py b/src/scripts/dev_tools/addchain.py index 885c980d874..225f2c5a74a 100755 --- a/src/scripts/dev_tools/addchain.py +++ b/src/scripts/dev_tools/addchain.py @@ -12,23 +12,27 @@ import sys import subprocess + def addchain_gen(n): - search = subprocess.Popen(['addchain', 'search', str(n)], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + search = subprocess.Popen( + ["addchain", "search", str(n)], stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) (stdout, stderr) = search.communicate() - gen = subprocess.Popen(['addchain', 'gen'], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + gen = subprocess.Popen( + ["addchain", "gen"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) gen.stdin.write(stdout) (stdout, stderr) = gen.communicate() - return stdout.decode('utf8').split('\n') + return stdout.decode("utf8").split("\n") + def addchain_code(n, indent=3): vars = set([]) @@ -36,21 +40,21 @@ def addchain_code(n, indent=3): output = [] for line in addchain_gen(n): - if line == '': + if line == "": continue c = line.strip().split() - if c[0] == 'tmp': + if c[0] == "tmp": continue - decl = '' if c[1] in vars else 'auto ' + decl = "" if c[1] in vars else "auto " - if c[0] == 'double': - assert(len(c) == 3) + if c[0] == "double": + assert len(c) == 3 output.append("%s%s = %s.square()" % (decl, c[1], c[2])) vars.add(c[1]) - elif c[0] == 'add': - assert(len(c) == 4) + elif c[0] == "add": + assert len(c) == 4 if c[1] == c[2]: output.append("%s *= %s" % (c[1], c[3])) elif c[1] == c[3]: @@ -61,9 +65,8 @@ def addchain_code(n, indent=3): else: output.append("%s%s = %s * %s" % (decl, c[1], c[3], c[2])) vars.add(c[1]) - elif c[0] == 'shift': - - assert(len(c) == 4) + elif c[0] == "shift": + assert len(c) == 4 if c[1] != c[2]: output.append("%s%s = %s" % (decl, c[1], c[2])) output.append("%s.square_n(%s)" % (c[1], c[3])) @@ -71,12 +74,13 @@ def addchain_code(n, indent=3): else: raise Exception("Don't know what to do with %s" % (c[0])) - output.append('return z') + output.append("return z") ws = " " * indent - return '\n'.join(['%s%s;' % (ws, line) for line in output]) + return "\n".join(["%s%s;" % (ws, line) for line in output]) -def main(args = None): + +def main(args=None): if args is None: args = sys.argv if len(args) != 2: @@ -87,5 +91,6 @@ def main(args = None): print(addchain_code(n)) return 0 -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/dev_tools/analyze_timing_results.py b/src/scripts/dev_tools/analyze_timing_results.py index ef3b9972578..35c24cc4bd4 100755 --- a/src/scripts/dev_tools/analyze_timing_results.py +++ b/src/scripts/dev_tools/analyze_timing_results.py @@ -10,20 +10,22 @@ import sys import numpy + def compute_stats_for(nm, results): return { - 'name': nm, - 'count': len(results), - 'mean': numpy.mean(results), - 'median': numpy.median(results), - 'min': min(results), - 'max': max(results), - 'stddev': numpy.std(results), - '99pct': numpy.percentile(results, 99), - '90pct': numpy.percentile(results, 90) - } - -def main(args = None): + "name": nm, + "count": len(results), + "mean": numpy.mean(results), + "median": numpy.median(results), + "min": min(results), + "max": max(results), + "stddev": numpy.std(results), + "99pct": numpy.percentile(results, 99), + "90pct": numpy.percentile(results, 90), + } + + +def main(args=None): if args is None: args = sys.argv @@ -31,7 +33,7 @@ def main(args = None): print("Usage: %s " % (args[0])) return 1 - timing_line = re.compile('([0-9]+);([0-9]+);([0-9]+)') + timing_line = re.compile("([0-9]+);([0-9]+);([0-9]+)") key_ids = set([]) results = {} @@ -41,7 +43,7 @@ def main(args = None): if match is None: print("Failed to match on '%s'" % (line)) - #cnt = int(match.group(1)) + # cnt = int(match.group(1)) id = int(match.group(2)) time = int(match.group(3)) @@ -52,12 +54,22 @@ def main(args = None): for id in key_ids: s = compute_stats_for("secret %d" % (id), results[id]) - print("%d: min=%.02f max=%.02f mean=%.02f median=%.02f std=%.02f 90pct=%.02f 99pct=%.02f" % ( - id, s['min'], s['max'], s['mean'], s['median'], s['stddev'], s['90pct'], s['99pct'])) + print( + "%d: min=%.02f max=%.02f mean=%.02f median=%.02f std=%.02f 90pct=%.02f 99pct=%.02f" + % ( + id, + s["min"], + s["max"], + s["mean"], + s["median"], + s["stddev"], + s["90pct"], + s["99pct"], + ) + ) stats[id] = s - for field in ['mean', 'median', 'min', 'max', 'stddev', '90pct', '99pct']: - + for field in ["mean", "median", "min", "max", "stddev", "90pct", "99pct"]: res = [] for id in key_ids: res.append(stats[id][field]) @@ -66,5 +78,6 @@ def main(args = None): print("range of %s: %.02f (%.02f - %.02f)" % (field, range, min(res), max(res))) -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/dev_tools/file_size_check.py b/src/scripts/dev_tools/file_size_check.py index 50ada693489..d936f467266 100755 --- a/src/scripts/dev_tools/file_size_check.py +++ b/src/scripts/dev_tools/file_size_check.py @@ -5,25 +5,28 @@ import re import sys + def lines_in(f): lines = 0 - for line in f.decode('utf8').splitlines(): - if line == '': + for line in f.decode("utf8").splitlines(): + if line == "": continue - if line.startswith('#'): + if line.startswith("#"): continue lines += 1 return lines + def run_cc(cmd): - preproc = subprocess.run(cmd.split(' '), stdout=subprocess.PIPE) + preproc = subprocess.run(cmd.split(" "), stdout=subprocess.PIPE) return lines_in(preproc.stdout) -cc = json.loads(open('build/compile_commands.json').read()) -src_file = re.compile('-E src/.*/([a-z0-9/_]+.cpp) ') +cc = json.loads(open("build/compile_commands.json").read()) + +src_file = re.compile("-E src/.*/([a-z0-9/_]+.cpp) ") search_for = None @@ -32,7 +35,7 @@ def run_cc(cmd): total_lines = 0 for c in cc: - cmd = c['command'].replace(' -c ', ' -E ').split(' -o')[0] + " -o -" + cmd = c["command"].replace(" -c ", " -E ").split(" -o")[0] + " -o -" file_name = src_file.search(cmd) if file_name is None: continue diff --git a/src/scripts/dev_tools/format_wycheproof_ecdsa.py b/src/scripts/dev_tools/format_wycheproof_ecdsa.py index 7bf2af8f4de..4551ad98ce7 100755 --- a/src/scripts/dev_tools/format_wycheproof_ecdsa.py +++ b/src/scripts/dev_tools/format_wycheproof_ecdsa.py @@ -10,62 +10,65 @@ import json import re + def map_group_name(group): - if group.startswith('brainpoolP'): - return group.replace('P', '') + if group.startswith("brainpoolP"): + return group.replace("P", "") return group -def main(args = None): + +def main(args=None): if args is None: args = sys.argv - brainpool_t = re.compile('brainpoolP[0-9]+t1') + brainpool_t = re.compile("brainpoolP[0-9]+t1") last_group = None last_msg = None for file in args[1:]: - if file.find('p1363') > 0: + if file.find("p1363") > 0: continue - if file.find('webcrypto') > 0: + if file.find("webcrypto") > 0: continue data = json.loads(open(file).read()) - assert(data['algorithm'] == 'ECDSA') + assert data["algorithm"] == "ECDSA" - for group in data['testGroups']: - if brainpool_t.match(group['key']['curve']): + for group in data["testGroups"]: + if brainpool_t.match(group["key"]["curve"]): continue - if last_group != group['key']['curve']: - print("Group =", map_group_name(group['key']['curve'])) - last_group = group['key']['curve'] - print("Hash =", group['sha']) + if last_group != group["key"]["curve"]: + print("Group =", map_group_name(group["key"]["curve"])) + last_group = group["key"]["curve"] + print("Hash =", group["sha"]) print() - print("Px = 0x%s" % (group['key']['wx'])) - print("Py = 0x%s" % (group['key']['wy'])) + print("Px = 0x%s" % (group["key"]["wx"])) + print("Py = 0x%s" % (group["key"]["wy"])) - for test in group['tests']: - print("\n# Test %d (%s)" % (test['tcId'], test['comment'])) + for test in group["tests"]: + print("\n# Test %d (%s)" % (test["tcId"], test["comment"])) - if last_msg != test['msg']: - print("Msg =", test['msg']) - last_msg = test['msg'] - print("Signature =", test['sig']) + if last_msg != test["msg"]: + print("Msg =", test["msg"]) + last_msg = test["msg"] + print("Signature =", test["sig"]) - accept = test['result'][0].upper() == 'V' + accept = test["result"][0].upper() == "V" # We force this to V since all other "Acceptable" signatures # in the test set are ones we do want to reject, eg due to # invalid DER encodings - if(test['comment'] == 'Hash weaker than DL-group'): + if test["comment"] == "Hash weaker than DL-group": accept = True print("Valid =", 1 if accept else 0) print() -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/dev_tools/gen_dilithium_kat.py b/src/scripts/dev_tools/gen_dilithium_kat.py index 6c4e830ca77..805b4cac65b 100755 --- a/src/scripts/dev_tools/gen_dilithium_kat.py +++ b/src/scripts/dev_tools/gen_dilithium_kat.py @@ -16,6 +16,7 @@ import hashlib import binascii + class KatReader: def __init__(self, file): self.file = file @@ -28,10 +29,10 @@ def next_value(self): if line == "": return (None, None) - if line.startswith('#') or line == "\n": + if line.startswith("#") or line == "\n": continue - key, val = line.strip().split(' = ') + key, val = line.strip().split(" = ") return (key, val) @@ -42,94 +43,102 @@ def read_kats(self): key, val = self.next_value() if key is None: - return # eof + return # eof - if key not in ['count', 'seed', 'mlen', 'msg', 'pk', 'sk', 'smlen', 'sm']: + if key not in ["count", "seed", "mlen", "msg", "pk", "sk", "smlen", "sm"]: raise Exception("Unknown key %s" % (key)) - if key in ['count', 'mlen', 'smlen']: + if key in ["count", "mlen", "smlen"]: kat[key] = int(val) - elif key == 'sm': + elif key == "sm": # remove message appended to signature - kat[key] = val[:-kat['mlen']*2] + kat[key] = val[: -kat["mlen"] * 2] else: kat[key] = val - if key == 'sm': + if key == "sm": yield kat kat = {} + def sha3_256(v): # v is assumed to be hex h = hashlib.sha3_256() h.update(binascii.unhexlify(v)) return h.hexdigest() + def compress_kat(kat): - first = kat['count'] == 0 - del kat['count'] - del kat['smlen'] - del kat['mlen'] + first = kat["count"] == 0 + del kat["count"] + del kat["smlen"] + del kat["mlen"] # rename keys - kat['Seed'] = kat.pop('seed') - kat['Msg'] = kat.pop('msg') + kat["Seed"] = kat.pop("seed") + kat["Msg"] = kat.pop("msg") - kat['HashPk'] = sha3_256(kat.pop('pk')) - kat['HashSk'] = sha3_256(kat.pop('sk')) + kat["HashPk"] = sha3_256(kat.pop("pk")) + kat["HashSk"] = sha3_256(kat.pop("sk")) - sig = kat.pop('sm') + sig = kat.pop("sm") if first: - kat['Sig'] = sig - kat['HashSig'] = sha3_256(sig) + kat["Sig"] = sig + kat["HashSig"] = sha3_256(sig) return kat + def map_mode(mode): - if mode == 'Dilithium2': - return '4x4' - if mode == 'Dilithium2-AES': - return '4x4_AES' - if mode == 'Dilithium3': - return '6x5' - if mode == 'Dilithium3-AES': - return '6x5_AES' - if mode == 'Dilithium5': - return '8x7' - if mode == 'Dilithium5-AES': - return '8x7_AES' - - raise Exception('Unknown Dilithium mode', mode) - -def main(args = None): + if mode == "Dilithium2": + return "4x4" + if mode == "Dilithium2-AES": + return "4x4_AES" + if mode == "Dilithium3": + return "6x5" + if mode == "Dilithium3-AES": + return "6x5_AES" + if mode == "Dilithium5": + return "8x7" + if mode == "Dilithium5-AES": + return "8x7_AES" + + raise Exception("Unknown Dilithium mode", mode) + + +def main(args=None): if args is None: args = sys.argv randomized = True is_mldsa = True - type = 'Randomized' if randomized else 'Deterministic' - name = 'ml-dsa' if is_mldsa else 'dilithium' + type = "Randomized" if randomized else "Deterministic" + name = "ml-dsa" if is_mldsa else "dilithium" for file in args[1:]: mode = map_mode(open(file).readline().strip()[2:]) reader = KatReader(open(file)) - output = open('src/tests/data/pubkey/%s_%s_%s.vec' % (name, mode, type), 'w') + output = open("src/tests/data/pubkey/%s_%s_%s.vec" % (name, mode, type), "w") print("# See src/scripts/dev_tools/gen_dilithium_kat.py\n", file=output) - print("[%s_%s]" % (name.upper() if is_mldsa else name.capitalize(), mode), file=output) + print( + "[%s_%s]" % (name.upper() if is_mldsa else name.capitalize(), mode), + file=output, + ) for kat in reader.read_kats(): kat = compress_kat(kat) - for key in ['Seed', 'Msg', 'HashPk', 'HashSk', 'Sig', 'HashSig']: + for key in ["Seed", "Msg", "HashPk", "HashSk", "Sig", "HashSig"]: if key in kat: - print(key, '=', kat[key], file=output) + print(key, "=", kat[key], file=output) print("\n", file=output) output.close() -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/dev_tools/gen_ec_groups.py b/src/scripts/dev_tools/gen_ec_groups.py index c0873be0cb7..21aaf44d032 100755 --- a/src/scripts/dev_tools/gen_ec_groups.py +++ b/src/scripts/dev_tools/gen_ec_groups.py @@ -30,41 +30,45 @@ be installed. """ + def curve_info(src): - re_kv = re.compile('([A-Za-z]+) = ([0-9A-Za-z-_\\. ]+)') + re_kv = re.compile("([A-Za-z]+) = ([0-9A-Za-z-_\\. ]+)") current = {} for line in src: line = line.strip() - if line == '': + if line == "": continue kv = re_kv.match(line) if kv is None: - raise Exception('Unknown line', line) + raise Exception("Unknown line", line) key = kv.group(1) val = kv.group(2) - if key in ['Name']: + if key in ["Name"]: current[key] = val - elif key in ['OID', 'Impl']: - current[key] = val.split(' ') - elif key in ['A']: - if val == '-3': - current[key] = current['P'] - 3 + elif key in ["OID", "Impl"]: + current[key] = val.split(" ") + elif key in ["A"]: + if val == "-3": + current[key] = current["P"] - 3 else: current[key] = int(val, 16) - elif key in ['P', 'B', 'X', 'Y', 'N']: + elif key in ["P", "B", "X", "Y", "N"]: current[key] = int(val, 16) - if key == 'N': + if key == "N": current["N32"] = current["N"] & 0xFFFFFFFF - current["OIDExpr"] = ['OID{%s}' % (oid.replace('.', ', ')) for oid in current['OID']] + current["OIDExpr"] = [ + "OID{%s}" % (oid.replace(".", ", ")) for oid in current["OID"] + ] yield current current = {} + def format_names(names): # This would be quite complicated to render in the Jinja template language so # we pre-render it as a string and insert it directly @@ -73,15 +77,15 @@ def format_names(names): pcurves = [] pcurves_no_generic = [] - for (nm, impl) in names: - if 'pcurve' in impl: + for nm, impl in names: + if "pcurve" in impl: pcurves.append(nm) - if 'generic' not in impl: + if "generic" not in impl: pcurves_no_generic.append(nm) - elif 'generic' in impl: + elif "generic" in impl: generic.append(nm) else: - assert 'legacy' in impl + assert "legacy" in impl legacy.append(nm) legacy_macro = "defined(BOTAN_HAS_LEGACY_EC_POINT)" @@ -93,24 +97,30 @@ def format_names(names): yield "#if defined(%s) || %s" % (nm_macro, legacy_macro) yield " // Not supported by pcurves_generic" else: - yield "#if defined(%s) || %s || %s" % (nm_macro, legacy_macro, generic_macro) - yield ' \"%s\",' % (nm) + yield "#if defined(%s) || %s || %s" % ( + nm_macro, + legacy_macro, + generic_macro, + ) + yield ' "%s",' % (nm) yield "#endif\n" yield "#if %s || %s" % (legacy_macro, generic_macro) for nm in sorted(generic): - yield ' \"%s\",' % (nm) + yield ' "%s",' % (nm) yield "#endif\n" yield "#if %s" % (legacy_macro) for nm in sorted(legacy): - yield ' \"%s\",' % (nm) + yield ' "%s",' % (nm) yield "#endif\n" + def datestamp(): current_date = datetime.datetime.now() return int(current_date.strftime("%Y%m%d")) + class OmitFirstLine: def __init__(self): self.first_line = True @@ -120,12 +130,13 @@ def __call__(self, line): self.first_line = False return r + def main(): - curves = [c for c in curve_info(open('./src/build-data/ec_groups.txt'))] + curves = [c for c in curve_info(open("./src/build-data/ec_groups.txt"))] pcurves = [] for c in curves: - if 'pcurve' in c['Impl']: + if "pcurve" in c["Impl"]: pcurves.append(c) this_script = sys.argv[0] @@ -134,39 +145,56 @@ def main(): env = Environment(loader=FileSystemLoader("src/build-data/templates")) # write ec_named.cpp - with open('./src/lib/pubkey/ec_group/ec_named.cpp', encoding='utf8', mode='w') as ec_named: + with open( + "./src/lib/pubkey/ec_group/ec_named.cpp", encoding="utf8", mode="w" + ) as ec_named: template = env.get_template("ec_named.cpp.in") - named_groups = "\n".join(format_names([(c['Name'], c['Impl']) for c in curves])) + named_groups = "\n".join(format_names([(c["Name"], c["Impl"]) for c in curves])) - ec_named.write(template.render(script=this_script, date=date, curves=curves, named_groups=named_groups.strip())) + ec_named.write( + template.render( + script=this_script, + date=date, + curves=curves, + named_groups=named_groups.strip(), + ) + ) ec_named.write("\n") # write pcurves_instance.h - with open('./src/lib/math/pcurves/pcurves_instance.h', encoding='utf8', mode='w') as pcurves_h: + with open( + "./src/lib/math/pcurves/pcurves_instance.h", encoding="utf8", mode="w" + ) as pcurves_h: template = env.get_template("pcurves_instance.h.in") pcurves_h.write(template.render(script=this_script, date=date, pcurves=pcurves)) pcurves_h.write("\n") # write pcurves.cpp - with open('./src/lib/math/pcurves/pcurves.cpp', encoding='utf8', mode='w') as pcurves_cpp: + with open( + "./src/lib/math/pcurves/pcurves.cpp", encoding="utf8", mode="w" + ) as pcurves_cpp: template = env.get_template("pcurves.cpp.in") - pcurves_cpp.write(template.render(script=this_script, date=date, pcurves=pcurves)) + pcurves_cpp.write( + template.render(script=this_script, date=date, pcurves=pcurves) + ) pcurves_cpp.write("\n") # Check if any pcurves modules need a new stub impl for pcurve in pcurves: curve = pcurve["Name"] - mod_dir = './src/lib/math/pcurves/pcurves_%s' % (curve) - info_path = os.path.join(mod_dir, 'info.txt') - impl_path = os.path.join(mod_dir, f'pcurves_{curve}.cpp') + mod_dir = "./src/lib/math/pcurves/pcurves_%s" % (curve) + info_path = os.path.join(mod_dir, "info.txt") + impl_path = os.path.join(mod_dir, f"pcurves_{curve}.cpp") if os.access(impl_path, os.R_OK): continue - addchain_fe2 = addchain_code(pcurve['P'] - 3, 0) - addchain_fe_sqrt = addchain_code((pcurve['P'] + 1) // 4, 0) if pcurve['P'] % 4 == 3 else None - addchain_scalar = addchain_code(pcurve['N'] - 2, 0) + addchain_fe2 = addchain_code(pcurve["P"] - 3, 0) + addchain_fe_sqrt = ( + addchain_code((pcurve["P"] + 1) // 4, 0) if pcurve["P"] % 4 == 3 else None + ) + addchain_scalar = addchain_code(pcurve["N"] - 2, 0) try: os.makedirs(mod_dir) @@ -175,11 +203,15 @@ def main(): raise module_define = f"PCURVES_{curve.upper()}" - if not re.match('^[0-9A-Za-z_]{3,30}$', module_define): - raise ValueError(f"Invalid preprocessor define name ({module_define}) for new pcurve module") - - with open(info_path, 'w', encoding='utf8') as info_file: - info_file.write(dedent(f"""\ + if not re.match("^[0-9A-Za-z_]{3,30}$", module_define): + raise ValueError( + f"Invalid preprocessor define name ({module_define}) for new pcurve module" + ) + + with open(info_path, "w", encoding="utf8") as info_file: + info_file.write( + dedent( + f"""\ {module_define} -> {datestamp()} @@ -191,23 +223,33 @@ def main(): pcurves_impl - """)) + """ + ) + ) - with open(impl_path, 'w', encoding='utf8') as src_file: + with open(impl_path, "w", encoding="utf8") as src_file: crandall = (1 << pcurve["P"].bit_length()) - pcurve["P"] if crandall > 2**32: crandall = 0 template = env.get_template("pcurves_stub.cpp.in") - src_file.write(template.render(curve = pcurve, - crandall=crandall, - addchain_fe2=indent(addchain_fe2, 9 * ' ', OmitFirstLine()), - addchain_fe_sqrt=indent(addchain_fe_sqrt, 9 * ' ', OmitFirstLine()) if addchain_fe_sqrt else None, - addchain_scalar=indent(addchain_scalar, 9 * ' ', OmitFirstLine()))) + src_file.write( + template.render( + curve=pcurve, + crandall=crandall, + addchain_fe2=indent(addchain_fe2, 9 * " ", OmitFirstLine()), + addchain_fe_sqrt=( + indent(addchain_fe_sqrt, 9 * " ", OmitFirstLine()) + if addchain_fe_sqrt + else None + ), + addchain_scalar=indent(addchain_scalar, 9 * " ", OmitFirstLine()), + ) + ) src_file.write("\n") - return 0 -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/dev_tools/gen_ffi_decls.py b/src/scripts/dev_tools/gen_ffi_decls.py index 6cf189b2648..ee779020371 100755 --- a/src/scripts/dev_tools/gen_ffi_decls.py +++ b/src/scripts/dev_tools/gen_ffi_decls.py @@ -11,92 +11,92 @@ import traceback from pycparser import c_ast, parse_file -ffi_header = 'src/lib/ffi/ffi.h' +ffi_header = "src/lib/ffi/ffi.h" -def to_ctype(typ, is_ptr): - if typ.startswith('botan_') and typ.endswith('_t'): - return 'c_void_p' +def to_ctype(typ, is_ptr): + if typ.startswith("botan_") and typ.endswith("_t"): + return "c_void_p" - if typ == 'botan_view_ctx': - return 'c_void_p' + if typ == "botan_view_ctx": + return "c_void_p" - if typ == 'botan_view_bin_fn': - return 'VIEW_BIN_CALLBACK' - if typ == 'botan_view_str_fn': - return 'VIEW_STR_CALLBACK' + if typ == "botan_view_bin_fn": + return "VIEW_BIN_CALLBACK" + if typ == "botan_view_str_fn": + return "VIEW_STR_CALLBACK" if is_ptr is False: - if typ == 'uint32': - return 'c_uint32' - elif typ == 'size_t': - return 'c_size_t' - elif typ == 'uint8_t': - return 'c_uint8' - elif typ == 'uint32_t': - return 'c_uint32' - elif typ == 'uint64_t': - return 'c_uint64' - elif typ == 'int': - return 'c_int' - elif typ == 'unsigned': - return 'c_uint' + if typ == "uint32": + return "c_uint32" + elif typ == "size_t": + return "c_size_t" + elif typ == "uint8_t": + return "c_uint8" + elif typ == "uint32_t": + return "c_uint32" + elif typ == "uint64_t": + return "c_uint64" + elif typ == "int": + return "c_int" + elif typ == "unsigned": + return "c_uint" else: - if typ == 'void': - return 'c_void_p' - elif typ in ['char', 'uint8_t']: # hack - return 'c_char_p' - elif typ == 'size_t': - return 'POINTER(c_size_t)' - #elif typ == 'uint8_t': + if typ == "void": + return "c_void_p" + elif typ in ["char", "uint8_t"]: # hack + return "c_char_p" + elif typ == "size_t": + return "POINTER(c_size_t)" + # elif typ == 'uint8_t': # return 'POINTER(c_uint8)' - elif typ == 'uint32_t': - return 'POINTER(c_uint32)' - elif typ == 'uint64_t': - return 'POINTER(c_uint64)' - elif typ == 'int': - return 'POINTER(c_int)' + elif typ == "uint32_t": + return "POINTER(c_uint32)" + elif typ == "uint64_t": + return "POINTER(c_uint64)" + elif typ == "int": + return "POINTER(c_int)" raise Exception("Unknown type %s/%d" % (typ, is_ptr)) + class FuncDefVisitor(c_ast.NodeVisitor): def __init__(self): self.fn_decls = [] def emit_decls(self): - decls = '' - for (fn_name, fn_args) in self.fn_decls: + decls = "" + for fn_name, fn_args in self.fn_decls: decl = " ffi_api(dll.%s," % (fn_name) if len(fn_args) > 4: decl += "\n " else: - decl += ' ' + decl += " " - decl += '[' + ', '.join(fn_args) + '])\n' + decl += "[" + ", ".join(fn_args) + "])\n" decls += decl return decls def visit_FuncDecl(self, node): - if not isinstance(node.type, c_ast.TypeDecl): - #print("ignoring", node.type) + # print("ignoring", node.type) return - if node.type.type.names != ['int']: - #print("ignoring", node.type) + if node.type.type.names != ["int"]: + # print("ignoring", node.type) return # all functions returning ints: fn_name = node.type.declname ignored = [ - 'botan_view_bin_fn', - 'botan_view_str_fn', - 'botan_same_mem', # deprecated - 'botan_mceies_encrypt', # dead - 'botan_mceies_decrypt', # dead - 'botan_rng_init_custom', # fixme + "botan_view_bin_fn", + "botan_view_str_fn", + "botan_same_mem", # deprecated + "botan_mceies_encrypt", # dead + "botan_mceies_decrypt", # dead + "botan_rng_init_custom", # fixme ] if fn_name in ignored: return @@ -105,7 +105,6 @@ def visit_FuncDecl(self, node): try: for param in node.args.params: - is_ptr = False typ = None @@ -130,10 +129,20 @@ def visit_FuncDecl(self, node): print(traceback.format_exc()) print("FAILED for '%s': %s" % (fn_name, e)) -ast = parse_file(ffi_header, use_cpp=True, cpp_args=['-Ibuild/include', '-std=c89', '-DBOTAN_DLL=', '-DBOTAN_NO_DEPRECATED_WARNINGS']) -#print(ast) + +ast = parse_file( + ffi_header, + use_cpp=True, + cpp_args=[ + "-Ibuild/include", + "-std=c89", + "-DBOTAN_DLL=", + "-DBOTAN_NO_DEPRECATED_WARNINGS", + ], +) +# print(ast) v = FuncDefVisitor() v.visit(ast) -#print("\n".join(v.fn_decls)) +# print("\n".join(v.fn_decls)) print(v.emit_decls()) diff --git a/src/scripts/dev_tools/gen_frodo_kat.py b/src/scripts/dev_tools/gen_frodo_kat.py index 861aebedda1..c34b0b5d2db 100644 --- a/src/scripts/dev_tools/gen_frodo_kat.py +++ b/src/scripts/dev_tools/gen_frodo_kat.py @@ -19,6 +19,7 @@ import binascii import os + class KatReader: def __init__(self, file): self.file = file @@ -31,10 +32,10 @@ def next_value(self): if line == "": return (None, None) - if line.startswith('#') or line == "\n": + if line.startswith("#") or line == "\n": continue - key, val = line.strip().split(' = ') + key, val = line.strip().split(" = ") return (key, val) @@ -45,39 +46,42 @@ def read_kats(self): key, val = self.next_value() if key is None: - return # eof + return # eof - if key not in ['count', 'seed', 'pk', 'sk', 'ct', 'ss']: + if key not in ["count", "seed", "pk", "sk", "ct", "ss"]: raise Exception("Unknown key %s" % (key)) - if key == 'count': + if key == "count": kat[key] = int(val) else: kat[key] = val - if key == 'ss': + if key == "ss": yield kat kat = {} + def shake_256_16(v): # v is assumed to be hex h = hashlib.shake_256() h.update(binascii.unhexlify(v)) return h.hexdigest(16) + def compress_kat(kat): - del kat['count'] # not needed + del kat["count"] # not needed # rename keys - kat['Seed'] = kat.pop('seed') - kat['SS'] = kat.pop('ss') - kat['PK'] = shake_256_16(kat.pop('pk')) - kat['SK'] = shake_256_16(kat.pop('sk')) - kat['CT'] = shake_256_16(kat.pop('ct')) + kat["Seed"] = kat.pop("seed") + kat["SS"] = kat.pop("ss") + kat["PK"] = shake_256_16(kat.pop("pk")) + kat["SK"] = shake_256_16(kat.pop("sk")) + kat["CT"] = shake_256_16(kat.pop("ct")) return kat -def map_mode(mode, is_ephemeral = False): + +def map_mode(mode, is_ephemeral=False): out = None if mode == "PQCkemKAT_19888": out = "eFrodoKEM-640-AES" @@ -93,14 +97,15 @@ def map_mode(mode, is_ephemeral = False): out = "eFrodoKEM-1344-SHAKE" if out is None: - raise Exception('Unknown FrodoKEM mode', mode) + raise Exception("Unknown FrodoKEM mode", mode) if is_ephemeral: return out else: return out[1:] # remove 'e' to obtain 'FrodoKEM' -def main(args = None): + +def main(args=None): """Set True for eFrodo, otherwise the non-ephemeral variant is assumed. Necessary because the files have the same name for either variant.""" is_ephemeral = False @@ -108,8 +113,11 @@ def main(args = None): if args is None: args = sys.argv - with open('src/tests/data/pubkey/frodokem_kat.vec', 'w') as output: - print("# This file was auto-generated from the reference implementation's KATs", file=output) + with open("src/tests/data/pubkey/frodokem_kat.vec", "w") as output: + print( + "# This file was auto-generated from the reference implementation's KATs", + file=output, + ) print("# See src/scripts/dev_tools/gen_frodo_kat.py\n", file=output) for file in args[1:]: @@ -123,8 +131,9 @@ def main(args = None): kat = compress_kat(kat) for key in kat.keys(): - print(key, '=', kat[key], file=output) + print(key, "=", kat[key], file=output) print("", file=output) -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/dev_tools/gen_kyber_kat.py b/src/scripts/dev_tools/gen_kyber_kat.py index eed0592ec40..6d7697e4a4e 100755 --- a/src/scripts/dev_tools/gen_kyber_kat.py +++ b/src/scripts/dev_tools/gen_kyber_kat.py @@ -33,6 +33,7 @@ import os import argparse + class KatReader: def __init__(self, file): self.file = file @@ -44,10 +45,10 @@ def next_value(self): if line == "": return (None, None) - if line.startswith('#') or line == "\n": + if line.startswith("#") or line == "\n": continue - key, val = line.strip().split(' = ') + key, val = line.strip().split(" = ") return (key, val) @@ -58,56 +59,70 @@ def read_kats(self): key, val = self.next_value() if key is None: - return # eof + return # eof - if key in ['msg']: + if key in ["msg"]: continue - if key not in ['count', 'z', 'd', 'seed', 'pk', 'sk', 'ct', 'ss', 'ct_n', 'ss_n']: + if key not in [ + "count", + "z", + "d", + "seed", + "pk", + "sk", + "ct", + "ss", + "ct_n", + "ss_n", + ]: raise Exception("Unknown key %s" % (key)) - if key == 'count': + if key == "count": kat[key] = int(val) else: kat[key] = val - if key == 'ss': + if key == "ss": yield kat kat = {} + def shake_256_16(v): # v is assumed to be hex h = hashlib.shake_256() h.update(binascii.unhexlify(v)) return h.hexdigest(16) + def sha256_16(v): # v is assumed to be hex h = hashlib.sha256() h.update(binascii.unhexlify(v)) return h.hexdigest()[:32] + def compress_kat(kat, mode): - del kat['count'] # Not needed + del kat["count"] # Not needed - hash_fn = sha256_16 if '90s' in mode else shake_256_16 + hash_fn = sha256_16 if "90s" in mode else shake_256_16 # For ML-KEM we use the private seed as the private key's storage format if mode == "ML-KEM": - kat['sk'] = kat['d'] + kat['z'] - del kat['d'] - del kat['z'] + kat["sk"] = kat["d"] + kat["z"] + del kat["d"] + del kat["z"] # rename keys and hash large values to reduce size of KAT vectors - kat['Seed'] = kat.pop('seed') - kat['SS'] = kat.pop('ss') - kat['PK'] = hash_fn(kat.pop('pk')) - kat['SK'] = hash_fn(kat.pop('sk')) - kat['CT'] = hash_fn(kat.pop('ct')) + kat["Seed"] = kat.pop("seed") + kat["SS"] = kat.pop("ss") + kat["PK"] = hash_fn(kat.pop("pk")) + kat["SK"] = hash_fn(kat.pop("sk")) + kat["CT"] = hash_fn(kat.pop("ct")) if mode == "ML-KEM": - kat['CT_N'] = kat.pop('ct_n') - kat['SS_N'] = kat.pop('ss_n') + kat["CT_N"] = kat.pop("ct_n") + kat["SS_N"] = kat.pop("ss_n") return kat @@ -115,10 +130,24 @@ def compress_kat(kat, mode): def parse_arguments(): parser = argparse.ArgumentParser() parser.add_argument("files", nargs="+", help="Input files") - parser.add_argument("--kyber-r3", action="store_true", help="Enable Kyber R3 mode", default=False) - parser.add_argument("--ml-kem-ipd", action="store_true", help="Enable ML-KEM initial public draft mode", default=False) - parser.add_argument("--ml-kem", action="store_true", help="Enable ML-KEM final mode", default=False) - parser.add_argument("--kats-per-mode", type=int, help="Number of KATs to generate per mode", default=25) + parser.add_argument( + "--kyber-r3", action="store_true", help="Enable Kyber R3 mode", default=False + ) + parser.add_argument( + "--ml-kem-ipd", + action="store_true", + help="Enable ML-KEM initial public draft mode", + default=False, + ) + parser.add_argument( + "--ml-kem", action="store_true", help="Enable ML-KEM final mode", default=False + ) + parser.add_argument( + "--kats-per-mode", + type=int, + help="Number of KATs to generate per mode", + default=25, + ) return parser.parse_args() @@ -154,9 +183,9 @@ def map_mode(file_name, mode): if file_name == "kat_MLKEM_768": return "ML-KEM-768" else: - raise Exception('Unknown mode', mode) + raise Exception("Unknown mode", mode) - raise Exception('Unknown Kyber KAT file name', file_name) + raise Exception("Unknown Kyber KAT file name", file_name) def selected_mode(args): @@ -189,17 +218,23 @@ def output_file(mode): raise Exception("Unknown mode", mode) -def main(args = None): +def main(args=None): if args is None: return 1 mode = selected_mode(args) - with open(output_file(mode), 'w') as output: + with open(output_file(mode), "w") as output: if mode == "ML-KEM": - print("# This file was auto-generated from github.com/post-quantum-cryptography/KAT", file=output) + print( + "# This file was auto-generated from github.com/post-quantum-cryptography/KAT", + file=output, + ) else: - print("# This file was auto-generated from the reference implementation's KATs", file=output) + print( + "# This file was auto-generated from the reference implementation's KATs", + file=output, + ) print("# See src/scripts/dev_tools/gen_kyber_kat.py\n", file=output) for file in args.files: @@ -209,13 +244,14 @@ def main(args = None): print(f"[{algo_mode}]", file=output) - for kat in list(reader.read_kats())[:args.kats_per_mode]: + for kat in list(reader.read_kats())[: args.kats_per_mode]: kat = compress_kat(kat, mode) for key in kat.keys(): - print(key, '=', kat[key], file=output) + print(key, "=", kat[key], file=output) print("", file=output) -if __name__ == '__main__': + +if __name__ == "__main__": args = parse_arguments() sys.exit(main(args)) diff --git a/src/scripts/dev_tools/gen_mlkem_acvp_kat.py b/src/scripts/dev_tools/gen_mlkem_acvp_kat.py index 903d83c965c..a3a130fe578 100644 --- a/src/scripts/dev_tools/gen_mlkem_acvp_kat.py +++ b/src/scripts/dev_tools/gen_mlkem_acvp_kat.py @@ -16,12 +16,14 @@ import argparse import sys + def shake_256_16(v): # v is assumed to be hex h = hashlib.shake_256() h.update(binascii.unhexlify(v)) return h.hexdigest(16) + def mlkem_load_keygen(req_fn, res_fn): with open(req_fn) as f: keygen_req = json.load(f) @@ -29,26 +31,27 @@ def mlkem_load_keygen(req_fn, res_fn): keygen_res = json.load(f) keygen_kat = [] - for qtg in keygen_req['testGroups']: - alg = qtg['parameterSet'] - tgid = qtg['tgId'] + for qtg in keygen_req["testGroups"]: + alg = qtg["parameterSet"] + tgid = qtg["tgId"] rtg = None - for tg in keygen_res['testGroups']: - if tg['tgId'] == tgid: - rtg = tg['tests'] + for tg in keygen_res["testGroups"]: + if tg["tgId"] == tgid: + rtg = tg["tests"] break - for qt in qtg['tests']: - tcid = qt['tcId'] + for qt in qtg["tests"]: + tcid = qt["tcId"] for t in rtg: - if t['tcId'] == tcid: + if t["tcId"] == tcid: qt.update(t) - qt['parameterSet'] = alg + qt["parameterSet"] = alg keygen_kat += [qt] return keygen_kat + def mlkem_load_encdec(req_fn, res_fn): with open(req_fn) as f: encdec_req = json.load(f) @@ -57,76 +60,91 @@ def mlkem_load_encdec(req_fn, res_fn): encaps_kat = [] decaps_kat = [] - for qtg in encdec_req['testGroups']: - alg = qtg['parameterSet'] - func = qtg['function'] - tgid = qtg['tgId'] + for qtg in encdec_req["testGroups"]: + alg = qtg["parameterSet"] + func = qtg["function"] + tgid = qtg["tgId"] rtg = None - for tg in encdec_res['testGroups']: - if tg['tgId'] == tgid: - rtg = tg['tests'] + for tg in encdec_res["testGroups"]: + if tg["tgId"] == tgid: + rtg = tg["tests"] break - for qt in qtg['tests']: - tcid = qt['tcId'] + for qt in qtg["tests"]: + tcid = qt["tcId"] for t in rtg: - if t['tcId'] == tcid: + if t["tcId"] == tcid: qt.update(t) - qt['parameterSet'] = alg - if func == 'encapsulation': + qt["parameterSet"] = alg + if func == "encapsulation": encaps_kat += [qt] - elif func == 'decapsulation': - qt['dk'] = qtg['dk'] + elif func == "decapsulation": + qt["dk"] = qtg["dk"] decaps_kat += [qt] else: - print('ERROR: Unknown function:', func) + print("ERROR: Unknown function:", func) return (encaps_kat, decaps_kat) + def group_by_parameter_set(keygen_kat): grouped_kat = {} for kat in keygen_kat: - parameter_set = kat['parameterSet'] + parameter_set = kat["parameterSet"] if parameter_set not in grouped_kat: grouped_kat[parameter_set] = [] grouped_kat[parameter_set].append(kat) return grouped_kat + def parse_arguments(): parser = argparse.ArgumentParser() - parser.add_argument("--keygen-directory", type=str, help="Directory 'ML-KEM-keyGen-FIPS203' containing the JSON files") - parser.add_argument("--encapdecap-directory", type=str, help="Directory 'ML-KEM-encapDecap-FIPS203' containing the JSON files") + parser.add_argument( + "--keygen-directory", + type=str, + help="Directory 'ML-KEM-keyGen-FIPS203' containing the JSON files", + ) + parser.add_argument( + "--encapdecap-directory", + type=str, + help="Directory 'ML-KEM-encapDecap-FIPS203' containing the JSON files", + ) return parser.parse_args() + def compress_kat_keygen(kat): - kat['Z'] = kat.pop('z') - kat['D'] = kat.pop('d') - kat['EK'] = shake_256_16(kat.pop('ek')) - kat['DK'] = shake_256_16(kat['D'] + kat['Z']) - del kat['dk'] # remove the original decaps key (Botan uses the private seed as storage format) + kat["Z"] = kat.pop("z") + kat["D"] = kat.pop("d") + kat["EK"] = shake_256_16(kat.pop("ek")) + kat["DK"] = shake_256_16(kat["D"] + kat["Z"]) + del kat[ + "dk" + ] # remove the original decaps key (Botan uses the private seed as storage format) return kat + def compress_kat_encaps(kat): - kat['EK'] = kat.pop('ek') - kat['M'] = kat.pop('m') - kat['K'] = kat.pop('k') - kat['C'] = shake_256_16(kat.pop('c')) + kat["EK"] = kat.pop("ek") + kat["M"] = kat.pop("m") + kat["K"] = kat.pop("k") + kat["C"] = shake_256_16(kat.pop("c")) return kat -def main(args = None): + +def main(args=None): if args is None: return 1 if args.keygen_directory is not None: - keygen_kat = mlkem_load_keygen( - args.keygen_directory + '/prompt.json', - args.keygen_directory + '/expectedResults.json') - with open("src/tests/data/pubkey/ml_kem_acvp_keygen.vec", 'w') as output: + args.keygen_directory + "/prompt.json", + args.keygen_directory + "/expectedResults.json", + ) + with open("src/tests/data/pubkey/ml_kem_acvp_keygen.vec", "w") as output: print("# This file was auto-generated from the ACVP KATs", file=output) print("# See src/scripts/dev_tools/gen_mlkem_acvp_kat.py\n", file=output) @@ -137,14 +155,15 @@ def main(args = None): kat = compress_kat_keygen(kat) for key in kat.keys(): if key in ["Z", "D", "EK", "DK"]: - print(key, '=', kat[key], file=output) + print(key, "=", kat[key], file=output) print("", file=output) if args.encapdecap_directory is not None: encaps_kat, decaps_kat = mlkem_load_encdec( - args.encapdecap_directory + '/prompt.json', - args.encapdecap_directory + '/expectedResults.json') + args.encapdecap_directory + "/prompt.json", + args.encapdecap_directory + "/expectedResults.json", + ) - with open("src/tests/data/pubkey/ml_kem_acvp_encap.vec", 'w') as output: + with open("src/tests/data/pubkey/ml_kem_acvp_encap.vec", "w") as output: print("# This file was auto-generated from the ACVP KATs", file=output) print("# See src/scripts/dev_tools/gen_mlkem_acvp_kat.py\n", file=output) @@ -155,10 +174,10 @@ def main(args = None): kat = compress_kat_encaps(kat) for key in kat.keys(): if key in ["M", "EK", "K", "C"]: - print(key, '=', kat[key], file=output) + print(key, "=", kat[key], file=output) print("", file=output) - with open("src/tests/data/pubkey/ml_kem_acvp_decap.vec", 'w') as output: + with open("src/tests/data/pubkey/ml_kem_acvp_decap.vec", "w") as output: print("# This file was auto-generated from the ACVP KATs", file=output) print("# See src/scripts/dev_tools/gen_mlkem_acvp_kat.py\n", file=output) @@ -176,7 +195,6 @@ def main(args = None): # print("", file=output) - -if __name__ == '__main__': +if __name__ == "__main__": args = parse_arguments() sys.exit(main(args)) diff --git a/src/scripts/dev_tools/gen_mp_comba.py b/src/scripts/dev_tools/gen_mp_comba.py index eea3fe01e10..0e805b44e13 100755 --- a/src/scripts/dev_tools/gen_mp_comba.py +++ b/src/scripts/dev_tools/gen_mp_comba.py @@ -8,49 +8,51 @@ # Used to generate src/lib/math/mp/mp_comba.cpp -def comba_indexes(N): +def comba_indexes(N): indexes = [] - for i in range(0, 2*N): + for i in range(0, 2 * N): x = [] - for j in range(max(0, i-N+1), min(N, i+1)): - x += [(j,i-j)] + for j in range(max(0, i - N + 1), min(N, i + 1)): + x += [(j, i - j)] indexes += [sorted(x)] return indexes -def comba_sqr_indexes(N): +def comba_sqr_indexes(N): indexes = [] - for i in range(0, 2*N): + for i in range(0, 2 * N): x = [] - for j in range(max(0, i-N+1), min(N, i+1)): - if j < i-j: - x += [(j,i-j)] + for j in range(max(0, i - N + 1), min(N, i + 1)): + if j < i - j: + x += [(j, i - j)] else: - x += [(i-j,j)] + x += [(i - j, j)] indexes += [sorted(x)] return indexes + def comba_multiply_code(N): indexes = comba_indexes(N) - for (i,idx) in zip(range(0, len(indexes)), indexes): + for i, idx in zip(range(0, len(indexes)), indexes): for pair in idx: print(" accum.mul(x[%d], y[%d]);" % (pair[0], pair[1])) print(" z[%d] = accum.extract();" % (i)) + def comba_square_code(N): indexes = comba_sqr_indexes(N) - for (rnd,idx) in zip(range(0, len(indexes)), indexes): - for (i,pair) in zip(range(0, len(idx)), idx): + for rnd, idx in zip(range(0, len(indexes)), indexes): + for i, pair in zip(range(0, len(idx)), idx): if pair[0] == pair[1]: print(" accum.mul(x[%d], x[%d]);" % (pair[0], pair[1])) elif i % 2 == 0: @@ -58,7 +60,8 @@ def comba_square_code(N): print(" z[%d] = accum.extract();" % (rnd)) -def main(args = None): + +def main(args=None): if args is None: args = sys.argv @@ -67,7 +70,8 @@ def main(args = None): else: sizes = map(int, args[1:]) - print("""/* + print( + """/* * Comba Multiplication and Squaring * * This file was automatically generated by %s on %s @@ -78,11 +82,13 @@ def main(args = None): #include namespace Botan { -""" % (sys.argv[0], datetime.date.today().strftime("%Y-%m-%d"))) +""" + % (sys.argv[0], datetime.date.today().strftime("%Y-%m-%d")) + ) for n in sizes: print("/*\n* Comba %dx%d Squaring\n*/" % (n, n)) - print("void bigint_comba_sqr%d(word z[%d], const word x[%d]) {" % (n, 2*n, n)) + print("void bigint_comba_sqr%d(word z[%d], const word x[%d]) {" % (n, 2 * n, n)) print(" word3 accum;\n") comba_square_code(n) @@ -90,7 +96,10 @@ def main(args = None): print("}\n") print("/*\n* Comba %dx%d Multiplication\n*/" % (n, n)) - print("void bigint_comba_mul%d(word z[%d], const word x[%d], const word y[%d]) {" % (n, 2*n, n, n)) + print( + "void bigint_comba_mul%d(word z[%d], const word x[%d], const word y[%d]) {" + % (n, 2 * n, n, n) + ) print(" word3 accum;\n") comba_multiply_code(n) @@ -101,5 +110,6 @@ def main(args = None): return 0 -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/dev_tools/gen_mp_monty.py b/src/scripts/dev_tools/gen_mp_monty.py index 75c0745ad1f..37b99b288aa 100755 --- a/src/scripts/dev_tools/gen_mp_monty.py +++ b/src/scripts/dev_tools/gen_mp_monty.py @@ -8,16 +8,22 @@ # Used to generate src/lib/math/mp/mp_monty_n.cpp -def monty_redc_code(n, p_dash1=False): +def monty_redc_code(n, p_dash1=False): lines = [] fn_name = "bigint_monty_redc_pdash1" if p_dash1 else "bigint_monty_redc" if p_dash1: - hdr = "void %s_%d(word r[%d], const word z[%d], const word p[%d], word ws[%d]) {\n" % (fn_name, n, n, 2*n, n, n) + hdr = ( + "void %s_%d(word r[%d], const word z[%d], const word p[%d], word ws[%d]) {\n" + % (fn_name, n, n, 2 * n, n, n) + ) else: - hdr = "void %s_%d(word r[%d], const word z[%d], const word p[%d], word p_dash, word ws[%d]) {\n" % (fn_name, n, n, 2*n, n, n) + hdr = ( + "void %s_%d(word r[%d], const word z[%d], const word p[%d], word p_dash, word ws[%d]) {\n" + % (fn_name, n, n, 2 * n, n, n) + ) ftr = "\n}\n" @@ -30,7 +36,7 @@ def monty_redc_code(n, p_dash1=False): for i in range(1, n): for j in range(0, i): - lines.append("accum.mul(ws[%d], p[%d]);" % (j, i-j)) + lines.append("accum.mul(ws[%d], p[%d]);" % (j, i - j)) lines.append("accum.add(z[%d]);" % (i)) if p_dash1: @@ -40,12 +46,12 @@ def monty_redc_code(n, p_dash1=False): for i in range(0, n - 1): for j in range(i + 1, n): - lines.append("accum.mul(ws[%d], p[%d]);" % (j, n + i-j)) + lines.append("accum.mul(ws[%d], p[%d]);" % (j, n + i - j)) - lines.append("accum.add(z[%d]);" % (n+i)) + lines.append("accum.add(z[%d]);" % (n + i)) lines.append("ws[%d] = accum.extract();" % (i)) - lines.append("accum.add(z[%d]);" % (2*n-1)) + lines.append("accum.add(z[%d]);" % (2 * n - 1)) lines.append("ws[%d] = accum.extract();" % (n - 1)) lines.append("word w1 = accum.extract();") @@ -54,7 +60,8 @@ def monty_redc_code(n, p_dash1=False): return hdr + "\n".join([" %s" % (line) for line in lines]) + ftr -def main(args = None): + +def main(args=None): if args is None: args = sys.argv @@ -63,7 +70,8 @@ def main(args = None): else: sizes = map(int, args[1:]) - print("""/* + print( + """/* * This file was automatically generated by %s on %s * All manual changes will be lost. Edit the script instead. * @@ -75,7 +83,9 @@ def main(args = None): #include namespace Botan { -""" % (sys.argv[0], datetime.date.today().strftime("%Y-%m-%d"))) +""" + % (sys.argv[0], datetime.date.today().strftime("%Y-%m-%d")) + ) for n in sizes: print(monty_redc_code(n)) @@ -84,7 +94,6 @@ def main(args = None): return 0 -if __name__ == '__main__': - sys.exit(main()) - +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/scripts/dev_tools/gen_oids.py b/src/scripts/dev_tools/gen_oids.py index 8eba522708e..5ae8a781266 100755 --- a/src/scripts/dev_tools/gen_oids.py +++ b/src/scripts/dev_tools/gen_oids.py @@ -14,21 +14,23 @@ import re from jinja2 import Environment, FileSystemLoader + # This must match OID::hash_code def hash_oid(oid): - word_size = 2 ** 64 + word_size = 2**64 h = 0x621F302327D9A49A - for part in map(int, oid.split('.')): + for part in map(int, oid.split(".")): h = (h * 193) % word_size h += part # This reduction step occurs in static_oids.cpp.in return h % 858701 + # This must match hash_oid_name in static_oids.cpp.in def hash_oid_name(name): - word_size = 2 ** 64 + word_size = 2**64 h = 0x8188B31879A4879A @@ -38,8 +40,10 @@ def hash_oid_name(name): return h % 805289 + def format_oid(oid): - return '{' + oid.replace('.', ', ') + '}' + return "{" + oid.replace(".", ", ") + "}" + def render_static_oid(m): res = [] @@ -47,37 +51,41 @@ def render_static_oid(m): name_hashes = {} oid_hashes = {} - for (k, v) in m.items(): - + for k, v in m.items(): # Verify no collisions between any of the values oid_hc = hash_oid(v) if oid_hc in oid_hashes: - raise Exception("Hash collision between %s and %s" % (v, oid_hashes[oid_hc])) + raise Exception( + "Hash collision between %s and %s" % (v, oid_hashes[oid_hc]) + ) oid_hashes[oid_hc] = v name_hc = hash_oid_name(k) if name_hc in name_hashes: - raise Exception("Hash collision between %s and %s" % (k, name_hashes[name_hc])) + raise Exception( + "Hash collision between %s and %s" % (k, name_hashes[name_hc]) + ) name_hashes[name_hc] = k - res.append({ 'oid_hash': oid_hc, - 'name_hash': name_hc, - 'name': k, - 'oid': format_oid(v) }) + res.append( + {"oid_hash": oid_hc, "name_hash": name_hc, "name": k, "oid": format_oid(v)} + ) return res + def format_oid_with_name(m): - return [ {'name': kv[0], 'oid': format_oid(kv[1])} for kv in m ] + return [{"name": kv[0], "oid": format_oid(kv[1])} for kv in m] -def main(args = None): + +def main(args=None): """ Regenerate src/lib/asn1/static_oids.cpp """ if args is None: args = sys.argv - oid_lines = open('./src/build-data/oids.txt', encoding='utf8').readlines() + oid_lines = open("./src/build-data/oids.txt", encoding="utf8").readlines() oid_re = re.compile(r"^([0-9][0-9.]+) += +([A-Za-z0-9_\./\(\), -]+)$") hdr_re = re.compile(r"^\[([a-z0-9_]+)\]$") @@ -92,7 +100,7 @@ def main(args = None): if len(line) == 0: continue - if line[0] == '#': + if line[0] == "#": continue match = hdr_re.match(line) @@ -119,16 +127,23 @@ def main(args = None): env = Environment(loader=FileSystemLoader("src/build-data/templates")) - with open('./src/lib/asn1/static_oids.cpp', encoding='utf8', mode='w') as static_oids: + with open( + "./src/lib/asn1/static_oids.cpp", encoding="utf8", mode="w" + ) as static_oids: template = env.get_template("static_oids.cpp.in") - static_oids.write(template.render(script=this_script, - date=date, - static_oid_data=render_static_oid(str2oid), - dup_oids=format_oid_with_name(dup_oids), - aliases=format_oid_with_name(aliases))) + static_oids.write( + template.render( + script=this_script, + date=date, + static_oid_data=render_static_oid(str2oid), + dup_oids=format_oid_with_name(dup_oids), + aliases=format_oid_with_name(aliases), + ) + ) static_oids.write("\n") return 0 -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/dev_tools/gen_os_features.py b/src/scripts/dev_tools/gen_os_features.py index 7a447b06429..b848ec9c83f 100755 --- a/src/scripts/dev_tools/gen_os_features.py +++ b/src/scripts/dev_tools/gen_os_features.py @@ -22,26 +22,32 @@ # locale sys.path.append(botan_root) -from configure import OsInfo # noqa: E402 +from configure import OsInfo # noqa: E402 parser = argparse.ArgumentParser(description="") -parser.add_argument('--verbose', dest='verbose', action='store_const', - const=True, default=False, - help='Verbose output (default: false)') +parser.add_argument( + "--verbose", + dest="verbose", + action="store_const", + const=True, + default=False, + help="Verbose output (default: false)", +) args = parser.parse_args() + def update_os(): - TABLE_TITLE="OS Features" + TABLE_TITLE = "OS Features" files = [] - files += glob.glob(botan_root + '/src/build-data/os/*.txt') + files += glob.glob(botan_root + "/src/build-data/os/*.txt") files.sort() if len(files) == 0: print("No info.txt files found.") sys.exit(1) - f1 = open(os.path.join(botan_root, 'doc', 'dev_ref', 'os.rst'), 'w+') + f1 = open(os.path.join(botan_root, "doc", "dev_ref", "os.rst"), "w+") all_features = set() oss = {} @@ -75,21 +81,28 @@ def update_os(): print(" %s: %s" % (o[0:1], o), file=f1) print("", file=f1) - print('.. csv-table::', file=f1) - print(' :header: "Feature", "' + '", "'.join([o[0:1] for o in oslist]) + '"', file=f1) - print('', file=f1) + print(".. csv-table::", file=f1) + print( + ' :header: "Feature", "' + '", "'.join([o[0:1] for o in oslist]) + '"', + file=f1, + ) + print("", file=f1) for f in featurelist: line = ' "' + f + '"' for o in oslist: line += ', "' - line += 'X' if f in oss[o].target_features else ' ' + line += "X" if f in oss[o].target_features else " " line += '"' print(line, file=f1) print("", file=f1) print(".. note::", file=f1) - print(" This file is auto generated by ``src/scripts/%s``. Dont modify it manually." - % os.path.basename(sys.argv[0]), file=f1) + print( + " This file is auto generated by ``src/scripts/%s``. Dont modify it manually." + % os.path.basename(sys.argv[0]), + file=f1, + ) + -if __name__ == '__main__': +if __name__ == "__main__": update_os() diff --git a/src/scripts/dev_tools/gen_pqc_dsa_kats.py b/src/scripts/dev_tools/gen_pqc_dsa_kats.py index 498e8789106..c413c5bca21 100644 --- a/src/scripts/dev_tools/gen_pqc_dsa_kats.py +++ b/src/scripts/dev_tools/gen_pqc_dsa_kats.py @@ -18,16 +18,19 @@ random_bytes = os.urandom + # "ML-DSA-44" -> "ml-dsa-4x4" def transform_mldsa_string(s): s = s.lower() - transformed = re.sub(r'(\d)(\d)$', r'\1x\2', s) + transformed = re.sub(r"(\d)(\d)$", r"\1x\2", s) return transformed + def create_mldsa_ini_label(s, det): - algo = re.sub(r'(\d)(\d)$', r'\1x\2', s) + algo = re.sub(r"(\d)(\d)$", r"\1x\2", s) return f"[{algo}_{det}]" + def sha3_256(v): h = hashlib.sha3_256() h.update(v) @@ -39,23 +42,29 @@ def sha256(v): h.update(v) return h.digest() -def sign(sign_internal_func, sk_bytes, m, random_bytes, n, ctx=b"", deterministic=False): - if len(ctx) > 255: - raise ValueError( - f"ctx bytes must have length at most 255, ctx has length {len(ctx) = }" - ) - if deterministic: - rnd = None - else: - rnd = random_bytes(n) #drbg.random_bytes(n) - # Format the message using the context - m_prime = bytes([0]) + bytes([len(ctx)]) + ctx + m - # Compute the signature of m_prime - sig_bytes = sign_internal_func(m_prime, sk_bytes, rnd) - return sig_bytes - -def main(args = None): - msg = binascii.unhexlify("D81C4D8D734FCBFBEADE3D3F8A039FAA2A2C9957E835AD55B22E75BF57BB556AC8") + +def sign( + sign_internal_func, sk_bytes, m, random_bytes, n, ctx=b"", deterministic=False +): + if len(ctx) > 255: + raise ValueError( + f"ctx bytes must have length at most 255, ctx has length {len(ctx) = }" + ) + if deterministic: + rnd = None + else: + rnd = random_bytes(n) # drbg.random_bytes(n) + # Format the message using the context + m_prime = bytes([0]) + bytes([len(ctx)]) + ctx + m + # Compute the signature of m_prime + sig_bytes = sign_internal_func(m_prime, sk_bytes, rnd) + return sig_bytes + + +def main(args=None): + msg = binascii.unhexlify( + "D81C4D8D734FCBFBEADE3D3F8A039FAA2A2C9957E835AD55B22E75BF57BB556AC8" + ) if args is None: args = sys.argv @@ -66,11 +75,13 @@ def main(args = None): algo = args[1] # Initialize DRBG with the magic value - entropy = binascii.unhexlify("60496cd0a12512800a79161189b055ac3996ad24e578d3c5fc57c1e60fa2eb4e550d08e51e9db7b67f1a616681d9182d") + entropy = binascii.unhexlify( + "60496cd0a12512800a79161189b055ac3996ad24e578d3c5fc57c1e60fa2eb4e550d08e51e9db7b67f1a616681d9182d" + ) drbg = AES256_CTR_DRBG(seed=entropy) if algo == "slh_dsa": - output = open('src/tests/data/pubkey/slh_dsa.vec', 'w') + output = open("src/tests/data/pubkey/slh_dsa.vec", "w") print("# See src/scripts/dev_tools/gen_pqc_dsa_kats.py\n", file=output) @@ -80,13 +91,27 @@ def main(args = None): seed = drbg.random_bytes(len(entropy)) drbg_instance = AES256_CTR_DRBG(seed=seed) - keygen_rand = drbg_instance.random_bytes(3*alg.n) - sk_seed = keygen_rand[:alg.n] - sk_prf = keygen_rand[alg.n:2*alg.n] - pk_seed = keygen_rand[2*alg.n:] + keygen_rand = drbg_instance.random_bytes(3 * alg.n) + sk_seed = keygen_rand[: alg.n] + sk_prf = keygen_rand[alg.n : 2 * alg.n] + pk_seed = keygen_rand[2 * alg.n :] pk, sk = alg.slh_keygen_internal(sk_seed, sk_prf, pk_seed) - signature_rand = sign(alg.slh_sign_internal, sk, msg, drbg_instance.random_bytes, alg.n, deterministic=False) - signature_det = sign(alg.slh_sign_internal, sk, msg, drbg_instance.random_bytes, alg.n, deterministic=True) + signature_rand = sign( + alg.slh_sign_internal, + sk, + msg, + drbg_instance.random_bytes, + alg.n, + deterministic=False, + ) + signature_det = sign( + alg.slh_sign_internal, + sk, + msg, + drbg_instance.random_bytes, + alg.n, + deterministic=True, + ) print("SphincsParameterSet = %s" % param, file=output) print("seed = %s" % seed.hex().upper(), file=output) @@ -94,26 +119,46 @@ def main(args = None): print("pk = %s" % pk.hex().upper(), file=output) print("sk = %s" % sk.hex().upper(), file=output) if param == "SLH-DSA-SHAKE-128s": - print("SigDet = %s" % binascii.hexlify(signature_det).decode("utf-8").upper(), file=output) - print("HashSigDet = %s" % binascii.hexlify(hash_fn(signature_det)).decode("utf-8").upper(), file=output) - print("HashSigRand = %s" % binascii.hexlify(hash_fn(signature_rand)).decode("utf-8").upper(), file=output) + print( + "SigDet = %s" + % binascii.hexlify(signature_det).decode("utf-8").upper(), + file=output, + ) + print( + "HashSigDet = %s" + % binascii.hexlify(hash_fn(signature_det)).decode("utf-8").upper(), + file=output, + ) + print( + "HashSigRand = %s" + % binascii.hexlify(hash_fn(signature_rand)).decode("utf-8").upper(), + file=output, + ) print("", file=output) elif algo == "ml_dsa": for param in ML_DSA_PARAM.keys(): - for det_str, deterministic in [("Deterministic", True), ("Randomized", False)]: + for det_str, deterministic in [ + ("Deterministic", True), + ("Randomized", False), + ]: alg = ML_DSA(param) - output = open('src/tests/data/pubkey/%s_%s.vec' % (transform_mldsa_string(param), det_str), 'w') + output = open( + "src/tests/data/pubkey/%s_%s.vec" + % (transform_mldsa_string(param), det_str), + "w", + ) print("# See src/scripts/dev_tools/gen_pqc_dsa_kats.py", file=output) print("", file=output) print(create_mldsa_ini_label(param, det_str), file=output) print("", file=output) hash_fn = sha3_256 + def mldsa_sign_internal(m, sk, rnd): # For some reason the interfaces vary between FIPS 204 and FIPS 205... if rnd is None: - rnd = bytes([0]*32) + rnd = bytes([0] * 32) return alg.sign_internal(sk, m, rnd) samples = 25 @@ -127,18 +172,39 @@ def mldsa_sign_internal(m, sk, rnd): drbg_instance = AES256_CTR_DRBG(seed=seed) keygen_rand = drbg_instance.random_bytes(32) pk, sk = alg.keygen_internal(keygen_rand) - signature = sign(mldsa_sign_internal, sk, msg, drbg_instance.random_bytes, 32, deterministic=deterministic) + signature = sign( + mldsa_sign_internal, + sk, + msg, + drbg_instance.random_bytes, + 32, + deterministic=deterministic, + ) print("Seed = %s" % seed.hex().upper(), file=output) print("Msg = %s" % msg.hex().upper(), file=output) - print("HashPk = %s" % binascii.hexlify(hash_fn(pk)).decode("utf-8").upper(), file=output) - print("HashSk = %s" % binascii.hexlify(hash_fn(keygen_rand)).decode("utf-8").upper(), file=output) - print("HashSig = %s" % binascii.hexlify(hash_fn(signature)).decode("utf-8").upper(), file=output) + print( + "HashPk = %s" + % binascii.hexlify(hash_fn(pk)).decode("utf-8").upper(), + file=output, + ) + print( + "HashSk = %s" + % binascii.hexlify(hash_fn(keygen_rand)) + .decode("utf-8") + .upper(), + file=output, + ) + print( + "HashSig = %s" + % binascii.hexlify(hash_fn(signature)).decode("utf-8").upper(), + file=output, + ) print("", file=output) else: print("Usage: gen_pqc_dsa_kats.py ") return 1 -if __name__ == '__main__': - sys.exit(main()) +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/scripts/dev_tools/gen_sphincsplus_kat.py b/src/scripts/dev_tools/gen_sphincsplus_kat.py index 116043fa197..5a54b6127a1 100755 --- a/src/scripts/dev_tools/gen_sphincsplus_kat.py +++ b/src/scripts/dev_tools/gen_sphincsplus_kat.py @@ -16,6 +16,7 @@ import hashlib import binascii + class KatReader: def __init__(self, file): self.file = file @@ -28,10 +29,10 @@ def next_value(self): if line == "": return (None, None) - if line.startswith('#') or line == "\n": + if line.startswith("#") or line == "\n": continue - key, val = line.strip().split(' = ') + key, val = line.strip().split(" = ") return (key, val) @@ -42,17 +43,17 @@ def read_kats(self): key, val = self.next_value() if key is None: - return # eof + return # eof - if key not in ['count', 'seed', 'mlen', 'msg', 'pk', 'sk', 'smlen', 'sm']: + if key not in ["count", "seed", "mlen", "msg", "pk", "sk", "smlen", "sm"]: raise Exception("Unknown key %s" % (key)) - if key in ['count', 'mlen', 'smlen']: + if key in ["count", "mlen", "smlen"]: kat[key] = int(val) else: kat[key] = val - if key == 'sm': + if key == "sm": yield kat kat = {} @@ -69,12 +70,14 @@ def sha256(v): return h.digest() -def main(args = None): +def main(args=None): if args is None: args = sys.argv if len(args) < 3: - print("Usage: %s <*.rsp file> [optional: limit of KATs]") + print( + "Usage: %s <*.rsp file> [optional: limit of KATs]" + ) return 1 param = args[1] @@ -92,17 +95,21 @@ def main(args = None): cnt += 1 # Remove the input message from the end of the 'sm' field - signature = binascii.unhexlify(kat["sm"][:-kat["mlen"]*2]) + signature = binascii.unhexlify(kat["sm"][: -kat["mlen"] * 2]) print("SphincsParameterSet = %s" % param) print("seed = %s" % kat["seed"]) print("msg = %s" % kat["msg"]) print("pk = %s" % kat["pk"]) print("sk = %s" % kat["sk"]) - print("HashSig = %s" % binascii.hexlify(hash_fn(signature)).decode("utf-8").upper()) + print( + "HashSig = %s" + % binascii.hexlify(hash_fn(signature)).decode("utf-8").upper() + ) print() return 0 -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/dev_tools/gen_tls_suite_info.py b/src/scripts/dev_tools/gen_tls_suite_info.py index 7ee53f299a5..2c6691b6cab 100755 --- a/src/scripts/dev_tools/gen_tls_suite_info.py +++ b/src/scripts/dev_tools/gen_tls_suite_info.py @@ -15,118 +15,115 @@ import hashlib import optparse -def to_ciphersuite_info(code, name): - sig_and_kex = '' - cipher_and_mac = '' +def to_ciphersuite_info(code, name): + sig_and_kex = "" + cipher_and_mac = "" - with_substr = '_WITH_' + with_substr = "_WITH_" if with_substr in name: # TLS 1.2 or earlier cipher suites - (sig_and_kex,cipher_and_mac) = name.split(with_substr) + (sig_and_kex, cipher_and_mac) = name.split(with_substr) else: # TLS 1.3 cipher suites, no sig_and_kex cipher_and_mac = name - if sig_and_kex == '': + if sig_and_kex == "": # UNDEFINED means that the information is not coded in the cipher suite - sig_algo = 'UNDEFINED' - kex_algo = 'UNDEFINED' - elif sig_and_kex == 'RSA': - sig_algo = 'IMPLICIT' - kex_algo = 'RSA' - elif 'PSK' in sig_and_kex: - sig_algo = 'IMPLICIT' + sig_algo = "UNDEFINED" + kex_algo = "UNDEFINED" + elif sig_and_kex == "RSA": + sig_algo = "IMPLICIT" + kex_algo = "RSA" + elif "PSK" in sig_and_kex: + sig_algo = "IMPLICIT" kex_algo = sig_and_kex - elif 'SRP' in sig_and_kex: - srp_info = sig_and_kex.split('_') - if len(srp_info) == 2: # 'SRP_' + hash + elif "SRP" in sig_and_kex: + srp_info = sig_and_kex.split("_") + if len(srp_info) == 2: # 'SRP_' + hash kex_algo = sig_and_kex - sig_algo = 'IMPLICIT' + sig_algo = "IMPLICIT" else: - kex_algo = '_'.join(srp_info[0:-1]) + kex_algo = "_".join(srp_info[0:-1]) sig_algo = srp_info[-1] else: - (kex_algo, sig_algo) = sig_and_kex.split('_') + (kex_algo, sig_algo) = sig_and_kex.split("_") - cipher_and_mac = cipher_and_mac.split('_') + cipher_and_mac = cipher_and_mac.split("_") mac_algo = cipher_and_mac[-1] cipher = cipher_and_mac[:-1] - if mac_algo == '8' and cipher[-1] == 'CCM': + if mac_algo == "8" and cipher[-1] == "CCM": cipher = cipher[:-1] - mac_algo = 'CCM_8' - elif cipher[-2] == 'CCM' and cipher[-1] == '8': + mac_algo = "CCM_8" + elif cipher[-2] == "CCM" and cipher[-1] == "8": cipher = cipher[:-1] - mac_algo = 'CCM_8' + mac_algo = "CCM_8" - if mac_algo == 'CCM': - cipher += ['CCM'] - mac_algo = 'SHA256' - elif mac_algo == 'CCM_8': - cipher += ['CCM(8)'] - mac_algo = 'SHA256' + if mac_algo == "CCM": + cipher += ["CCM"] + mac_algo = "SHA256" + elif mac_algo == "CCM_8": + cipher += ["CCM(8)"] + mac_algo = "SHA256" cipher_info = { - 'CHACHA20': ('ChaCha',32), - 'IDEA': ('IDEA',16), - 'DES': ('DES',8), - '3DES': ('3DES',24), - 'CAMELLIA': ('Camellia',None), - 'AES': ('AES',None), - 'SEED': ('SEED',16), - 'ARIA': ('ARIA',None), - } + "CHACHA20": ("ChaCha", 32), + "IDEA": ("IDEA", 16), + "DES": ("DES", 8), + "3DES": ("3DES", 24), + "CAMELLIA": ("Camellia", None), + "AES": ("AES", None), + "SEED": ("SEED", 16), + "ARIA": ("ARIA", None), + } tls_to_botan_names = { - 'UNDEFINED': 'UNDEFINED', - 'IMPLICIT': 'IMPLICIT', - - 'anon': 'ANONYMOUS', - 'MD5': 'MD5', - 'SHA': 'SHA-1', - 'SHA256': 'SHA-256', - 'SHA384': 'SHA-384', - 'SHA512': 'SHA-512', - - 'CHACHA': 'ChaCha', - '3DES': 'TripleDES', - - 'DSS': 'DSA', - 'ECDSA': 'ECDSA', - 'RSA': 'RSA', - 'SRP_SHA': 'SRP_SHA', - 'DHE': 'DH', - 'DH': 'DH', - 'ECDHE': 'ECDH', - 'ECDH': 'ECDH', - '': '', - 'PSK': 'PSK', - 'DHE_PSK': 'DHE_PSK', - 'PSK_DHE': 'DHE_PSK', - 'ECDHE_PSK': 'ECDHE_PSK', - } + "UNDEFINED": "UNDEFINED", + "IMPLICIT": "IMPLICIT", + "anon": "ANONYMOUS", + "MD5": "MD5", + "SHA": "SHA-1", + "SHA256": "SHA-256", + "SHA384": "SHA-384", + "SHA512": "SHA-512", + "CHACHA": "ChaCha", + "3DES": "TripleDES", + "DSS": "DSA", + "ECDSA": "ECDSA", + "RSA": "RSA", + "SRP_SHA": "SRP_SHA", + "DHE": "DH", + "DH": "DH", + "ECDHE": "ECDH", + "ECDH": "ECDH", + "": "", + "PSK": "PSK", + "DHE_PSK": "DHE_PSK", + "PSK_DHE": "DHE_PSK", + "ECDHE_PSK": "ECDHE_PSK", + } mac_keylen = { - 'MD5': 16, - 'SHA-1': 20, - 'SHA-256': 32, - 'SHA-384': 48, - 'SHA-512': 64, - } + "MD5": 16, + "SHA-1": 20, + "SHA-256": 32, + "SHA-384": 48, + "SHA-512": 64, + } mac_algo = tls_to_botan_names[mac_algo] sig_algo = tls_to_botan_names[sig_algo] kex_algo = tls_to_botan_names[kex_algo] - if kex_algo == 'RSA': - kex_algo = 'STATIC_RSA' + if kex_algo == "RSA": + kex_algo = "STATIC_RSA" - if sig_algo in ['DSA']: + if sig_algo in ["DSA"]: return None - if kex_algo in ['SRP_SHA', 'DHE_PSK']: + if kex_algo in ["SRP_SHA", "DHE_PSK"]: return None (cipher_algo, cipher_keylen) = cipher_info[cipher[0]] @@ -134,100 +131,191 @@ def to_ciphersuite_info(code, name): if cipher_keylen is None: cipher_keylen = int(cipher[1]) / 8 - if cipher_algo in ['AES', 'Camellia', 'ARIA']: - cipher_algo += '-%d' % (cipher_keylen*8) - - mode = '' - - if cipher[0] == 'CHACHA20' and cipher[1] == 'POLY1305': - return (name, code, sig_algo, kex_algo, "ChaCha20Poly1305", cipher_keylen, "AEAD", 0, mac_algo, 'AEAD_XOR_12') + if cipher_algo in ["AES", "Camellia", "ARIA"]: + cipher_algo += "-%d" % (cipher_keylen * 8) + + mode = "" + + if cipher[0] == "CHACHA20" and cipher[1] == "POLY1305": + return ( + name, + code, + sig_algo, + kex_algo, + "ChaCha20Poly1305", + cipher_keylen, + "AEAD", + 0, + mac_algo, + "AEAD_XOR_12", + ) mode = cipher[-1] - if mode not in ['CBC', 'GCM', 'CCM(8)', 'CCM', 'OCB']: - print("#warning Unknown mode '%s' for ciphersuite %s (0x%d)" % (' '.join(cipher), name, code)) - - if mode != 'CBC': - if mode == 'OCB': - cipher_algo += '/OCB(12)' + if mode not in ["CBC", "GCM", "CCM(8)", "CCM", "OCB"]: + print( + "#warning Unknown mode '%s' for ciphersuite %s (0x%d)" + % (" ".join(cipher), name, code) + ) + + if mode != "CBC": + if mode == "OCB": + cipher_algo += "/OCB(12)" else: - cipher_algo += '/' + mode - - if mode == 'CBC': - return (name, code, sig_algo, kex_algo, cipher_algo, cipher_keylen, mac_algo, mac_keylen[mac_algo], mac_algo, 'CBC_MODE') - elif mode == 'OCB': - return (name, code, sig_algo, kex_algo, cipher_algo, cipher_keylen, "AEAD", 0, mac_algo, 'AEAD_XOR_12') + cipher_algo += "/" + mode + + if mode == "CBC": + return ( + name, + code, + sig_algo, + kex_algo, + cipher_algo, + cipher_keylen, + mac_algo, + mac_keylen[mac_algo], + mac_algo, + "CBC_MODE", + ) + elif mode == "OCB": + return ( + name, + code, + sig_algo, + kex_algo, + cipher_algo, + cipher_keylen, + "AEAD", + 0, + mac_algo, + "AEAD_XOR_12", + ) else: - return (name, code, sig_algo, kex_algo, cipher_algo, cipher_keylen, "AEAD", 0, mac_algo, 'AEAD_IMPLICIT_4') + return ( + name, + code, + sig_algo, + kex_algo, + cipher_algo, + cipher_keylen, + "AEAD", + 0, + mac_algo, + "AEAD_IMPLICIT_4", + ) + def open_input(args): - iana_url = 'https://www.iana.org/assignments/tls-parameters/tls-parameters.txt' + iana_url = "https://www.iana.org/assignments/tls-parameters/tls-parameters.txt" if len(args) == 1: try: - return open('tls-parameters.txt') + return open("tls-parameters.txt") except OSError: pass import urllib.request import urllib.error import urllib.parse + return urllib.request.urlopen(iana_url) else: - return open(args[1]) + return open(args[1]) + """ Handle command line options """ -def process_command_line(args): - - parser = optparse.OptionParser() - - parser.add_option('--with-ocb', action='store_true', default=True, - help='enable OCB AEAD suites') - parser.add_option('--without-ocb', action='store_false', dest='with_ocb', - help='disable OCB AEAD suites') - parser.add_option('--with-aria-cbc', action='store_true', default=False, - help='enable ARIA CBC suites') - parser.add_option('--without-aria-cbc', action='store_false', dest='with_aria_cbc', - help='disable ARIA CBC suites') - parser.add_option('--save-download', action='store_true', default=False, - help='save downloaded tls-parameters.txt to cwd') +def process_command_line(args): + parser = optparse.OptionParser() - parser.add_option('--output', '-o', - help='file to write output to (default %default)', - default='src/lib/tls/tls_suite_info.cpp') + parser.add_option( + "--with-ocb", action="store_true", default=True, help="enable OCB AEAD suites" + ) + parser.add_option( + "--without-ocb", + action="store_false", + dest="with_ocb", + help="disable OCB AEAD suites", + ) + + parser.add_option( + "--with-aria-cbc", + action="store_true", + default=False, + help="enable ARIA CBC suites", + ) + parser.add_option( + "--without-aria-cbc", + action="store_false", + dest="with_aria_cbc", + help="disable ARIA CBC suites", + ) + + parser.add_option( + "--save-download", + action="store_true", + default=False, + help="save downloaded tls-parameters.txt to cwd", + ) + + parser.add_option( + "--output", + "-o", + help="file to write output to (default %default)", + default="src/lib/tls/tls_suite_info.cpp", + ) return parser.parse_args(args) -def main(args = None): + +def main(args=None): if args is None: args = sys.argv - weak_crypto = ['EXPORT', 'RC2', 'IDEA', 'RC4', '_DES_', 'WITH_NULL', 'GOST', '_anon_'] - static_dh = ['ECDH_ECDSA', 'ECDH_RSA', 'DH_DSS', 'DH_RSA'] # not supported - removed_algos = ['SEED', 'CAMELLIA_128_CBC', 'CAMELLIA_256_CBC'] - protocol_goop = ['SCSV', 'KRB5'] - maybe_someday = ['RSA_PSK', 'ECCPWD', 'AEGIS'] - macciphersuites = ['SHA256_SHA256', 'SHA384_SHA384'] - shang_mi = ['SM4_GCM_SM3', 'SM4_CCM_SM3'] # RFC8998 - not_supported = weak_crypto + static_dh + protocol_goop + maybe_someday + removed_algos + macciphersuites + shang_mi + weak_crypto = [ + "EXPORT", + "RC2", + "IDEA", + "RC4", + "_DES_", + "WITH_NULL", + "GOST", + "_anon_", + ] + static_dh = ["ECDH_ECDSA", "ECDH_RSA", "DH_DSS", "DH_RSA"] # not supported + removed_algos = ["SEED", "CAMELLIA_128_CBC", "CAMELLIA_256_CBC"] + protocol_goop = ["SCSV", "KRB5"] + maybe_someday = ["RSA_PSK", "ECCPWD", "AEGIS"] + macciphersuites = ["SHA256_SHA256", "SHA384_SHA384"] + shang_mi = ["SM4_GCM_SM3", "SM4_CCM_SM3"] # RFC8998 + not_supported = ( + weak_crypto + + static_dh + + protocol_goop + + maybe_someday + + removed_algos + + macciphersuites + + shang_mi + ) (options, args) = process_command_line(args) if not options.with_aria_cbc: - not_supported += ['ARIA_128_CBC', 'ARIA_256_CBC'] + not_supported += ["ARIA_128_CBC", "ARIA_256_CBC"] - ciphersuite_re = re.compile(' +0x([0-9a-fA-F][0-9a-fA-F]),0x([0-9a-fA-F][0-9a-fA-F]) + TLS_([A-Za-z_0-9]+) ') + ciphersuite_re = re.compile( + " +0x([0-9a-fA-F][0-9a-fA-F]),0x([0-9a-fA-F][0-9a-fA-F]) + TLS_([A-Za-z_0-9]+) " + ) suites = {} - contents = '' + contents = "" for line in open_input(args): if not isinstance(line, str): - line = line.decode('utf8') + line = line.decode("utf8") contents += line match = ciphersuite_re.match(line) if match: @@ -239,17 +327,17 @@ def main(args = None): if ns in name: should_use = False - if should_use:# and name.find('_WITH_') > 0: + if should_use: # and name.find('_WITH_') > 0: info = to_ciphersuite_info(code, name) if info is not None: suites[code] = info sha256 = hashlib.sha256() - sha256.update(contents.encode('utf8')) + sha256.update(contents.encode("utf8")) contents_hash = sha256.hexdigest() if options.save_download: - out = open('tls-parameters.txt', 'w') + out = open("tls-parameters.txt", "w") out.write(contents) out.close() @@ -260,12 +348,12 @@ def define_custom_ciphersuite(name, code): if options.with_ocb: # OCB ciphersuites draft-zauner-tls-aes-ocb-04 - define_custom_ciphersuite('ECDHE_RSA_WITH_AES_256_OCB_SHA256', 'FFC3') - define_custom_ciphersuite('ECDHE_ECDSA_WITH_AES_256_OCB_SHA256', 'FFC5') - define_custom_ciphersuite('PSK_WITH_AES_256_OCB_SHA256', 'FFC7') - define_custom_ciphersuite('ECDHE_PSK_WITH_AES_256_OCB_SHA256', 'FFCB') + define_custom_ciphersuite("ECDHE_RSA_WITH_AES_256_OCB_SHA256", "FFC3") + define_custom_ciphersuite("ECDHE_ECDSA_WITH_AES_256_OCB_SHA256", "FFC5") + define_custom_ciphersuite("PSK_WITH_AES_256_OCB_SHA256", "FFC7") + define_custom_ciphersuite("ECDHE_PSK_WITH_AES_256_OCB_SHA256", "FFCB") - suite_info = '' + suite_info = "" def header(): return """/* @@ -297,8 +385,21 @@ def header(): info = suites[code] assert len(info) == 10 - suite_expr = 'Ciphersuite(0x%s, "%s", Auth_Method::%s, Kex_Algo::%s, "%s", %d, "%s", %d, KDF_Algo::%s, Nonce_Format::%s)' % ( - code, info[0], info[2], info[3], info[4], info[5], info[6], info[7], info[8].replace('-','_'), info[9]) + suite_expr = ( + 'Ciphersuite(0x%s, "%s", Auth_Method::%s, Kex_Algo::%s, "%s", %d, "%s", %d, KDF_Algo::%s, Nonce_Format::%s)' + % ( + code, + info[0], + info[2], + info[3], + info[4], + info[5], + info[6], + info[7], + info[8].replace("-", "_"), + info[9], + ) + ) suite_info += " " + suite_expr + ",\n" @@ -312,14 +413,15 @@ def header(): } // namespace Botan::TLS """ - if options.output == '-': + if options.output == "-": print(suite_info) else: - out = open(options.output, 'w') + out = open(options.output, "w") out.write(suite_info) out.close() return 0 -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/dev_tools/macro_checks.py b/src/scripts/dev_tools/macro_checks.py index 51a4faac100..582dae07c7c 100755 --- a/src/scripts/dev_tools/macro_checks.py +++ b/src/scripts/dev_tools/macro_checks.py @@ -10,10 +10,10 @@ import os import re -src_dir = 'src' -lib_dir = os.path.join(src_dir, 'lib') +src_dir = "src" +lib_dir = os.path.join(src_dir, "lib") -info_modules = load_info_files(lib_dir, 'Modules', "info.txt", ModuleInfo) +info_modules = load_info_files(lib_dir, "Modules", "info.txt", ModuleInfo) all_defines = set() @@ -21,23 +21,24 @@ for define in module._defines: all_defines.add(define) -extras = ['MP_DWORD', - 'VALGRIND', - 'ONLINE_REVOCATION_CHECKS', - 'SANITIZER_UNDEFINED', - 'HW_AES_SUPPORT'] +extras = [ + "MP_DWORD", + "VALGRIND", + "ONLINE_REVOCATION_CHECKS", + "SANITIZER_UNDEFINED", + "HW_AES_SUPPORT", +] for extra in extras: all_defines.add(extra) -macro = re.compile('BOTAN_HAS_([A-Za-z0-9_]+)') +macro = re.compile("BOTAN_HAS_([A-Za-z0-9_]+)") for dirname, subdirs, files in os.walk(src_dir): for fname in files: - if fname.endswith('.h') or fname.endswith('.cpp'): + if fname.endswith(".h") or fname.endswith(".cpp"): contents = open(os.path.join(dirname, fname)).read() for m in re.finditer(macro, contents): if m.group(1) not in all_defines: - print('In %s found unknown feature macro %s' % (fname, m.group(1))) - + print("In %s found unknown feature macro %s" % (fname, m.group(1))) diff --git a/src/scripts/dev_tools/run_clang_format.py b/src/scripts/dev_tools/run_clang_format.py index 01125770b12..d0c70f8db75 100755 --- a/src/scripts/dev_tools/run_clang_format.py +++ b/src/scripts/dev_tools/run_clang_format.py @@ -8,7 +8,7 @@ import subprocess import sys -import optparse # pylint: disable=deprecated-module +import optparse # pylint: disable=deprecated-module import multiprocessing import difflib import time @@ -16,65 +16,73 @@ import re from multiprocessing.pool import ThreadPool + def run_command(cmdline): - proc = subprocess.Popen(cmdline, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + proc = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = proc.communicate() - stdout = stdout.decode('utf8') - stderr = stderr.decode('utf8') + stdout = stdout.decode("utf8") + stderr = stderr.decode("utf8") return (stdout, stderr) + def apply_clang_format(clang_format, source_file): - cmdline = [clang_format, '-i', source_file] + cmdline = [clang_format, "-i", source_file] (stdout, stderr) = run_command(cmdline) - if stdout != '' or stderr != '': - print("Running '%s' stdout: '%s' stderr: '%s''" % (' '.join(cmdline), stdout, stderr)) + if stdout != "" or stderr != "": + print( + "Running '%s' stdout: '%s' stderr: '%s''" + % (" ".join(cmdline), stdout, stderr) + ) return False return True + def run_diff(source_file, formatted_contents): - original_contents = open(source_file, encoding='utf8').read() + original_contents = open(source_file, encoding="utf8").read() if original_contents == formatted_contents: - return '' + return "" + + return "\n".join( + difflib.unified_diff( + original_contents.splitlines(), + formatted_contents.splitlines(), + fromfile="%s (original)" % (source_file), + tofile="%s" % (source_file), + lineterm="", + ) + ) - return '\n'.join(difflib.unified_diff( - original_contents.splitlines(), - formatted_contents.splitlines(), - fromfile="%s (original)" % (source_file), - tofile="%s" % (source_file), - lineterm="", - )) def check_clang_format(clang_format, source_file): cmdline = [clang_format, source_file] (stdout, stderr) = run_command(cmdline) - if stderr != '': - print("Running '%s' stderr: '%s''" % (' '.join(cmdline), stderr)) + if stderr != "": + print("Running '%s' stderr: '%s''" % (" ".join(cmdline), stderr)) return False diff = run_diff(source_file, stdout) - if diff != '': + if diff != "": print(diff) return False return True + def list_source_files_in(directory): - excluded = ['pkcs11t.h', 'pkcs11f.h', 'pkcs11.h'] + excluded = ["pkcs11t.h", "pkcs11f.h", "pkcs11.h"] - for (dirpath, _, filenames) in os.walk(directory): + for dirpath, _, filenames in os.walk(directory): for filename in filenames: - if filename.endswith('.cpp') or filename.endswith('.h'): - + if filename.endswith(".cpp") or filename.endswith(".h"): if filename not in excluded: yield os.path.join(dirpath, filename) + def filter_files(files, filters): if len(filters) == 0: return files @@ -89,13 +97,16 @@ def filter_files(files, filters): return files_to_fmt + # Run clang-version -version and return the major version def clang_format_version(clang_format): - clang_format_version_re = re.compile(r'^(.* )?clang-format version ([0-9]+)\.([0-9]+)\.([0-9]+)') + clang_format_version_re = re.compile( + r"^(.* )?clang-format version ([0-9]+)\.([0-9]+)\.([0-9]+)" + ) - (stdout, stderr) = run_command([clang_format, '-version']) + (stdout, stderr) = run_command([clang_format, "-version"]) - if stderr != '': + if stderr != "": print("Error trying to get clang-format version number: '%s'" % (stderr)) return None @@ -107,17 +118,18 @@ def clang_format_version(clang_format): return int(version.group(2)) -def main(args = None): + +def main(args=None): if args is None: args = sys.argv parser = optparse.OptionParser() - parser.add_option('-j', '--jobs', action='store', type='int', default=0) - parser.add_option('--src-dir', metavar='DIR', default='src') - parser.add_option('--check', action='store_true', default=False) - parser.add_option('--clang-format-binary', metavar='PATH', default='clang-format') - parser.add_option('--skip-version-check', action='store_true', default=False) + parser.add_option("-j", "--jobs", action="store", type="int", default=0) + parser.add_option("--src-dir", metavar="DIR", default="src") + parser.add_option("--check", action="store_true", default=False) + parser.add_option("--clang-format-binary", metavar="PATH", default="clang-format") + parser.add_option("--skip-version-check", action="store_true", default=False) (options, args) = parser.parse_args(args) @@ -135,8 +147,13 @@ def main(args = None): req_version = 17 if version != req_version: - print("This script requires clang-format %d but current version is %d" % (req_version, version)) - print("Use --skip-version-check to carry on, however formatting may be incorrect") + print( + "This script requires clang-format %d but current version is %d" + % (req_version, version) + ) + print( + "Use --skip-version-check to carry on, however formatting may be incorrect" + ) return 1 jobs = options.jobs @@ -160,7 +177,7 @@ def main(args = None): return 1 # this file is incredibly slow to format so start it early - slow_to_format = ['os_utils.cpp'] + slow_to_format = ["os_utils.cpp"] first = [x for x in files_to_fmt if os.path.basename(x) in slow_to_format] rest = [x for x in files_to_fmt if x not in first] @@ -170,9 +187,25 @@ def main(args = None): results = [] for file in files_to_fmt: if options.check: - results.append(pool.apply_async(check_clang_format, (clang_format, file,))) + results.append( + pool.apply_async( + check_clang_format, + ( + clang_format, + file, + ), + ) + ) else: - results.append(pool.apply_async(apply_clang_format, (clang_format, file,))) + results.append( + pool.apply_async( + apply_clang_format, + ( + clang_format, + file, + ), + ) + ) fail_execution = False @@ -187,5 +220,6 @@ def main(args = None): return -1 if fail_execution else 0 -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/dev_tools/run_clang_tidy.py b/src/scripts/dev_tools/run_clang_tidy.py index a9b8f6c4a43..62438dcec29 100755 --- a/src/scripts/dev_tools/run_clang_tidy.py +++ b/src/scripts/dev_tools/run_clang_tidy.py @@ -10,7 +10,7 @@ import json import multiprocessing from multiprocessing.pool import ThreadPool -import optparse # pylint: disable=deprecated-module +import optparse # pylint: disable=deprecated-module import os import random import re @@ -22,127 +22,132 @@ import uuid enabled_checks = [ - 'bugprone-*', - 'cert-*', - 'clang-analyzer-*', - 'cppcoreguidelines-*', - 'hicpp-*', - 'misc-*', - 'modernize-*', - 'performance-*', - 'portability-*', - 'readability-*', + "bugprone-*", + "cert-*", + "clang-analyzer-*", + "cppcoreguidelines-*", + "hicpp-*", + "misc-*", + "modernize-*", + "performance-*", + "portability-*", + "readability-*", ] # these are ones that we might want to be clean for in the future, # but currently are not disabled_needs_work = [ - '*-named-parameter', - 'cppcoreguidelines-use-default-member-init', - 'bugprone-unchecked-optional-access', - 'bugprone-empty-catch', - 'cppcoreguidelines-avoid-const-or-ref-data-members', - 'misc-const-correctness', # pretty noisy - 'misc-include-cleaner', - 'modernize-pass-by-value', - 'modernize-use-ranges', # limited by compiler support currently - 'performance-avoid-endl', - 'readability-convert-member-functions-to-static', - 'readability-inconsistent-declaration-parameter-name', # should fix this, blocked by https://github.com/llvm/llvm-project/issues/60845 - 'readability-simplify-boolean-expr', # sometimes ok - 'readability-static-accessed-through-instance', + "*-named-parameter", + "cppcoreguidelines-use-default-member-init", + "bugprone-unchecked-optional-access", + "bugprone-empty-catch", + "cppcoreguidelines-avoid-const-or-ref-data-members", + "misc-const-correctness", # pretty noisy + "misc-include-cleaner", + "modernize-pass-by-value", + "modernize-use-ranges", # limited by compiler support currently + "performance-avoid-endl", + "readability-convert-member-functions-to-static", + "readability-inconsistent-declaration-parameter-name", # should fix this, blocked by https://github.com/llvm/llvm-project/issues/60845 + "readability-simplify-boolean-expr", # sometimes ok + "readability-static-accessed-through-instance", ] # these we are probably not interested in ever being clang-tidy clean for disabled_not_interested = [ - '*-array-to-pointer-decay', - '*-avoid-c-arrays', # triggers also on foo(T x[], size_t len) decls - '*-else-after-return', - '*-function-size', - '*-magic-numbers', # can't stop the magic - '*-narrowing-conversions', - '*-no-array-decay', - '*-use-auto', # not universally a good idea - '*-use-emplace', # often less clear - '*-deprecated-headers', # wrong for system headers like stdlib.h - 'cert-dcl21-cpp', # invalid, and removed already in clang-tidy 19 - 'bugprone-easily-swappable-parameters', - 'bugprone-implicit-widening-of-multiplication-result', - 'cppcoreguidelines-pro-bounds-pointer-arithmetic', - 'cppcoreguidelines-pro-bounds-constant-array-index', - 'cppcoreguidelines-pro-type-const-cast', # see above - 'cppcoreguidelines-pro-type-reinterpret-cast', # not possible thanks though - 'hicpp-no-assembler', - 'hicpp-signed-bitwise', # impossible to avoid in C/C++, int promotion rules :/ - 'misc-no-recursion', - 'modernize-use-trailing-return-type', # fine, but we're not using it everywhere - 'modernize-return-braced-init-list', # thanks I hate it - 'modernize-use-default-member-init', - 'modernize-use-designated-initializers', - 'modernize-use-nodiscard', - 'modernize-use-using', # fine not great - 'portability-simd-intrinsics', - 'readability-avoid-return-with-void-value', # Jack likes doing this - 'readability-function-cognitive-complexity', - 'readability-identifier-length', # lol, lmao - 'readability-math-missing-parentheses', - 'readability-non-const-parameter', - 'readability-redundant-inline-specifier', # Jack likes doing this - 'readability-redundant-access-specifiers', # reneme likes doing this - 'readability-use-anyofallof', # not more readable + "*-array-to-pointer-decay", + "*-avoid-c-arrays", # triggers also on foo(T x[], size_t len) decls + "*-else-after-return", + "*-function-size", + "*-magic-numbers", # can't stop the magic + "*-narrowing-conversions", + "*-no-array-decay", + "*-use-auto", # not universally a good idea + "*-use-emplace", # often less clear + "*-deprecated-headers", # wrong for system headers like stdlib.h + "cert-dcl21-cpp", # invalid, and removed already in clang-tidy 19 + "bugprone-easily-swappable-parameters", + "bugprone-implicit-widening-of-multiplication-result", + "cppcoreguidelines-pro-bounds-pointer-arithmetic", + "cppcoreguidelines-pro-bounds-constant-array-index", + "cppcoreguidelines-pro-type-const-cast", # see above + "cppcoreguidelines-pro-type-reinterpret-cast", # not possible thanks though + "hicpp-no-assembler", + "hicpp-signed-bitwise", # impossible to avoid in C/C++, int promotion rules :/ + "misc-no-recursion", + "modernize-use-trailing-return-type", # fine, but we're not using it everywhere + "modernize-return-braced-init-list", # thanks I hate it + "modernize-use-default-member-init", + "modernize-use-designated-initializers", + "modernize-use-nodiscard", + "modernize-use-using", # fine not great + "portability-simd-intrinsics", + "readability-avoid-return-with-void-value", # Jack likes doing this + "readability-function-cognitive-complexity", + "readability-identifier-length", # lol, lmao + "readability-math-missing-parentheses", + "readability-non-const-parameter", + "readability-redundant-inline-specifier", # Jack likes doing this + "readability-redundant-access-specifiers", # reneme likes doing this + "readability-use-anyofallof", # not more readable ] disabled_checks = sorted(disabled_needs_work + disabled_not_interested) + def create_check_option(enabled, disabled): - return ','.join(enabled) + ',' + ','.join(['-' + d for d in disabled]) + return ",".join(enabled) + "," + ",".join(["-" + d for d in disabled]) + def render_clang_tidy_file(target_dir, enabled, disabled): - filepath = os.path.join(target_dir, '.clang-tidy') - print(f'regenerating {filepath}') + filepath = os.path.join(target_dir, ".clang-tidy") + print(f"regenerating {filepath}") with open(filepath, "w", encoding="utf-8") as clang_tidy_file: - clang_tidy_file.writelines([ - '---\n', - f'# This file was automatically generated by {sys.argv[0]} --regenerate-inline-config-file\n', - '#\n', - '# All manual edits to this file will be lost. Edit the script\n', - '# then regenerate this configuration file.\n', - '\n', - 'Checks: >\n'] + - [ f' {check},\n' for check in sorted(enabled)] + - [ f' -{check},\n' for check in sorted(disabled)] + - ['---\n']) + clang_tidy_file.writelines( + [ + "---\n", + f"# This file was automatically generated by {sys.argv[0]} --regenerate-inline-config-file\n", + "#\n", + "# All manual edits to this file will be lost. Edit the script\n", + "# then regenerate this configuration file.\n", + "\n", + "Checks: >\n", + ] + + [f" {check},\n" for check in sorted(enabled)] + + [f" -{check},\n" for check in sorted(disabled)] + + ["---\n"] + ) + def load_compile_commands(build_dir): - compile_commands_file = os.path.join(build_dir, 'compile_commands.json') - compile_commands = open(compile_commands_file, encoding='utf8').read() + compile_commands_file = os.path.join(build_dir, "compile_commands.json") + compile_commands = open(compile_commands_file, encoding="utf8").read() return (compile_commands_file, json.loads(compile_commands)) + def run_command(cmdline): - proc = subprocess.Popen(cmdline, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + proc = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, _stderr) = proc.communicate() - stdout = stdout.decode('utf8') + stdout = stdout.decode("utf8") # stderr is discarded return stdout -def run_clang_tidy(compile_commands_file, - check_config, - source_file, - options): - cmdline = ['clang-tidy', - '--quiet', - '-checks=%s' % (check_config), - '-p', compile_commands_file, - '-header-filter=.*'] +def run_clang_tidy(compile_commands_file, check_config, source_file, options): + cmdline = [ + "clang-tidy", + "--quiet", + "-checks=%s" % (check_config), + "-p", + compile_commands_file, + "-header-filter=.*", + ] if options.fixit: - cmdline.append('-fix') + cmdline.append("-fix") if options.export_fixes_dir: os.makedirs(options.export_fixes_dir, exist_ok=True) @@ -161,6 +166,7 @@ def run_clang_tidy(compile_commands_file, return True + def file_matches(file, args): if args is None or len(args) == 0: return True @@ -170,42 +176,44 @@ def file_matches(file, args): return True return False + def preproc_file(compile_commands): - cmd = shlex.split(compile_commands['command']) + cmd = shlex.split(compile_commands["command"]) dash_o = cmd.index("-o") if dash_o >= 0: - cmd.pop(dash_o + 1) # remove object file name - cmd.pop(dash_o) # remove -o + cmd.pop(dash_o + 1) # remove object file name + cmd.pop(dash_o) # remove -o if "-c" in cmd: cmd.remove("-c") cmd.append("-E") result = subprocess.run( - cmd, - check=True, - stdout=subprocess.PIPE, - universal_newlines=True) + cmd, check=True, stdout=subprocess.PIPE, universal_newlines=True + ) + + return hashlib.sha256(result.stdout.encode("utf-8")).hexdigest() - return hashlib.sha256(result.stdout.encode('utf-8')).hexdigest() def hash_args(**kwargs): outer_hash = hashlib.sha256() for key in sorted(kwargs.keys()): value = kwargs[key] - outer_hash.update(hashlib.sha256(key.encode('utf-8')).digest()) - outer_hash.update(hashlib.sha256(value.encode('utf-8')).digest()) + outer_hash.update(hashlib.sha256(key.encode("utf-8")).digest()) + outer_hash.update(hashlib.sha256(value.encode("utf-8")).digest()) return outer_hash.hexdigest() + class CacheDatabase: """ Caches SUCCESSFUL clang-tidy runs on source files to speed up whole-tree runs of clang-tidy. The cache key is calculated as the hash of the source file and a set of meta information such as 'clang-tidy --version' and the check configuration. """ + CACHE_DEBUG = False def __init__(self, db_path): @@ -218,7 +226,7 @@ def hit(self, **kwargs): cache_key = hash_args(**kwargs) cur = self.db.cursor() - cur.execute("SELECT key from clang_tidy_clean where key = ?", (cache_key, )) + cur.execute("SELECT key from clang_tidy_clean where key = ?", (cache_key,)) res = cur.fetchall() if len(res) > 0: @@ -236,7 +244,7 @@ def record(self, **kwargs): cur = self.db.cursor() if self.CACHE_DEBUG: print("Cache save for", cache_key) - cur.execute("INSERT OR IGNORE INTO clang_tidy_clean values(?)", (cache_key, )) + cur.execute("INSERT OR IGNORE INTO clang_tidy_clean values(?)", (cache_key,)) self.db.commit() def count(self): @@ -262,29 +270,41 @@ def cleanup(self, limit): for cache_key in entries[:to_prune]: if self.CACHE_DEBUG: print("Pruning", cache_key[0]) - cur.execute("DELETE FROM clang_tidy_clean WHERE key = ?", (cache_key[0], )) + cur.execute("DELETE FROM clang_tidy_clean WHERE key = ?", (cache_key[0],)) self.db.commit() -def main(args = None): # pylint: disable=too-many-return-statements + +def main(args=None): # pylint: disable=too-many-return-statements if args is None: args = sys.argv parser = optparse.OptionParser() - parser.add_option('-j', '--jobs', action='store', type='int', default=multiprocessing.cpu_count() + 1, - help='set number of jobs to run (default %default)') - parser.add_option('--verbose', action='store_true', default=False) - parser.add_option('--fixit', action='store_true', default=False) - parser.add_option('--build-dir', default='build') - parser.add_option('--list-checks', action='store_true', default=False) - parser.add_option('--regenerate-inline-config-file', action='store_true', default=False) - parser.add_option('--only-changed-files', action='store_true', default=False) - parser.add_option('--only-matching', metavar='REGEX', default='.*') - parser.add_option('--take-file-list-from-stdin', action='store_true', default=False) - parser.add_option('--export-fixes-dir', default=None) - parser.add_option('--cache-db', metavar='DB', - help='Path to sqlite3 database file for caching', - default=os.getenv("BOTAN_CLANG_TIDY_CACHE")) + parser.add_option( + "-j", + "--jobs", + action="store", + type="int", + default=multiprocessing.cpu_count() + 1, + help="set number of jobs to run (default %default)", + ) + parser.add_option("--verbose", action="store_true", default=False) + parser.add_option("--fixit", action="store_true", default=False) + parser.add_option("--build-dir", default="build") + parser.add_option("--list-checks", action="store_true", default=False) + parser.add_option( + "--regenerate-inline-config-file", action="store_true", default=False + ) + parser.add_option("--only-changed-files", action="store_true", default=False) + parser.add_option("--only-matching", metavar="REGEX", default=".*") + parser.add_option("--take-file-list-from-stdin", action="store_true", default=False) + parser.add_option("--export-fixes-dir", default=None) + parser.add_option( + "--cache-db", + metavar="DB", + help="Path to sqlite3 database file for caching", + default=os.getenv("BOTAN_CLANG_TIDY_CACHE"), + ) # 100K entries with our current schema is about 15 Mb max_db_entries = int(os.getenv("BOTAN_CLANG_TIDY_CACHE_MAX_SIZE") or 100_000) @@ -296,16 +316,19 @@ def main(args = None): # pylint: disable=too-many-return-statements return 1 if options.only_changed_files and options.take_file_list_from_stdin: - print("Cannot use both --only-changed-files and --take-file-list-from-stdin", file=sys.stderr) + print( + "Cannot use both --only-changed-files and --take-file-list-from-stdin", + file=sys.stderr, + ) return 1 files_to_check = [] if options.only_changed_files: - changes = run_command(['git', 'diff', '--name-only', '-r', 'master']) + changes = run_command(["git", "diff", "--name-only", "-r", "master"]) files_to_check = [] for file in changes.split(): - if file.endswith('.cpp') or file.endswith('.h'): + if file.endswith(".cpp") or file.endswith(".h"): files_to_check.append(os.path.basename(file)) if len(files_to_check) == 0: @@ -316,9 +339,9 @@ def main(args = None): # pylint: disable=too-many-return-statements for line in sys.stdin: file = os.path.basename(line.strip()) - if file.endswith('.cpp') or file.endswith('.h'): + if file.endswith(".cpp") or file.endswith(".h"): files_to_check.append(file) - elif file in ['run_clang_tidy.py']: + elif file in ["run_clang_tidy.py"]: scan_all = True if scan_all: @@ -333,11 +356,13 @@ def main(args = None): # pylint: disable=too-many-return-statements check_config = create_check_option(enabled_checks, disabled_checks) if options.list_checks: - print(run_command(['clang-tidy', '-list-checks', '-checks', check_config]), end='') + print( + run_command(["clang-tidy", "-list-checks", "-checks", check_config]), end="" + ) return 0 if options.regenerate_inline_config_file: - render_clang_tidy_file('src', enabled_checks, disabled_checks) + render_clang_tidy_file("src", enabled_checks, disabled_checks) return 0 pool = ThreadPool(options.jobs) @@ -353,30 +378,35 @@ def main(args = None): # pylint: disable=too-many-return-statements if cache is not None: if options.verbose: - print("Using cache at %s with %d entries" % (options.cache_db, cache.count())) + print( + "Using cache at %s with %d entries" % (options.cache_db, cache.count()) + ) results = [] - clang_tidy_version = subprocess.run(['clang-tidy', '--version'], - check=True, - stdout=subprocess.PIPE).stdout.decode('utf-8') + clang_tidy_version = subprocess.run( + ["clang-tidy", "--version"], check=True, stdout=subprocess.PIPE + ).stdout.decode("utf-8") for info in compile_commands: - results.append(pool.apply_async(preproc_file, (info, ))) + results.append(pool.apply_async(preproc_file, (info,))) - assert(len(results) == len(compile_commands)) - for (i, result) in enumerate(results): - compile_commands[i]['preproc'] = result.get() + assert len(results) == len(compile_commands) + for i, result in enumerate(results): + compile_commands[i]["preproc"] = result.get() if options.verbose: - print("Preprocessing/hashing %d files took %.02f sec" % (len(results), time.time() - start_time)) + print( + "Preprocessing/hashing %d files took %.02f sec" + % (len(results), time.time() - start_time) + ) sys.stdout.flush() start_time = time.time() results = [] for info in compile_commands: - file = info['file'] + file = info["file"] if len(files_to_check) > 0 and os.path.basename(file) not in files_to_check: continue @@ -386,7 +416,11 @@ def main(args = None): # pylint: disable=too-many-return-statements cache_hit = False if cache is not None: - if cache.hit(config=check_config, source_file=info['preproc'], clang_tidy=clang_tidy_version): + if cache.hit( + config=check_config, + source_file=info["preproc"], + clang_tidy=clang_tidy_version, + ): cache_hit = True if cache_hit and options.verbose: @@ -396,23 +430,30 @@ def main(args = None): # pylint: disable=too-many-return-statements if not cache_hit: files_checked += 1 - results.append((info, pool.apply_async( - run_clang_tidy, - (compile_commands_file, - check_config, - file, - options)))) + results.append( + ( + info, + pool.apply_async( + run_clang_tidy, + (compile_commands_file, check_config, file, options), + ), + ) + ) fail_cnt = 0 - for (info, result) in results: + for info, result in results: success = result.get() if options.verbose: - print("Checked", info['file']) + print("Checked", info["file"]) sys.stdout.flush() if success and cache is not None: - cache.record(config=check_config, source_file=info['preproc'], clang_tidy=clang_tidy_version) + cache.record( + config=check_config, + source_file=info["preproc"], + clang_tidy=clang_tidy_version, + ) if not success: fail_cnt += 1 @@ -430,5 +471,6 @@ def main(args = None): # pylint: disable=too-many-return-statements print("Found clang-tidy errors in %d files" % (fail_cnt)) return 1 -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/dev_tools/show_dependencies.py b/src/scripts/dev_tools/show_dependencies.py index 28d3ed60ec9..704ea3ca52f 100755 --- a/src/scripts/dev_tools/show_dependencies.py +++ b/src/scripts/dev_tools/show_dependencies.py @@ -25,42 +25,55 @@ # locale sys.path.append(botan_root) -from configure import ModuleInfo # noqa: E402 - -parser = argparse.ArgumentParser(description= - 'Show Botan module dependencies. ' - 'The output is reduced by indirect dependencies, ' - 'i.e. you must look at the result recursively to get all dependencies.') - -parser.add_argument('mode', - choices=["list", "draw"], - help='The output mode') -parser.add_argument('--format', - nargs='?', - choices=["pdf", "png"], - default="pdf", - help='The file format (drawing mode only)') -parser.add_argument('--engine', - nargs='?', - choices=["fdp", "dot"], - default="dot", - help='The graph engine (drawing mode only)') -parser.add_argument('--all', dest='all', action='store_const', - const=True, default=False, - help='Show all dependencies. Default: direct dependencies only. (list mode only)') -parser.add_argument('--verbose', dest='verbose', action='store_const', - const=True, default=False, - help='Verbose output (default: false)') +from configure import ModuleInfo # noqa: E402 + +parser = argparse.ArgumentParser( + description="Show Botan module dependencies. " + "The output is reduced by indirect dependencies, " + "i.e. you must look at the result recursively to get all dependencies." +) + +parser.add_argument("mode", choices=["list", "draw"], help="The output mode") +parser.add_argument( + "--format", + nargs="?", + choices=["pdf", "png"], + default="pdf", + help="The file format (drawing mode only)", +) +parser.add_argument( + "--engine", + nargs="?", + choices=["fdp", "dot"], + default="dot", + help="The graph engine (drawing mode only)", +) +parser.add_argument( + "--all", + dest="all", + action="store_const", + const=True, + default=False, + help="Show all dependencies. Default: direct dependencies only. (list mode only)", +) +parser.add_argument( + "--verbose", + dest="verbose", + action="store_const", + const=True, + default=False, + help="Verbose output (default: false)", +) args = parser.parse_args() files = [] -files += glob.glob(botan_root + '/src/lib/*/*/*/*/*/*/info.txt') -files += glob.glob(botan_root + '/src/lib/*/*/*/*/*/info.txt') -files += glob.glob(botan_root + '/src/lib/*/*/*/*/info.txt') -files += glob.glob(botan_root + '/src/lib/*/*/*/info.txt') -files += glob.glob(botan_root + '/src/lib/*/*/info.txt') -files += glob.glob(botan_root + '/src/lib/*/info.txt') -files += glob.glob(botan_root + '/src/lib/info.txt') +files += glob.glob(botan_root + "/src/lib/*/*/*/*/*/*/info.txt") +files += glob.glob(botan_root + "/src/lib/*/*/*/*/*/info.txt") +files += glob.glob(botan_root + "/src/lib/*/*/*/*/info.txt") +files += glob.glob(botan_root + "/src/lib/*/*/*/info.txt") +files += glob.glob(botan_root + "/src/lib/*/*/info.txt") +files += glob.glob(botan_root + "/src/lib/*/info.txt") +files += glob.glob(botan_root + "/src/lib/info.txt") files.sort() if len(files) == 0: @@ -69,15 +82,19 @@ modules = [] -def dicts(t): return {k: dicts(t[k]) for k in t} -def paths(t, path = [], level=0): - ret = [] +def dicts(t): + return {k: dicts(t[k]) for k in t} + + +def paths(t, path=[], level=0): + ret = [] for key in t: ret.append(path + [key]) - ret += paths(t[key], path + [key], level+1) + ret += paths(t[key], path + [key], level + 1) return ret + if args.verbose: print("Getting dependencies from into.txt files ...") @@ -92,7 +109,7 @@ def paths(t, path = [], level=0): if args.verbose: print(str(len(modules)) + " modules:") - names=[m.basename for m in modules] + names = [m.basename for m in modules] names.sort() print(names) print("") @@ -100,9 +117,11 @@ def paths(t, path = [], level=0): if args.verbose: print("resolving dependencies ...") + def cartinality(depdict): return sum([len(depdict[k]) for k in depdict]) + registered_dependencies = dict() all_dependencies = dict() direct_dependencies = dict() @@ -111,6 +130,7 @@ def cartinality(depdict): lst = module.dependencies(None) registered_dependencies[module.basename] = set(lst) - set([module.basename]) + # Get all_dependencies from registered_dependencies def add_dependency(): for key in all_dependencies: @@ -118,7 +138,9 @@ def add_dependency(): new_modules_for_key = None for currently_in in all_dependencies[key]: if currently_in in all_dependencies: - potentially_new_modules_for_key = all_dependencies[currently_in] - set([key]) + potentially_new_modules_for_key = all_dependencies[currently_in] - set( + [key] + ) if not potentially_new_modules_for_key <= all_dependencies[key]: new_modules_for_key = potentially_new_modules_for_key.copy() break @@ -134,7 +156,7 @@ def add_dependency(): all_dependencies = OrderedDict(sorted(all_dependencies.items())) direct_dependencies = OrderedDict(sorted(direct_dependencies.items())) -#print(direct_dependencies) +# print(direct_dependencies) last_card = -1 while True: @@ -145,6 +167,7 @@ def add_dependency(): last_card = card add_dependency() + # Return true iff a depends on b, # i.e. b is in the dependencies of a def depends_on(a, b): @@ -153,6 +176,7 @@ def depends_on(a, b): else: return b in direct_dependencies[a] + def remove_indirect_dependencies(): for mod in direct_dependencies: for one in direct_dependencies[mod]: @@ -163,6 +187,7 @@ def remove_indirect_dependencies(): return # Go to next mod + last_card = -1 while True: card = cartinality(direct_dependencies) @@ -172,14 +197,16 @@ def remove_indirect_dependencies(): last_card = card remove_indirect_dependencies() + def openfile(f): # pylint: disable=no-member # os.startfile is available on Windows only - if sys.platform.startswith('linux'): + if sys.platform.startswith("linux"): subprocess.call(["xdg-open", f]) else: os.startfile(f) + if args.verbose: print("Done resolving dependencies.") @@ -198,7 +225,7 @@ def openfile(f): tmpdir = tempfile.mkdtemp(prefix="botan-") g2 = gv.Digraph(format=args.format, engine=args.engine) - g2.attr('graph', rankdir='RL') # draw horizontally + g2.attr("graph", rankdir="RL") # draw horizontally for key in direct_dependencies: g2.node(key) for dep in direct_dependencies[key]: @@ -206,7 +233,7 @@ def openfile(f): if args.verbose: print("Rendering graph ...") - filename = g2.render(filename='graph', directory=tmpdir) + filename = g2.render(filename="graph", directory=tmpdir) if args.verbose: print("Opening " + filename + " ...") diff --git a/src/scripts/dev_tools/xmss_test_vector_from_reference.py b/src/scripts/dev_tools/xmss_test_vector_from_reference.py index 1548b86c76c..64481678d4e 100755 --- a/src/scripts/dev_tools/xmss_test_vector_from_reference.py +++ b/src/scripts/dev_tools/xmss_test_vector_from_reference.py @@ -19,13 +19,12 @@ async def run(cmd, args): - vc = await asyncio.create_subprocess_exec(cmd, *args, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE) + vc = await asyncio.create_subprocess_exec( + cmd, *args, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) (stdout, stderr) = await vc.communicate() if stderr: - raise RuntimeError( - "Process execution failed with: " + stderr.decode("utf-8")) + raise RuntimeError("Process execution failed with: " + stderr.decode("utf-8")) return stdout @@ -40,7 +39,13 @@ def fromhex(s): KEYGEN_UTIL = "ui/xmss_keypair" SIGN_UTIL = "ui/xmss_sign" -TEST_MESSAGES = [b'', fromhex("01020304"), fromhex("f1cceaeaae1838a11e8f9244ba16387663a38f661e160d7ded41a5d535066732b28f101412489dc73d6206ca43976dfee50faa23862b6defff6a873cad75ac069670e6203e970cada047cd10a3d3a5a2d4fb05c4d68ac3b88b7760cef22075504ab2808e175b54dff1659da07581ae7da0f287e18bcfc31bccf9ebb7ccb61a1321b3f0da52050d7220a291f94c71db4e9d315510372bb0be8362e156363ccf10903dc7b3fd6a6816e0a3c1ee2a79cbc683805aa7ff9346c977cdb7eddd1eb6c4b2686007f75a339a27ea25e5092ff01eee99a5241d43b548efdd667aa5171d5fc4089b5273840384b2ef390e56736263df23533f5f8330b53aabb68c24ddfed9aaafdd5679adcb877e5f0e7270cbcd7d3938136fa6cf038e27bdf03825a63917693d8b3e653950fc5059bda02e8c7c5f457d86ef684138028d18044c277c23fdab00491866ff354f2ec6722f56d4ce9f2ecef50b2f4f2a85c55b6eb6dc5d66b13e67b87b0071a5b2e4bc7ab92757a683867326ce18dba8ff2beb7bbbaa314e65953e861c8b10bce481f607fad0f690a9c0eb4c0155917707a02db1d22d88f2a14584f10ab13746fed752a4b7f62ebc85b34360d8ab964e280bd96e51b32ee6a589c62b63f42d7babfe6c7da6324ffb6f1dff4f6df586b1d2d34e23e3be915631bf143537268fb50ed13fff1a91856a451e91debc9fc337f666089b8e86efc410e97f1591ee0162d26fda6e6212b97886b96422b92b1220a6b6134287755ac3d12d2b96854a3e9e9cbc2477db2c3462695675ef8ab08a4968b79f6851a98b5f5056ecdb710864422cddb3d974451f66904377ab2056d6ebf7e398dcd075852537507c85a84ba14ee4c0aeb50c3bcb40f22e904eb7dbbda856b6556ab1e005e41eaf372f050c29f76f92c21647dfcbb40aad9f92201108d1ff3153163924c99cb6296f7e179779c8764627bed676e22c98b61793275da8f6f25b0122fc309c4a3f174816d90af39d0d15b457df37b8c00227655f9b3a3d2614bb9f2d1b273f67b2f92c182cd9bd64aef8038d0ccf6826083fe84135b4de6111e02909acdf175b1defcd11b24d01e50530aaa835dfdc5a6f237dbdc6ee3c3796d95306ae63b4401740b17ae5006a0bd4bf76bbe97df40a565a9483d99c0e8b17bd96dd72721da6b9d75323c262006d70ecca750caf81072151dbe43189cff3ea254d78b5fdf83530c1d937b3374fff41c3fd26c0f3124e38d9747e7213cf3daeaddf31b0ac7c72c6b04eb8ef17da3e9e93d09435ba8d707ea4c12576aa8ef23c760162645a30ec557b7688e8ede33d116ef39c9ee89865e2cbb854d35570fd92a7fc41ed95809e1a9c2d79ed79855505ac99d26c56d70b824bd098622cb5c704e70e7f281f215c3eb059edb8ce13731e")] +TEST_MESSAGES = [ + b"", + fromhex("01020304"), + fromhex( + "f1cceaeaae1838a11e8f9244ba16387663a38f661e160d7ded41a5d535066732b28f101412489dc73d6206ca43976dfee50faa23862b6defff6a873cad75ac069670e6203e970cada047cd10a3d3a5a2d4fb05c4d68ac3b88b7760cef22075504ab2808e175b54dff1659da07581ae7da0f287e18bcfc31bccf9ebb7ccb61a1321b3f0da52050d7220a291f94c71db4e9d315510372bb0be8362e156363ccf10903dc7b3fd6a6816e0a3c1ee2a79cbc683805aa7ff9346c977cdb7eddd1eb6c4b2686007f75a339a27ea25e5092ff01eee99a5241d43b548efdd667aa5171d5fc4089b5273840384b2ef390e56736263df23533f5f8330b53aabb68c24ddfed9aaafdd5679adcb877e5f0e7270cbcd7d3938136fa6cf038e27bdf03825a63917693d8b3e653950fc5059bda02e8c7c5f457d86ef684138028d18044c277c23fdab00491866ff354f2ec6722f56d4ce9f2ecef50b2f4f2a85c55b6eb6dc5d66b13e67b87b0071a5b2e4bc7ab92757a683867326ce18dba8ff2beb7bbbaa314e65953e861c8b10bce481f607fad0f690a9c0eb4c0155917707a02db1d22d88f2a14584f10ab13746fed752a4b7f62ebc85b34360d8ab964e280bd96e51b32ee6a589c62b63f42d7babfe6c7da6324ffb6f1dff4f6df586b1d2d34e23e3be915631bf143537268fb50ed13fff1a91856a451e91debc9fc337f666089b8e86efc410e97f1591ee0162d26fda6e6212b97886b96422b92b1220a6b6134287755ac3d12d2b96854a3e9e9cbc2477db2c3462695675ef8ab08a4968b79f6851a98b5f5056ecdb710864422cddb3d974451f66904377ab2056d6ebf7e398dcd075852537507c85a84ba14ee4c0aeb50c3bcb40f22e904eb7dbbda856b6556ab1e005e41eaf372f050c29f76f92c21647dfcbb40aad9f92201108d1ff3153163924c99cb6296f7e179779c8764627bed676e22c98b61793275da8f6f25b0122fc309c4a3f174816d90af39d0d15b457df37b8c00227655f9b3a3d2614bb9f2d1b273f67b2f92c182cd9bd64aef8038d0ccf6826083fe84135b4de6111e02909acdf175b1defcd11b24d01e50530aaa835dfdc5a6f237dbdc6ee3c3796d95306ae63b4401740b17ae5006a0bd4bf76bbe97df40a565a9483d99c0e8b17bd96dd72721da6b9d75323c262006d70ecca750caf81072151dbe43189cff3ea254d78b5fdf83530c1d937b3374fff41c3fd26c0f3124e38d9747e7213cf3daeaddf31b0ac7c72c6b04eb8ef17da3e9e93d09435ba8d707ea4c12576aa8ef23c760162645a30ec557b7688e8ede33d116ef39c9ee89865e2cbb854d35570fd92a7fc41ed95809e1a9c2d79ed79855505ac99d26c56d70b824bd098622cb5c704e70e7f281f215c3eb059edb8ce13731e" + ), +] class Keypair: @@ -69,23 +74,38 @@ def __init__(self, xmss_params, keypair): # transform the public/private keys into the format Botan expects sk_slice = self.keypair.find(self.oid, 4) self.public_key = self.keypair[:sk_slice] - secret_portion = self.keypair[sk_slice+4:-len(self.public_key)+4] + secret_portion = self.keypair[sk_slice + 4 : -len(self.public_key) + 4] n = int((len(secret_portion) - 4) / 2) self.idx = secret_portion[0:4] - self.secret_seed = secret_portion[4:n+4] - self.secret_prf = secret_portion[n+4:] - self.public_seed = self.public_key[n+4:n*2+4] - wots_derivation_method = "\x02" # WOTS+ derivation as described in NIST SP.800-208 - self.secret_key = self.public_key + self.idx + self.secret_prf + self.secret_seed + wots_derivation_method + self.secret_seed = secret_portion[4 : n + 4] + self.secret_prf = secret_portion[n + 4 :] + self.public_seed = self.public_key[n + 4 : n * 2 + 4] + wots_derivation_method = ( + "\x02" # WOTS+ derivation as described in NIST SP.800-208 + ) + self.secret_key = ( + self.public_key + + self.idx + + self.secret_prf + + self.secret_seed + + wots_derivation_method + ) async def sign(self, message): - with tempfile.NamedTemporaryFile() as msg_file, tempfile.NamedTemporaryFile() as key_file: + with ( + tempfile.NamedTemporaryFile() as msg_file, + tempfile.NamedTemporaryFile() as key_file, + ): key_file.write(self.keypair) key_file.flush() msg_file.write(message) msg_file.flush() signature_and_msg = await run(SIGN_UTIL, [key_file.name, msg_file.name]) - return signature_and_msg[:-len(message)] if len(message) != 0 else signature_and_msg + return ( + signature_and_msg[: -len(message)] + if len(message) != 0 + else signature_and_msg + ) async def join(strings): @@ -130,14 +150,17 @@ async def make_xmss_keygen_vec_entry(xmss_param): if len(sys.argv) < 3: - print("Usage: {} ".format( - sys.argv[0])) + print( + "Usage: {} ".format( + sys.argv[0] + ) + ) sys.exit(1) funs = { "sig": make_xmss_sig_vec_entry, "verify": make_xmss_verify_vec_entry, - "keygen": make_xmss_keygen_vec_entry + "keygen": make_xmss_keygen_vec_entry, } tv = sys.argv[1] @@ -149,4 +172,5 @@ async def make_xmss_keygen_vec_entry(xmss_param): async def main(): print(await join([funs[tv](p) for p in sys.argv[2:]])) + asyncio.run(main()) diff --git a/src/scripts/dist.py b/src/scripts/dist.py index b0c7e5b191d..c04b0f1fd5e 100755 --- a/src/scripts/dist.py +++ b/src/scripts/dist.py @@ -15,7 +15,7 @@ import hashlib import io import logging -import optparse # pylint: disable=deprecated-module +import optparse # pylint: disable=deprecated-module import os import re import shutil @@ -30,52 +30,59 @@ GZIP_HEADER_TIME = 0 + def fake_time(): return GZIP_HEADER_TIME + + time.time = fake_time def check_subprocess_results(subproc, name): (raw_stdout, raw_stderr) = subproc.communicate() - stderr = raw_stderr.decode('utf-8') + stderr = raw_stderr.decode("utf-8") if subproc.returncode != 0: - stdout = raw_stdout.decode('utf-8') - if stdout != '': + stdout = raw_stdout.decode("utf-8") + if stdout != "": logging.error(stdout) - if stderr != '': + if stderr != "": logging.error(stderr) - raise Exception('Running %s failed' % (name)) + raise Exception("Running %s failed" % (name)) - if stderr != '': + if stderr != "": logging.warning(stderr) return raw_stdout + def run_git(args): - cmd = ['git'] + args - logging.debug('Running %s', ' '.join(cmd)) + cmd = ["git"] + args + logging.debug("Running %s", " ".join(cmd)) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - return check_subprocess_results(proc, 'git') + return check_subprocess_results(proc, "git") + def maybe_gpg(val): - val = val.decode('utf-8') - if 'BEGIN PGP SIGNATURE' in val: - return val.split('\n')[-2] + val = val.decode("utf-8") + if "BEGIN PGP SIGNATURE" in val: + return val.split("\n")[-2] else: return val.strip() + def rel_time_to_epoch(year, month, day, hour, minute, second): dt = datetime.datetime(year, month, day, hour, minute, second) return (dt - datetime.datetime(1970, 1, 1)).total_seconds() + def datestamp(tag): - ts = maybe_gpg(run_git(['show', '--no-patch', '--format=%ai', tag])) + ts = maybe_gpg(run_git(["show", "--no-patch", "--format=%ai", tag])) - ts_matcher = re.compile(r'^(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2}) .*') + ts_matcher = re.compile(r"^(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2}) .*") - logging.debug('Git returned timestamp of %s for tag %s', ts, tag) + logging.debug("Git returned timestamp of %s for tag %s", ts, tag) match = ts_matcher.match(ts) if match is None: @@ -87,75 +94,100 @@ def datestamp(tag): return rel_date, rel_epoch + def revision_of(tag): - return maybe_gpg(run_git(['show', '--no-patch', '--format=%H', tag])) + return maybe_gpg(run_git(["show", "--no-patch", "--format=%H", tag])) + def extract_revision(revision, to): - tar_val = run_git(['archive', '--format=tar', '--prefix=%s/' % (to), revision]) + tar_val = run_git(["archive", "--format=tar", "--prefix=%s/" % (to), revision]) tar_f = tarfile.open(fileobj=io.BytesIO(tar_val)) if sys.version_info.major == 3 and sys.version_info.minor >= 12: - tar_f.extractall(filter='fully_trusted') + tar_f.extractall(filter="fully_trusted") else: tar_f.extractall() -def gpg_sign(keyid, passphrase_file, files, detached=True): - options = ['--armor', '--detach-sign'] if detached else ['--clearsign'] +def gpg_sign(keyid, passphrase_file, files, detached=True): + options = ["--armor", "--detach-sign"] if detached else ["--clearsign"] - gpg_cmd = ['gpg', '--batch'] + options + ['--local-user', keyid] + gpg_cmd = ["gpg", "--batch"] + options + ["--local-user", keyid] if passphrase_file is not None: - gpg_cmd[1:1] = ['--passphrase-file', passphrase_file] + gpg_cmd[1:1] = ["--passphrase-file", passphrase_file] for filename in files: - logging.info('Signing %s using PGP id %s', filename, keyid) + logging.info("Signing %s using PGP id %s", filename, keyid) cmd = gpg_cmd + [filename] - logging.debug('Running %s', ' '.join(cmd)) + logging.debug("Running %s", " ".join(cmd)) + + gpg = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - gpg = subprocess.Popen(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + check_subprocess_results(gpg, "gpg") - check_subprocess_results(gpg, 'gpg') + return [filename + ".asc" for filename in files] - return [filename + '.asc' for filename in files] def parse_args(args): parser = optparse.OptionParser( - "usage: %prog [options] \n" + - " %prog [options] snapshot " - ) - - parser.add_option('--verbose', action='store_true', - default=False, help='Extra debug output') - - parser.add_option('--quiet', action='store_true', - default=False, help='Only show errors') - - parser.add_option('--output-dir', metavar='DIR', default='.', - help='Where to place output (default %default)') - - parser.add_option('--print-output-names', action='store_true', - help='Print output archive filenames to stdout') - - parser.add_option('--archive-types', metavar='LIST', default='txz', - help='Set archive types to generate (default %default)') - - parser.add_option('--pgp-key-id', metavar='KEYID', - default='EFBADFBC', - help='PGP signing key (default %default, "none" to disable)') - - parser.add_option('--pgp-passphrase-file', metavar='FILE', - default=None, - help='PGP signing key passphrase file') - - parser.add_option('--write-hash-file', metavar='FILE', default=None, - help='Write a file with checksums') + "usage: %prog [options] \n" + + " %prog [options] snapshot " + ) + + parser.add_option( + "--verbose", action="store_true", default=False, help="Extra debug output" + ) + + parser.add_option( + "--quiet", action="store_true", default=False, help="Only show errors" + ) + + parser.add_option( + "--output-dir", + metavar="DIR", + default=".", + help="Where to place output (default %default)", + ) + + parser.add_option( + "--print-output-names", + action="store_true", + help="Print output archive filenames to stdout", + ) + + parser.add_option( + "--archive-types", + metavar="LIST", + default="txz", + help="Set archive types to generate (default %default)", + ) + + parser.add_option( + "--pgp-key-id", + metavar="KEYID", + default="EFBADFBC", + help='PGP signing key (default %default, "none" to disable)', + ) + + parser.add_option( + "--pgp-passphrase-file", + metavar="FILE", + default=None, + help="PGP signing key passphrase file", + ) + + parser.add_option( + "--write-hash-file", + metavar="FILE", + default=None, + help="Write a file with checksums", + ) return parser.parse_args(args) + def remove_file_if_exists(fspath): try: os.unlink(fspath) @@ -163,26 +195,27 @@ def remove_file_if_exists(fspath): if ex.errno != errno.ENOENT: raise -def rewrite_version_file(version_file, target_version, snapshot_branch, rev_id, rel_date): +def rewrite_version_file( + version_file, target_version, snapshot_branch, rev_id, rel_date +): if snapshot_branch: assert target_version == snapshot_branch - contents = open(version_file, encoding='utf8').readlines() + contents = open(version_file, encoding="utf8").readlines() - version_re = re.compile('release_(major|minor|patch) = ([0-9]+)') - version_suffix_re = re.compile('release_suffix = \'(-(alpha|beta|rc)[0-9]+)\'') + version_re = re.compile("release_(major|minor|patch) = ([0-9]+)") + version_suffix_re = re.compile("release_suffix = '(-(alpha|beta|rc)[0-9]+)'") def content_rewriter(target_version): version_info = {} - release_type = 'release' + release_type = "release" # Not included in old version files so set a default version_info["suffix"] = "" for line in contents: - if not snapshot_branch: match = version_re.match(line) if match: @@ -191,19 +224,19 @@ def content_rewriter(target_version): match = version_suffix_re.match(line) if match: suffix = match.group(1) - version_info['suffix'] = suffix - if suffix.find('alpha') >= 0: - release_type = 'alpha' - elif suffix.find('beta') >= 0: - release_type = 'beta' - elif suffix.find('rc') >= 0: - release_type = 'rc' - - if line == 'release_vc_rev = None\n': - yield 'release_vc_rev = \'git:%s\'\n' % (rev_id) - elif line == 'release_datestamp = 0\n': - yield 'release_datestamp = %d\n' % (rel_date) - elif line == "release_type = \'unreleased\'\n": + version_info["suffix"] = suffix + if suffix.find("alpha") >= 0: + release_type = "alpha" + elif suffix.find("beta") >= 0: + release_type = "beta" + elif suffix.find("rc") >= 0: + release_type = "rc" + + if line == "release_vc_rev = None\n": + yield "release_vc_rev = 'git:%s'\n" % (rev_id) + elif line == "release_datestamp = 0\n": + yield "release_datestamp = %d\n" % (rel_date) + elif line == "release_type = 'unreleased'\n": if target_version == snapshot_branch: yield "release_type = 'snapshot:%s'\n" % (snapshot_branch) else: @@ -214,69 +247,80 @@ def content_rewriter(target_version): if not snapshot_branch: for req_var in ["major", "minor", "patch", "suffix"]: if req_var not in version_info: - raise Exception('Missing version field for %s in version file' % (req_var)) + raise Exception( + "Missing version field for %s in version file" % (req_var) + ) - marked_version = "%d.%d.%d%s" % (version_info["major"], - version_info["minor"], - version_info["patch"], - version_info["suffix"]) + marked_version = "%d.%d.%d%s" % ( + version_info["major"], + version_info["minor"], + version_info["patch"], + version_info["suffix"], + ) if marked_version != target_version: - raise Exception('Release version file %s does not match tagged version %s' % ( - marked_version, target_version)) + raise Exception( + "Release version file %s does not match tagged version %s" + % (marked_version, target_version) + ) + + new_contents = "".join(list(content_rewriter(target_version))) + open(version_file, "w", encoding="utf8").write(new_contents) - new_contents = ''.join(list(content_rewriter(target_version))) - open(version_file, 'w', encoding='utf8').write(new_contents) -def write_archive(version, output_basename, archive_type, rel_epoch, all_files, hash_file): +def write_archive( + version, output_basename, archive_type, rel_epoch, all_files, hash_file +): def archive_suffix(archive_type): - if archive_type == 'tgz': - return 'tgz' - elif archive_type == 'tbz': - return 'tar.bz2' - elif archive_type == 'txz': - return 'tar.xz' - elif archive_type == 'tar': - return 'tar' + if archive_type == "tgz": + return "tgz" + elif archive_type == "tbz": + return "tar.bz2" + elif archive_type == "txz": + return "tar.xz" + elif archive_type == "tar": + return "tar" else: raise Exception("Unknown archive type '%s'" % (archive_type)) - output_archive = output_basename + '.' + archive_suffix(archive_type) + output_archive = output_basename + "." + archive_suffix(archive_type) logging.info('Writing archive "%s"', output_archive) remove_file_if_exists(output_archive) - remove_file_if_exists(output_archive + '.asc') + remove_file_if_exists(output_archive + ".asc") def write_mode(archive_type): - if archive_type == 'tgz': - return 'w:gz' - elif archive_type == 'tbz': - return 'w:bz2' - elif archive_type == 'txz': - return 'w:xz' - elif archive_type == 'tar': - return 'w' + if archive_type == "tgz": + return "w:gz" + elif archive_type == "tbz": + return "w:bz2" + elif archive_type == "txz": + return "w:xz" + elif archive_type == "tar": + return "w" else: raise Exception("Unknown archive type '%s'" % (archive_type)) # gzip format embeds the original filename, tarfile.py does the wrong # thing unless the output name ends in .gz. So pass an explicit # fileobj in that case, and supply a name in the form tarfile expects. - archive_suffix = '.tar.gz' if archive_type == 'tgz' else '.tar' + archive_suffix = ".tar.gz" if archive_type == "tgz" else ".tar" def archive_format(version): # A change in Python meant that 2.14 and 2.15 were released with a # tarfile using POSIX pax format (the new default for tarfile module) # instead of the previously used GNU format. - if version in ['2.14.0', '2.15.0']: + if version in ["2.14.0", "2.15.0"]: return tarfile.PAX_FORMAT else: return tarfile.GNU_FORMAT - archive = tarfile.open(output_basename + archive_suffix, - write_mode(archive_type), - format=archive_format(version), - fileobj=open(output_archive, 'wb')) + archive = tarfile.open( + output_basename + archive_suffix, + write_mode(archive_type), + format=archive_format(version), + fileobj=open(output_archive, "wb"), + ) for f in all_files: tarinfo = archive.gettarinfo(f) @@ -285,22 +329,25 @@ def archive_format(version): tarinfo.uname = "botan" tarinfo.gname = "botan" tarinfo.mtime = rel_epoch - archive.addfile(tarinfo, open(f, 'rb')) + archive.addfile(tarinfo, open(f, "rb")) archive.close() - archive_contents = open(output_archive, 'rb').read() + archive_contents = open(output_archive, "rb").read() - sha256 = hashlib.new('sha256') + sha256 = hashlib.new("sha256") sha256.update(archive_contents) archive_hash = sha256.hexdigest().upper() - logging.info('%s is %.2f MiB', output_archive, len(archive_contents) / (1024.0*1024.0)) - logging.info('SHA-256(%s) = %s', output_archive, archive_hash) + logging.info( + "%s is %.2f MiB", output_archive, len(archive_contents) / (1024.0 * 1024.0) + ) + logging.info("SHA-256(%s) = %s", output_archive, archive_hash) if hash_file is not None: hash_file.write("%s %s\n" % (archive_hash, output_archive)) return output_archive + def configure_logging(options): class ExitOnErrorLogHandler(logging.StreamHandler): def emit(self, record): @@ -319,10 +366,11 @@ def log_level(): return logging.INFO lh = ExitOnErrorLogHandler(sys.stderr) - lh.setFormatter(logging.Formatter('%(levelname) 7s: %(message)s')) + lh.setFormatter(logging.Formatter("%(levelname) 7s: %(message)s")) logging.getLogger().addHandler(lh) logging.getLogger().setLevel(log_level()) + def main(args=None): if args is None: args = sys.argv[1:] @@ -332,126 +380,141 @@ def main(args=None): configure_logging(options) if len(args) != 1 and len(args) != 2: - logging.error('Usage: %s [options] ', sys.argv[0]) + logging.error("Usage: %s [options] ", sys.argv[0]) snapshot_branch = None target_version = None - archives = options.archive_types.split(',') if options.archive_types != '' else [] + archives = options.archive_types.split(",") if options.archive_types != "" else [] for archive_type in archives: - if archive_type not in ['tar', 'tgz', 'tbz', 'txz']: + if archive_type not in ["tar", "tgz", "tbz", "txz"]: logging.error('Unknown archive type "%s"', archive_type) - if args[0] == 'snapshot': + if args[0] == "snapshot": if len(args) != 2: - logging.error('Missing branch name for snapshot command') + logging.error("Missing branch name for snapshot command") snapshot_branch = args[1] else: if len(args) != 1: - logging.error('Usage error, try --help') + logging.error("Usage error, try --help") target_version = args[0] if snapshot_branch: - logging.info('Creating snapshot release from branch %s', snapshot_branch) + logging.info("Creating snapshot release from branch %s", snapshot_branch) target_version = snapshot_branch elif len(args) == 1: try: - logging.info('Creating release for version %s', target_version) + logging.info("Creating release for version %s", target_version) except ValueError: - logging.error('Invalid version number %s', target_version) + logging.error("Invalid version number %s", target_version) rev_id = revision_of(target_version) - if rev_id == '': - logging.error('No tag matching %s found', target_version) + if rev_id == "": + logging.error("No tag matching %s found", target_version) rel_date, rel_epoch = datestamp(target_version) if rel_date == 0 or rel_epoch == 0: - logging.error('No date found for version, git error?') + logging.error("No date found for version, git error?") - logging.info('Found %s at revision id %s released %d', - target_version, rev_id, rel_date) + logging.info( + "Found %s at revision id %s released %d", target_version, rev_id, rel_date + ) - global GZIP_HEADER_TIME # pylint: disable=global-statement + global GZIP_HEADER_TIME # pylint: disable=global-statement GZIP_HEADER_TIME = rel_epoch def output_name(): if snapshot_branch: - if snapshot_branch == 'master': - return 'Botan-snapshot-%s' % (rel_date) + if snapshot_branch == "master": + return "Botan-snapshot-%s" % (rel_date) else: - return 'Botan-snapshot-%s-%s' % (snapshot_branch, rel_date) + return "Botan-snapshot-%s-%s" % (snapshot_branch, rel_date) else: - return 'Botan-' + target_version + return "Botan-" + target_version output_basename = output_name() - logging.debug('Output basename %s', output_basename) + logging.debug("Output basename %s", output_basename) if os.access(output_basename, os.X_OK): - logging.info('Removing existing output dir %s', output_basename) + logging.info("Removing existing output dir %s", output_basename) shutil.rmtree(output_basename) extract_revision(rev_id, output_basename) all_files = [] - for (curdir, _, files) in os.walk(output_basename): + for curdir, _, files in os.walk(output_basename): all_files += [os.path.join(curdir, f) for f in files] all_files.sort(key=lambda f: (os.path.dirname(f), os.path.basename(f))) def find_version_file(): - # location of file with version information has moved over time - for possible_version_file in ['src/build-data/version.txt', 'version.txt', 'botan_version.py']: + for possible_version_file in [ + "src/build-data/version.txt", + "version.txt", + "botan_version.py", + ]: full_path = os.path.join(output_basename, possible_version_file) if os.access(full_path, os.R_OK): return full_path - logging.error('Cannot locate version file') + logging.error("Cannot locate version file") return None version_file = find_version_file() if not os.access(version_file, os.R_OK): - logging.error('Cannot read %s', version_file) + logging.error("Cannot read %s", version_file) - rewrite_version_file(version_file, target_version, snapshot_branch, rev_id, rel_date) + rewrite_version_file( + version_file, target_version, snapshot_branch, rev_id, rel_date + ) try: os.makedirs(options.output_dir) except OSError as ex: if ex.errno != errno.EEXIST: - logging.error('Creating dir %s failed %s', options.output_dir, ex) + logging.error("Creating dir %s failed %s", options.output_dir, ex) output_files = [] hash_file = None if options.write_hash_file is not None: - hash_file = open(options.write_hash_file, 'w', encoding='utf8') + hash_file = open(options.write_hash_file, "w", encoding="utf8") for archive_type in archives: - output_files.append(write_archive(target_version, - output_basename, - archive_type, - rel_epoch, - all_files, - hash_file)) + output_files.append( + write_archive( + target_version, + output_basename, + archive_type, + rel_epoch, + all_files, + hash_file, + ) + ) if hash_file is not None: hash_file.close() shutil.rmtree(output_basename) - if options.pgp_key_id != 'none': + if options.pgp_key_id != "none": if options.write_hash_file is not None: - output_files += gpg_sign(options.pgp_key_id, options.pgp_passphrase_file, - [options.write_hash_file], False) + output_files += gpg_sign( + options.pgp_key_id, + options.pgp_passphrase_file, + [options.write_hash_file], + False, + ) else: - output_files += gpg_sign(options.pgp_key_id, options.pgp_passphrase_file, - output_files, True) + output_files += gpg_sign( + options.pgp_key_id, options.pgp_passphrase_file, output_files, True + ) - if options.output_dir != '.': + if options.output_dir != ".": for output_file in output_files: - logging.debug('Moving %s to %s', output_file, options.output_dir) + logging.debug("Moving %s to %s", output_file, options.output_dir) shutil.move(output_file, os.path.join(options.output_dir, output_file)) if options.print_output_names: @@ -460,7 +523,8 @@ def find_version_file(): return 0 -if __name__ == '__main__': + +if __name__ == "__main__": try: sys.exit(main()) except Exception as e: diff --git a/src/scripts/gdb/strubtest.py b/src/scripts/gdb/strubtest.py index a48552cc205..e0ee6f77be8 100644 --- a/src/scripts/gdb/strubtest.py +++ b/src/scripts/gdb/strubtest.py @@ -20,24 +20,30 @@ import gdb + def stack_pointer(frame): - return frame.read_register('sp') + return frame.read_register("sp") + def frame_pointer(frame): - return frame.read_register('fp') + return frame.read_register("fp") + def current_stackframe_memory_span(frame): start, end = stack_pointer(frame), frame_pointer(frame) if frame.architecture().name() == "aarch64" and end < start: - start, end = end, start # On aarch64, the stack grows downwards! + start, end = end, start # On aarch64, the stack grows downwards! return (start, end - start) + def report_error(error): gdb.write(f"Error: {error}\n", gdb.STDERR) + def report_status(status): gdb.write(f"{status}\n", gdb.STDOUT) + class PostStrubLocation(gdb.FinishBreakpoint): """ This (temporary) breakpoint is placed by the StrubTarget at the end of the @@ -57,15 +63,22 @@ def stackframe_size(self): return self.payload_stack_memory[1] def is_stackframe_scrubbed(self): - return all(b'\x00' == b for b in self.stackframe_memory()) + return all(b"\x00" == b for b in self.stackframe_memory()) def stop(self): if self.stackframe_size() == 0: - report_error(f"{self.function_name} has an empty stackframe, cannot validate") + report_error( + f"{self.function_name} has an empty stackframe, cannot validate" + ) elif not self.is_stackframe_scrubbed(): - report_error(f"{self.function_name} didn't get its stack frame scrubbed after usage") + report_error( + f"{self.function_name} didn't get its stack frame scrubbed after usage" + ) else: - report_status(f"Success: stackframe of {self.function_name} contains {self.stackframe_size()} zero bytes after invcoation") + report_status( + f"Success: stackframe of {self.function_name} contains {self.stackframe_size()} zero bytes after invcoation" + ) + class TargetReturnLocation(gdb.Breakpoint): """ @@ -80,7 +93,7 @@ def find_and_register_in(frame, function_name): arch = frame.architecture() assert arch.name() == "aarch64", "TargetReturnLocation is meant for aarch64" disass = arch.disassemble(frame.block().start, frame.block().end) - addrs = [f"0x{instr['addr']:x}" for instr in disass if instr['asm'] == 'ret'] + addrs = [f"0x{instr['addr']:x}" for instr in disass if instr["asm"] == "ret"] if not addrs: report_error(f"no ret instructions found in {function_name}") else: @@ -104,6 +117,7 @@ def stop(self): PostStrubLocation(caller_frame, stackframe, self.function_name) self.hit = True + class StrubTarget(gdb.Breakpoint): """ This special breakpoint shall be set to a symbol that was marked with @@ -146,6 +160,7 @@ def stop(self): # Don't stop for interactive inspection at this location return False + class StrubTest(gdb.Command): """ User-defined gdb command to set up a stack scrubbing (strub) test allowing @@ -165,10 +180,13 @@ def invoke(self, argstring, _): bp = StrubTarget(argstring) if bp.pending: bp.delete() - report_error(f"the provided symbol '{argstring}' does not appear to be available") + report_error( + f"the provided symbol '{argstring}' does not appear to be available" + ) report_status("strubtest setup done") + # Register the custom command with GDB. It may be invoked from GDB's CLI: # # strubtest diff --git a/src/scripts/install.py b/src/scripts/install.py index 5e997eebcdf..aa15af99dd4 100755 --- a/src/scripts/install.py +++ b/src/scripts/install.py @@ -11,29 +11,39 @@ import errno import json import logging -import optparse # pylint: disable=deprecated-module +import optparse # pylint: disable=deprecated-module import os import shutil import sys import traceback -def parse_command_line(args): +def parse_command_line(args): parser = optparse.OptionParser() - parser.add_option('--verbose', action='store_true', default=False, - help='Show debug messages') - parser.add_option('--quiet', action='store_true', default=False, - help='Show only warnings and errors') - - build_group = optparse.OptionGroup(parser, 'Source options') - build_group.add_option('--build-dir', metavar='DIR', default='build', - help='Location of build output (default \'%default\')') + parser.add_option( + "--verbose", action="store_true", default=False, help="Show debug messages" + ) + parser.add_option( + "--quiet", + action="store_true", + default=False, + help="Show only warnings and errors", + ) + + build_group = optparse.OptionGroup(parser, "Source options") + build_group.add_option( + "--build-dir", + metavar="DIR", + default="build", + help="Location of build output (default '%default')", + ) parser.add_option_group(build_group) - install_group = optparse.OptionGroup(parser, 'Install options') - install_group.add_option('--umask', metavar='MASK', default='022', - help='Umask to set (default %default)') + install_group = optparse.OptionGroup(parser, "Install options") + install_group.add_option( + "--umask", metavar="MASK", default="022", help="Umask to set (default %default)" + ) parser.add_option_group(install_group) (options, args) = parser.parse_args(args) @@ -65,15 +75,19 @@ def prepend_destdir(path): want relative paths to work and leverage the os awareness of os.path.join(). """ - destdir = os.environ.get('DESTDIR', "") + destdir = os.environ.get("DESTDIR", "") if destdir: # DESTDIR is non-empty, but we only join absolute paths on UNIX-like file systems if os.path.sep != "/": - raise PrependDestdirError("Only UNIX-like file systems using forward slash " \ - "separator supported when DESTDIR is set.") + raise PrependDestdirError( + "Only UNIX-like file systems using forward slash " + "separator supported when DESTDIR is set." + ) if not os.path.isabs(path): - raise PrependDestdirError("--prefix must be an absolute path when DESTDIR is set.") + raise PrependDestdirError( + "--prefix must be an absolute path when DESTDIR is set." + ) path = os.path.normpath(path) # Remove / or \ prefixes if existent to accommodate for os.path.join() @@ -81,19 +95,22 @@ def prepend_destdir(path): path = os.path.join(destdir, path) if not is_subdir(destdir, path): - raise PrependDestdirError("path escapes DESTDIR (path='%s', destdir='%s')" % (path, destdir)) + raise PrependDestdirError( + "path escapes DESTDIR (path='%s', destdir='%s')" % (path, destdir) + ) return path def makedirs(dirname, exist_ok=True): try: - logging.debug('Creating directory %s', dirname) + logging.debug("Creating directory %s", dirname) os.makedirs(dirname) except OSError as ex: if ex.errno != errno.EEXIST or not exist_ok: raise ex + # Clear link and create new one def force_symlink(target, linkname): try: @@ -103,18 +120,19 @@ def force_symlink(target, linkname): raise ex os.symlink(target, linkname) + def calculate_exec_mode(options): out = 0o777 - if 'umask' in os.__dict__: + if "umask" in os.__dict__: umask = int(options.umask, 8) - logging.debug('Setting umask to %s', oct(umask)) + logging.debug("Setting umask to %s", oct(umask)) os.umask(int(options.umask, 8)) - out &= (umask ^ 0o777) + out &= umask ^ 0o777 return out + def main(args): - logging.basicConfig(stream=sys.stdout, - format='%(levelname) 7s: %(message)s') + logging.basicConfig(stream=sys.stdout, format="%(levelname) 7s: %(message)s") (options, args) = parse_command_line(args) @@ -122,77 +140,91 @@ def main(args): build_dir = options.build_dir def copy_file(src, dst): - logging.debug('Copying %s to %s', src, dst) + logging.debug("Copying %s to %s", src, dst) shutil.copyfile(src, dst) def copy_executable(src, dst): copy_file(src, dst) - logging.debug('Make %s executable', dst) + logging.debug("Make %s executable", dst) os.chmod(dst, exe_mode) - with open(os.path.join(build_dir, 'build_config.json'), encoding='utf8') as f: + with open(os.path.join(build_dir, "build_config.json"), encoding="utf8") as f: cfg = json.load(f) - ver_major = int(cfg['version_major']) - ver_minor = int(cfg['version_minor']) - ver_patch = int(cfg['version_patch']) - target_os = cfg['os'] - build_shared_lib = bool(cfg['build_shared_lib']) - build_static_lib = bool(cfg['build_static_lib']) - build_cli = bool(cfg['build_cli_exe']) - out_dir = cfg['out_dir'] + ver_major = int(cfg["version_major"]) + ver_minor = int(cfg["version_minor"]) + ver_patch = int(cfg["version_patch"]) + target_os = cfg["os"] + build_shared_lib = bool(cfg["build_shared_lib"]) + build_static_lib = bool(cfg["build_static_lib"]) + build_cli = bool(cfg["build_cli_exe"]) + out_dir = cfg["out_dir"] - bin_dir = cfg['bindir'] - lib_dir = cfg['libdir'] - target_include_dir = cfg['installed_include_dir'] - pkgconfig_dir = os.path.join(lib_dir, 'pkgconfig') - cmake_dir = os.path.join(lib_dir, 'cmake', 'Botan-%s' % cfg["version"]) + bin_dir = cfg["bindir"] + lib_dir = cfg["libdir"] + target_include_dir = cfg["installed_include_dir"] + pkgconfig_dir = os.path.join(lib_dir, "pkgconfig") + cmake_dir = os.path.join(lib_dir, "cmake", "Botan-%s" % cfg["version"]) - prefix = cfg['prefix'] + prefix = cfg["prefix"] for d in [prefix, lib_dir, bin_dir, target_include_dir]: makedirs(prepend_destdir(d)) - for header in cfg['public_headers']: - full_header_path = os.path.join(cfg['public_include_path'], header) - copy_file(full_header_path, - prepend_destdir(os.path.join(target_include_dir, header))) + for header in cfg["public_headers"]: + full_header_path = os.path.join(cfg["public_include_path"], header) + copy_file( + full_header_path, prepend_destdir(os.path.join(target_include_dir, header)) + ) - for header in cfg['external_headers']: - full_header_path = os.path.join(cfg['external_include_path'], header) - copy_file(full_header_path, - prepend_destdir(os.path.join(target_include_dir, header))) + for header in cfg["external_headers"]: + full_header_path = os.path.join(cfg["external_include_path"], header) + copy_file( + full_header_path, prepend_destdir(os.path.join(target_include_dir, header)) + ) if build_static_lib: - static_lib = cfg['static_lib_name'] - copy_file(os.path.join(out_dir, static_lib), - prepend_destdir(os.path.join(lib_dir, os.path.basename(static_lib)))) + static_lib = cfg["static_lib_name"] + copy_file( + os.path.join(out_dir, static_lib), + prepend_destdir(os.path.join(lib_dir, os.path.basename(static_lib))), + ) if build_shared_lib: if target_os == "windows": - libname = cfg['libname'] - soname_base = libname + '.dll' - implib = cfg['implib_name'] - copy_executable(os.path.join(out_dir, soname_base), - prepend_destdir(os.path.join(bin_dir, soname_base))) - copy_file(os.path.join(out_dir, implib), - prepend_destdir(os.path.join(lib_dir, os.path.basename(implib)))) + libname = cfg["libname"] + soname_base = libname + ".dll" + implib = cfg["implib_name"] + copy_executable( + os.path.join(out_dir, soname_base), + prepend_destdir(os.path.join(bin_dir, soname_base)), + ) + copy_file( + os.path.join(out_dir, implib), + prepend_destdir(os.path.join(lib_dir, os.path.basename(implib))), + ) elif target_os == "mingw": - shared_lib_name = cfg['shared_lib_name'] - copy_executable(os.path.join(out_dir, shared_lib_name), - prepend_destdir(os.path.join(bin_dir, shared_lib_name))) - implib_name = shared_lib_name + '.a' - copy_executable(os.path.join(out_dir, implib_name), - prepend_destdir(os.path.join(lib_dir, implib_name))) + shared_lib_name = cfg["shared_lib_name"] + copy_executable( + os.path.join(out_dir, shared_lib_name), + prepend_destdir(os.path.join(bin_dir, shared_lib_name)), + ) + implib_name = shared_lib_name + ".a" + copy_executable( + os.path.join(out_dir, implib_name), + prepend_destdir(os.path.join(lib_dir, implib_name)), + ) else: - soname_patch = cfg['soname_patch'] - soname_abi = cfg['soname_abi'] - soname_base = cfg['soname_base'] + soname_patch = cfg["soname_patch"] + soname_abi = cfg["soname_abi"] + soname_base = cfg["soname_base"] - copy_executable(os.path.join(out_dir, soname_patch), - prepend_destdir(os.path.join(lib_dir, soname_patch))) + copy_executable( + os.path.join(out_dir, soname_patch), + prepend_destdir(os.path.join(lib_dir, soname_patch)), + ) - if cfg['symlink_shared_lib']: + if cfg["symlink_shared_lib"]: prev_cwd = os.getcwd() try: os.chdir(prepend_destdir(lib_dir)) @@ -202,59 +234,94 @@ def copy_executable(src, dst): os.chdir(prev_cwd) if build_cli: - copy_executable(cfg['cli_exe'], prepend_destdir(os.path.join(bin_dir, cfg['cli_exe_name']))) + copy_executable( + cfg["cli_exe"], prepend_destdir(os.path.join(bin_dir, cfg["cli_exe_name"])) + ) - if 'botan_pkgconfig' in cfg: + if "botan_pkgconfig" in cfg: makedirs(prepend_destdir(pkgconfig_dir)) - copy_file(cfg['botan_pkgconfig'], - prepend_destdir(os.path.join(pkgconfig_dir, os.path.basename(cfg['botan_pkgconfig'])))) - - if 'botan_cmake_config' in cfg and 'botan_cmake_version_config' in cfg: + copy_file( + cfg["botan_pkgconfig"], + prepend_destdir( + os.path.join(pkgconfig_dir, os.path.basename(cfg["botan_pkgconfig"])) + ), + ) + + if "botan_cmake_config" in cfg and "botan_cmake_version_config" in cfg: makedirs(prepend_destdir(cmake_dir)) - copy_file(cfg['botan_cmake_config'], - prepend_destdir(os.path.join(cmake_dir, os.path.basename(cfg['botan_cmake_config'])))) - copy_file(cfg['botan_cmake_version_config'], - prepend_destdir(os.path.join(cmake_dir, os.path.basename(cfg['botan_cmake_version_config'])))) - - if 'ffi' in cfg['mod_list'] and cfg['build_shared_lib'] is True and cfg['install_python_module'] is True: - for ver in cfg['python_version'].split(','): - py_lib_path = os.path.join(lib_dir, 'python%s' % (ver), 'site-packages') - logging.debug('Installing python module to %s', py_lib_path) + copy_file( + cfg["botan_cmake_config"], + prepend_destdir( + os.path.join(cmake_dir, os.path.basename(cfg["botan_cmake_config"])) + ), + ) + copy_file( + cfg["botan_cmake_version_config"], + prepend_destdir( + os.path.join( + cmake_dir, os.path.basename(cfg["botan_cmake_version_config"]) + ) + ), + ) + + if ( + "ffi" in cfg["mod_list"] + and cfg["build_shared_lib"] is True + and cfg["install_python_module"] is True + ): + for ver in cfg["python_version"].split(","): + py_lib_path = os.path.join(lib_dir, "python%s" % (ver), "site-packages") + logging.debug("Installing python module to %s", py_lib_path) makedirs(prepend_destdir(py_lib_path)) - py_dir = cfg['python_dir'] + py_dir = cfg["python_dir"] - copy_file(os.path.join(py_dir, 'botan3.py'), - prepend_destdir(os.path.join(py_lib_path, 'botan3.py'))) + copy_file( + os.path.join(py_dir, "botan3.py"), + prepend_destdir(os.path.join(py_lib_path, "botan3.py")), + ) - if cfg['with_documentation']: - target_doc_dir = os.path.join(prefix, cfg['docdir'], - 'botan-%d.%d.%d' % (ver_major, ver_minor, ver_patch)) + if cfg["with_documentation"]: + target_doc_dir = os.path.join( + prefix, cfg["docdir"], "botan-%d.%d.%d" % (ver_major, ver_minor, ver_patch) + ) shutil.rmtree(prepend_destdir(target_doc_dir), True) - shutil.copytree(cfg['doc_output_dir'], prepend_destdir(target_doc_dir)) - - copy_file(os.path.join(cfg['base_dir'], 'license.txt'), - prepend_destdir(os.path.join(target_doc_dir, 'license.txt'))) - copy_file(os.path.join(cfg['base_dir'], 'news.rst'), - prepend_destdir(os.path.join(target_doc_dir, 'news.txt'))) - for f in [f for f in os.listdir(cfg['doc_dir']) if f.endswith('.txt')]: - copy_file(os.path.join(cfg['doc_dir'], f), prepend_destdir(os.path.join(target_doc_dir, f))) - - if cfg['with_rst2man']: - man1_dir = prepend_destdir(os.path.join(prefix, os.path.join(cfg['mandir'], 'man1'))) + shutil.copytree(cfg["doc_output_dir"], prepend_destdir(target_doc_dir)) + + copy_file( + os.path.join(cfg["base_dir"], "license.txt"), + prepend_destdir(os.path.join(target_doc_dir, "license.txt")), + ) + copy_file( + os.path.join(cfg["base_dir"], "news.rst"), + prepend_destdir(os.path.join(target_doc_dir, "news.txt")), + ) + for f in [f for f in os.listdir(cfg["doc_dir"]) if f.endswith(".txt")]: + copy_file( + os.path.join(cfg["doc_dir"], f), + prepend_destdir(os.path.join(target_doc_dir, f)), + ) + + if cfg["with_rst2man"]: + man1_dir = prepend_destdir( + os.path.join(prefix, os.path.join(cfg["mandir"], "man1")) + ) makedirs(man1_dir) - copy_file(os.path.join(cfg['build_dir'], 'botan.1'), - os.path.join(man1_dir, 'botan.1')) + copy_file( + os.path.join(cfg["build_dir"], "botan.1"), + os.path.join(man1_dir, "botan.1"), + ) - logging.info('Botan %s installation to %s complete', cfg['version'], cfg['prefix']) + logging.info("Botan %s installation to %s complete", cfg["version"], cfg["prefix"]) return 0 -if __name__ == '__main__': + +if __name__ == "__main__": try: sys.exit(main(sys.argv)) except Exception as e: - logging.error('Failure: %s', str(e)) + logging.error("Failure: %s", str(e)) logging.info(traceback.format_exc()) sys.exit(1) diff --git a/src/scripts/python_unittests.py b/src/scripts/python_unittests.py index a6a22f3f0b5..2d0c7ed20b5 100755 --- a/src/scripts/python_unittests.py +++ b/src/scripts/python_unittests.py @@ -13,53 +13,110 @@ import sys import unittest -sys.path.append("../..") # Botan repo root -from configure import AmalgamationHelper # pylint: disable=wrong-import-position -from configure import ModulesChooser # pylint: disable=wrong-import-position +sys.path.append("../..") # Botan repo root +from configure import AmalgamationHelper # pylint: disable=wrong-import-position +from configure import ModulesChooser # pylint: disable=wrong-import-position + class AmalgamationHelperTests(unittest.TestCase): def test_matcher_std_includes(self): - self.assertEqual(AmalgamationHelper.is_unconditional_std_include("#include "), "string") - self.assertEqual(AmalgamationHelper.is_unconditional_std_include("#include // comment"), "string") - - self.assertEqual(AmalgamationHelper.is_unconditional_std_include("#include "), None) - self.assertEqual(AmalgamationHelper.is_unconditional_std_include("#include "), None) - self.assertEqual(AmalgamationHelper.is_unconditional_std_include(" #include "), None) + self.assertEqual( + AmalgamationHelper.is_unconditional_std_include("#include "), + "string", + ) + self.assertEqual( + AmalgamationHelper.is_unconditional_std_include( + "#include // comment" + ), + "string", + ) + + self.assertEqual( + AmalgamationHelper.is_unconditional_std_include("#include "), None + ) + self.assertEqual( + AmalgamationHelper.is_unconditional_std_include("#include "), None + ) + self.assertEqual( + AmalgamationHelper.is_unconditional_std_include(" #include "), None + ) def test_matcher_botan_include(self): - self.assertEqual(AmalgamationHelper.is_botan_include("#include "), - "oids.h") - self.assertEqual(AmalgamationHelper.is_botan_include("#include "), - "internal/socket.h") - self.assertEqual(AmalgamationHelper.is_botan_include("#include // comment"), - "oids.h") - self.assertEqual(AmalgamationHelper.is_botan_include("#include // comment"), - "internal/socket.h") - self.assertEqual(AmalgamationHelper.is_botan_include(" #include "), - "oids.h") - self.assertEqual(AmalgamationHelper.is_botan_include(" #include "), - "internal/socket.h") + self.assertEqual( + AmalgamationHelper.is_botan_include("#include "), "oids.h" + ) + self.assertEqual( + AmalgamationHelper.is_botan_include("#include "), + "internal/socket.h", + ) + self.assertEqual( + AmalgamationHelper.is_botan_include("#include // comment"), + "oids.h", + ) + self.assertEqual( + AmalgamationHelper.is_botan_include( + "#include // comment" + ), + "internal/socket.h", + ) + self.assertEqual( + AmalgamationHelper.is_botan_include(" #include "), "oids.h" + ) + self.assertEqual( + AmalgamationHelper.is_botan_include(" #include "), + "internal/socket.h", + ) self.assertEqual(AmalgamationHelper.is_botan_include("#include "), None) - self.assertEqual(AmalgamationHelper.is_botan_include("#include "), None) - self.assertEqual(AmalgamationHelper.is_botan_include("#include "), None) + self.assertEqual( + AmalgamationHelper.is_botan_include("#include "), None + ) + self.assertEqual( + AmalgamationHelper.is_botan_include("#include "), None + ) def test_matcher_any_includes(self): - self.assertEqual(AmalgamationHelper.is_any_include("#include "), "string") - self.assertEqual(AmalgamationHelper.is_any_include("#include "), "myfile.h") - self.assertEqual(AmalgamationHelper.is_any_include("#include "), "unistd.h") - self.assertEqual(AmalgamationHelper.is_any_include("#include "), - "botan/oids.h") - self.assertEqual(AmalgamationHelper.is_any_include(" #include "), "string") - self.assertEqual(AmalgamationHelper.is_any_include(" #include "), "myfile.h") - self.assertEqual(AmalgamationHelper.is_any_include(" #include "), "unistd.h") - self.assertEqual(AmalgamationHelper.is_any_include(" #include "), - "botan/oids.h") - self.assertEqual(AmalgamationHelper.is_any_include("#include // comment"), "string") - self.assertEqual(AmalgamationHelper.is_any_include("#include // comment"), "myfile.h") - self.assertEqual(AmalgamationHelper.is_any_include("#include // comment"), "unistd.h") - self.assertEqual(AmalgamationHelper.is_any_include("#include // comment"), - "botan/oids.h") + self.assertEqual( + AmalgamationHelper.is_any_include("#include "), "string" + ) + self.assertEqual( + AmalgamationHelper.is_any_include("#include "), "myfile.h" + ) + self.assertEqual( + AmalgamationHelper.is_any_include("#include "), "unistd.h" + ) + self.assertEqual( + AmalgamationHelper.is_any_include("#include "), "botan/oids.h" + ) + self.assertEqual( + AmalgamationHelper.is_any_include(" #include "), "string" + ) + self.assertEqual( + AmalgamationHelper.is_any_include(" #include "), "myfile.h" + ) + self.assertEqual( + AmalgamationHelper.is_any_include(" #include "), "unistd.h" + ) + self.assertEqual( + AmalgamationHelper.is_any_include(" #include "), + "botan/oids.h", + ) + self.assertEqual( + AmalgamationHelper.is_any_include("#include // comment"), "string" + ) + self.assertEqual( + AmalgamationHelper.is_any_include("#include // comment"), + "myfile.h", + ) + self.assertEqual( + AmalgamationHelper.is_any_include("#include // comment"), + "unistd.h", + ) + self.assertEqual( + AmalgamationHelper.is_any_include("#include // comment"), + "botan/oids.h", + ) + class ModulesChooserResolveDependencies(unittest.TestCase): def test_base(self): @@ -88,117 +145,74 @@ def test_no_dependencies_defined(self): def test_add_dependency(self): available_modules = set(["A", "B"]) - table = { - "A": ["B"], - "B": [] - } + table = {"A": ["B"], "B": []} ok, modules = ModulesChooser.resolve_dependencies(available_modules, table, "A") self.assertTrue(ok) self.assertEqual(modules, set(["A", "B"])) def test_add_dependencies_two_levels(self): available_modules = set(["A", "B", "C"]) - table = { - "A": ["B"], - "B": ["C"], - "C": [] - } + table = {"A": ["B"], "B": ["C"], "C": []} ok, modules = ModulesChooser.resolve_dependencies(available_modules, table, "A") self.assertTrue(ok) self.assertEqual(modules, set(["A", "B", "C"])) def test_circular(self): available_modules = set(["A", "B", "C"]) - table = { - "A": ["B"], - "B": ["C"], - "C": ["A"] - } + table = {"A": ["B"], "B": ["C"], "C": ["A"]} ok, modules = ModulesChooser.resolve_dependencies(available_modules, table, "A") self.assertTrue(ok) self.assertEqual(modules, set(["A", "B", "C"])) def test_not_available(self): available_modules = set(["A", "C"]) - table = { - "A": ["B"], - "B": ["C"], - "C": ["A"] - } + table = {"A": ["B"], "B": ["C"], "C": ["A"]} ok, _ = ModulesChooser.resolve_dependencies(available_modules, table, "B") self.assertFalse(ok) def test_dependency_not_available(self): available_modules = set(["A", "C"]) - table = { - "A": ["B"], - "B": ["C"], - "C": ["A"] - } + table = {"A": ["B"], "B": ["C"], "C": ["A"]} ok, _ = ModulesChooser.resolve_dependencies(available_modules, table, "A") self.assertFalse(ok) def test_dependency2_not_available(self): available_modules = set(["A", "B"]) - table = { - "A": ["B"], - "B": ["C"], - "C": ["A"] - } + table = {"A": ["B"], "B": ["C"], "C": ["A"]} ok, _ = ModulesChooser.resolve_dependencies(available_modules, table, "A") self.assertFalse(ok) def test_dependency_choices(self): available_modules = set(["A", "B", "C"]) - table = { - "A": ["B|C"], - "B": [], - "C": [] - } + table = {"A": ["B|C"], "B": [], "C": []} ok, modules = ModulesChooser.resolve_dependencies(available_modules, table, "A") self.assertTrue(ok) self.assertTrue(modules == set(["A", "B"]) or modules == set(["A", "C"])) def test_dependency_prefer_existing(self): available_modules = set(["A", "B", "C"]) - table = { - "A": ["C", "B|C"], - "B": [], - "C": [] - } + table = {"A": ["C", "B|C"], "B": [], "C": []} ok, modules = ModulesChooser.resolve_dependencies(available_modules, table, "A") self.assertTrue(ok) self.assertEqual(modules, set(["A", "C"])) def test_dependency_prefer_existing2(self): available_modules = set(["A", "B", "C"]) - table = { - "A": ["B", "B|C"], - "B": [], - "C": [] - } + table = {"A": ["B", "B|C"], "B": [], "C": []} ok, modules = ModulesChooser.resolve_dependencies(available_modules, table, "A") self.assertTrue(ok) self.assertEqual(modules, set(["A", "B"])) def test_dependency_choices_impossible(self): available_modules = set(["A", "C"]) - table = { - "A": ["B|C"], - "B": [], - "C": [] - } + table = {"A": ["B|C"], "B": [], "C": []} ok, modules = ModulesChooser.resolve_dependencies(available_modules, table, "A") self.assertTrue(ok) self.assertEqual(modules, set(["A", "C"])) def test_dependency_choices_impossible2(self): available_modules = set(["A", "B"]) - table = { - "A": ["B|C"], - "B": [], - "C": [] - } + table = {"A": ["B|C"], "B": [], "C": []} ok, modules = ModulesChooser.resolve_dependencies(available_modules, table, "A") self.assertTrue(ok) self.assertEqual(modules, set(["A", "B"])) @@ -212,13 +226,13 @@ def test_deep(self): "D": [], "E": ["F|G"], "F": ["A", "B"], - "G": ["A", "G"] + "G": ["A", "G"], } ok, modules = ModulesChooser.resolve_dependencies(available_modules, table, "G") self.assertTrue(ok) self.assertEqual(modules, set(["G", "A", "C", "E"])) -if __name__ == '__main__': +if __name__ == "__main__": unittest.TestCase.longMessage = True unittest.main() diff --git a/src/scripts/python_unittests_unix.py b/src/scripts/python_unittests_unix.py index fe9f06a62a8..2a2a279cb2a 100755 --- a/src/scripts/python_unittests_unix.py +++ b/src/scripts/python_unittests_unix.py @@ -15,9 +15,9 @@ import sys import unittest -sys.path.append("../..") # Botan repo root -from install import prepend_destdir # pylint: disable=wrong-import-position -from install import PrependDestdirError # pylint: disable=wrong-import-position +sys.path.append("../..") # Botan repo root +from install import prepend_destdir # pylint: disable=wrong-import-position +from install import PrependDestdirError # pylint: disable=wrong-import-position class PrependDestdir(unittest.TestCase): @@ -62,6 +62,6 @@ def test_escaping(self): prepend_destdir("/foo/../..") -if __name__ == '__main__': +if __name__ == "__main__": unittest.TestCase.longMessage = True unittest.main() diff --git a/src/scripts/repo_config.py b/src/scripts/repo_config.py index a4cfdcc13f5..c0a444e75a4 100644 --- a/src/scripts/repo_config.py +++ b/src/scripts/repo_config.py @@ -21,19 +21,26 @@ import re import sys -_DEFAULT_REPO_CONFIG_LOCATION = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'configs', 'repo_config.env')) +_DEFAULT_REPO_CONFIG_LOCATION = os.path.realpath( + os.path.join( + os.path.dirname(os.path.realpath(__file__)), "..", "configs", "repo_config.env" + ) +) + class RepoConfig(dict): - """ Reads the repository's configuration file and provides access to its variables. """ + """Reads the repository's configuration file and provides access to its variables.""" def __init__(self, env_file: str = _DEFAULT_REPO_CONFIG_LOCATION): self._file_path = env_file - parser = re.compile(r'^(?P\w+)\s*=\s*((?P[^\"]\S+)|\"(?P\S+)\")\s*$') - with open(self._file_path, 'r', encoding='utf-8') as f: + parser = re.compile( + r"^(?P\w+)\s*=\s*((?P[^\"]\S+)|\"(?P\S+)\")\s*$" + ) + with open(self._file_path, "r", encoding="utf-8") as f: for line in f.readlines(): if m := parser.match(line): - var = m.group('var') - val = m.group('val') or m.group('quoted_val') + var = m.group("var") + val = m.group("val") or m.group("quoted_val") try: self[var] = int(val) except ValueError: @@ -43,36 +50,46 @@ def __init__(self, env_file: str = _DEFAULT_REPO_CONFIG_LOCATION): def config_file_path(self): return self._file_path + def main(): def print_all(cfg: RepoConfig, *_): - print('\n'.join(f'{key}={value}' for key, value in cfg.items())) + print("\n".join(f"{key}={value}" for key, value in cfg.items())) def list_vars(cfg: RepoConfig, *_): - print('\n'.join(key for key in cfg)) + print("\n".join(key for key in cfg)) def get_var(cfg: RepoConfig, args): if args.var not in cfg: - print(f'Variable "{args.var}" not found in the configuration file.', file=sys.stderr) + print( + f'Variable "{args.var}" not found in the configuration file.', + file=sys.stderr, + ) raise KeyError() print(cfg[args.var]) def print_config_file_path(cfg, *_): print(cfg.config_file_path) - parser = argparse.ArgumentParser(description='Read and process a .env file.') - subparsers = parser.add_subparsers(dest='command', required=True) + parser = argparse.ArgumentParser(description="Read and process a .env file.") + subparsers = parser.add_subparsers(dest="command", required=True) - parser_all = subparsers.add_parser('all', help='Print all variables and their values.') + parser_all = subparsers.add_parser( + "all", help="Print all variables and their values." + ) parser_all.set_defaults(dispatch=print_all) - parser_all = subparsers.add_parser('list', help='Print all variable names.') + parser_all = subparsers.add_parser("list", help="Print all variable names.") parser_all.set_defaults(dispatch=list_vars) - parser_get = subparsers.add_parser('get', help='Get the value of a specific variable.') - parser_get.add_argument('var', type=str, help='The variable name to retrieve.') + parser_get = subparsers.add_parser( + "get", help="Get the value of a specific variable." + ) + parser_get.add_argument("var", type=str, help="The variable name to retrieve.") parser_get.set_defaults(dispatch=get_var) - parser_file = subparsers.add_parser('file', help='Print the path to the configuration file.') + parser_file = subparsers.add_parser( + "file", help="Print the path to the configuration file." + ) parser_file.set_defaults(dispatch=print_config_file_path) args = parser.parse_args() @@ -83,5 +100,6 @@ def print_config_file_path(cfg, *_): return 0 -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/rewrite_lcov.py b/src/scripts/rewrite_lcov.py index c0c12f5faa1..603f6bc81b0 100755 --- a/src/scripts/rewrite_lcov.py +++ b/src/scripts/rewrite_lcov.py @@ -10,7 +10,8 @@ import re import os -def main(args = None): + +def main(args=None): if args is None: args = sys.argv @@ -18,7 +19,9 @@ def main(args = None): print("Usage: %s " % (args[0])) return 1 - header_re = re.compile(R'^SF:(.*/build/include/(public|internal|external)/botan(/internal)?/.*)') + header_re = re.compile( + R"^SF:(.*/build/include/(public|internal|external)/botan(/internal)?/.*)" + ) new_content = "" @@ -37,11 +40,12 @@ def main(args = None): else: new_content += line - fd = open(args[1], 'w') + fd = open(args[1], "w") fd.write(new_content) fd.close() return 0 -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/run_limbo_tests.py b/src/scripts/run_limbo_tests.py index bb29034e0bf..95c88acfc3c 100755 --- a/src/scripts/run_limbo_tests.py +++ b/src/scripts/run_limbo_tests.py @@ -7,7 +7,7 @@ from botan3 import X509Cert from dateutil import parser import json -import optparse # pylint: disable=deprecated-module +import optparse # pylint: disable=deprecated-module import re import subprocess import sys @@ -15,139 +15,144 @@ ignored_tests = {} tests_that_succeed_unexpectedly = { - 'rfc5280::aki::critical-aki unexpected': 'Conflates CA and verifier requirements', - 'rfc5280::aki::critical-aki': 'Conflates CA and verifier requirements', - 'rfc5280::aki::intermediate-missing-aki': 'Conflates CA and verifier requirements', - 'rfc5280::aki::leaf-missing-aki': 'Conflates CA and verifier requirements', - 'rfc5280::ee-critical-aia-invalid': 'Conflates CA and verifier requirements', - 'rfc5280::nc::permitted-dns-match-noncritical': 'Conflates CA and verifier requirements', - 'rfc5280::pc::ica-noncritical-pc': 'Conflates CA and verifier requirements', - 'rfc5280::root-non-critical-basic-constraints': 'Conflates CA and verifier requirements', - 'rfc5280::san::noncritical-with-empty-subject': 'Conflates CA and verifier requirements', - 'rfc5280::serial::too-long': 'Conflates CA and verifier requirements', - 'rfc5280::serial::zero': 'Conflates CA and verifier requirements', - 'rfc5280::ski::intermediate-missing-ski': 'Conflates CA and verifier requirements', - 'rfc5280::ski::root-missing-ski': 'Conflates CA and verifier requirements', - - 'webpki::aki::root-with-aki-missing-keyidentifier': 'Conflates CA and verifier requirements', - 'webpki::aki::root-with-aki-authoritycertissuer': 'Conflates CA and verifier requirements', - 'webpki::aki::root-with-aki-authoritycertserialnumber': 'Conflates CA and verifier requirements', - 'webpki::aki::root-with-aki-all-fields': 'Conflates CA and verifier requirements', - 'webpki::ee-basicconstraints-ca': 'Conflates CA and verifier requirements', - 'webpki::eku::ee-without-eku': 'Conflates CA and verifier requirements', - 'webpki::san::no-san': 'Conflates CA and verifier requirements', - 'webpki::san::san-critical-with-nonempty-subject': 'Conflates CA and verifier requirements', - 'webpki::v1-cert': 'Conflates CA and verifier requirements', - 'webpki::forbidden-rsa-not-divisable-by-8-in-root': 'Conflates CA and verifier requirements', - 'webpki::forbidden-rsa-key-not-divisable-by-8-in-leaf': 'Conflates CA and verifier requirements', - 'webpki::forbidden-dsa-leaf': 'Conflates CA and verifier requirements', - 'webpki::forbidden-dsa-root': 'Conflates CA and verifier requirements', - - 'webpki::forbidden-p192-leaf': 'We do not place restrictions on the leaf key', - 'webpki::forbidden-weak-rsa-in-leaf': 'We do not place restrictions on the leaf key', - - 'webpki::san::wildcard-embedded-leftmost-san': 'CABF rule not RFC 5280', - 'webpki::ca-as-leaf': 'Not applicable outside of webpki', - - 'webpki::explicit-curve': 'Deprecated but not gone yet', - 'rfc5280::nc::invalid-dnsname-leading-period': 'Common extension', - - 'rfc5280::nc::nc-forbids-othername': 'Othername is a NULL which we drop', - 'webpki::san::wildcard-embedded-ulabel-san': 'Needs investigation', - 'webpki::malformed-aia': 'Needs investigation', - + "rfc5280::aki::critical-aki unexpected": "Conflates CA and verifier requirements", + "rfc5280::aki::critical-aki": "Conflates CA and verifier requirements", + "rfc5280::aki::intermediate-missing-aki": "Conflates CA and verifier requirements", + "rfc5280::aki::leaf-missing-aki": "Conflates CA and verifier requirements", + "rfc5280::ee-critical-aia-invalid": "Conflates CA and verifier requirements", + "rfc5280::nc::permitted-dns-match-noncritical": "Conflates CA and verifier requirements", + "rfc5280::pc::ica-noncritical-pc": "Conflates CA and verifier requirements", + "rfc5280::root-non-critical-basic-constraints": "Conflates CA and verifier requirements", + "rfc5280::san::noncritical-with-empty-subject": "Conflates CA and verifier requirements", + "rfc5280::serial::too-long": "Conflates CA and verifier requirements", + "rfc5280::serial::zero": "Conflates CA and verifier requirements", + "rfc5280::ski::intermediate-missing-ski": "Conflates CA and verifier requirements", + "rfc5280::ski::root-missing-ski": "Conflates CA and verifier requirements", + "webpki::aki::root-with-aki-missing-keyidentifier": "Conflates CA and verifier requirements", + "webpki::aki::root-with-aki-authoritycertissuer": "Conflates CA and verifier requirements", + "webpki::aki::root-with-aki-authoritycertserialnumber": "Conflates CA and verifier requirements", + "webpki::aki::root-with-aki-all-fields": "Conflates CA and verifier requirements", + "webpki::ee-basicconstraints-ca": "Conflates CA and verifier requirements", + "webpki::eku::ee-without-eku": "Conflates CA and verifier requirements", + "webpki::san::no-san": "Conflates CA and verifier requirements", + "webpki::san::san-critical-with-nonempty-subject": "Conflates CA and verifier requirements", + "webpki::v1-cert": "Conflates CA and verifier requirements", + "webpki::forbidden-rsa-not-divisable-by-8-in-root": "Conflates CA and verifier requirements", + "webpki::forbidden-rsa-key-not-divisable-by-8-in-leaf": "Conflates CA and verifier requirements", + "webpki::forbidden-dsa-leaf": "Conflates CA and verifier requirements", + "webpki::forbidden-dsa-root": "Conflates CA and verifier requirements", + "webpki::forbidden-p192-leaf": "We do not place restrictions on the leaf key", + "webpki::forbidden-weak-rsa-in-leaf": "We do not place restrictions on the leaf key", + "webpki::san::wildcard-embedded-leftmost-san": "CABF rule not RFC 5280", + "webpki::ca-as-leaf": "Not applicable outside of webpki", + "webpki::explicit-curve": "Deprecated but not gone yet", + "rfc5280::nc::invalid-dnsname-leading-period": "Common extension", + "rfc5280::nc::nc-forbids-othername": "Othername is a NULL which we drop", + "webpki::san::wildcard-embedded-ulabel-san": "Needs investigation", + "webpki::malformed-aia": "Needs investigation", # A number of tests (736, 737, ...) seem to make the implicit assumption # that if a name constraint applies to a certificate then we should not # ever use the CN as the hostname, even if the ee cert does not have a SAN - 'bettertls::nameconstraints::tc736': 'See comment above', - 'bettertls::nameconstraints::tc737': 'Same as 736', - 'bettertls::nameconstraints::tc738': 'Same as 736', - 'bettertls::nameconstraints::tc742': 'Same as 736', - 'bettertls::nameconstraints::tc743': 'Same as 736', - 'bettertls::nameconstraints::tc744': 'Same as 736', - 'bettertls::nameconstraints::tc745': 'Same as 736', - 'bettertls::nameconstraints::tc746': 'Same as 736', - 'bettertls::nameconstraints::tc747': 'Same as 736', - 'bettertls::nameconstraints::tc751': 'Same as 736', - 'bettertls::nameconstraints::tc752': 'Same as 736', - 'bettertls::nameconstraints::tc753': 'Same as 736', - 'bettertls::nameconstraints::tc754': 'Same as 736', - 'bettertls::nameconstraints::tc755': 'Same as 736', - 'bettertls::nameconstraints::tc756': 'Same as 736', - 'bettertls::nameconstraints::tc760': 'Same as 736', - 'bettertls::nameconstraints::tc761': 'Same as 736', - 'bettertls::nameconstraints::tc762': 'Same as 736', - 'bettertls::nameconstraints::tc763': 'Same as 736', - 'bettertls::nameconstraints::tc764': 'Same as 736', - 'bettertls::nameconstraints::tc765': 'Same as 736', - 'bettertls::nameconstraints::tc769': 'Same as 736', - 'bettertls::nameconstraints::tc770': 'Same as 736', - 'bettertls::nameconstraints::tc771': 'Same as 736', - 'bettertls::nameconstraints::tc772': 'Same as 736', - 'bettertls::nameconstraints::tc773': 'Same as 736', - 'bettertls::nameconstraints::tc774': 'Same as 736', - 'bettertls::nameconstraints::tc778': 'Same as 736', - 'bettertls::nameconstraints::tc779': 'Same as 736', - 'bettertls::nameconstraints::tc780': 'Same as 736', - 'bettertls::nameconstraints::tc781': 'Same as 736', - 'bettertls::nameconstraints::tc782': 'Same as 736', - 'bettertls::nameconstraints::tc783': 'Same as 736', - 'bettertls::nameconstraints::tc787': 'Same as 736', - 'bettertls::nameconstraints::tc788': 'Same as 736', - 'bettertls::nameconstraints::tc789': 'Same as 736', + "bettertls::nameconstraints::tc736": "See comment above", + "bettertls::nameconstraints::tc737": "Same as 736", + "bettertls::nameconstraints::tc738": "Same as 736", + "bettertls::nameconstraints::tc742": "Same as 736", + "bettertls::nameconstraints::tc743": "Same as 736", + "bettertls::nameconstraints::tc744": "Same as 736", + "bettertls::nameconstraints::tc745": "Same as 736", + "bettertls::nameconstraints::tc746": "Same as 736", + "bettertls::nameconstraints::tc747": "Same as 736", + "bettertls::nameconstraints::tc751": "Same as 736", + "bettertls::nameconstraints::tc752": "Same as 736", + "bettertls::nameconstraints::tc753": "Same as 736", + "bettertls::nameconstraints::tc754": "Same as 736", + "bettertls::nameconstraints::tc755": "Same as 736", + "bettertls::nameconstraints::tc756": "Same as 736", + "bettertls::nameconstraints::tc760": "Same as 736", + "bettertls::nameconstraints::tc761": "Same as 736", + "bettertls::nameconstraints::tc762": "Same as 736", + "bettertls::nameconstraints::tc763": "Same as 736", + "bettertls::nameconstraints::tc764": "Same as 736", + "bettertls::nameconstraints::tc765": "Same as 736", + "bettertls::nameconstraints::tc769": "Same as 736", + "bettertls::nameconstraints::tc770": "Same as 736", + "bettertls::nameconstraints::tc771": "Same as 736", + "bettertls::nameconstraints::tc772": "Same as 736", + "bettertls::nameconstraints::tc773": "Same as 736", + "bettertls::nameconstraints::tc774": "Same as 736", + "bettertls::nameconstraints::tc778": "Same as 736", + "bettertls::nameconstraints::tc779": "Same as 736", + "bettertls::nameconstraints::tc780": "Same as 736", + "bettertls::nameconstraints::tc781": "Same as 736", + "bettertls::nameconstraints::tc782": "Same as 736", + "bettertls::nameconstraints::tc783": "Same as 736", + "bettertls::nameconstraints::tc787": "Same as 736", + "bettertls::nameconstraints::tc788": "Same as 736", + "bettertls::nameconstraints::tc789": "Same as 736", } tests_that_fail_unexpectedly = { - 'rfc5280::nc::permitted-ipv6-match': 'IPv6 name constraints not implemented', - - 'cve::cve-2024-0567': 'Possible path building bug', - 'rfc5280::root-and-intermediate-swapped': 'Possible path building bug', - 'rfc5280::nc::permitted-self-issued': 'Possible path building bug', + "rfc5280::nc::permitted-ipv6-match": "IPv6 name constraints not implemented", + "cve::cve-2024-0567": "Possible path building bug", + "rfc5280::root-and-intermediate-swapped": "Possible path building bug", + "rfc5280::nc::permitted-self-issued": "Possible path building bug", } + def report_success(test_id, modified_result, type): if modified_result: return "GOOD %s %s (MODIFIED RESULT)" % (test_id, type) else: return "GOOD %s %s as expected" % (test_id, type) + def report_failure(test_id, modified_result, type): if modified_result: return "FAIL %s unexpectedly %s (MODIFIED RESULT)" % (test_id, type) else: return "FAIL %s unexpectedly %s" % (test_id, type) + def dump_x509(who, cert): print("%s certificate\n" % (who)) - dump_cmd = ['openssl', 'x509', '-text', '-noout'] - proc = subprocess.run(dump_cmd, - input=bytes(cert, 'utf8'), - capture_output=True) + dump_cmd = ["openssl", "x509", "-text", "-noout"] + proc = subprocess.run(dump_cmd, input=bytes(cert, "utf8"), capture_output=True) + + print(proc.stdout.decode("utf8")) - print(proc.stdout.decode('utf8')) def describe(test): - print(test['description']) - dump_x509('Peer', test['peer_certificate']) - for inter in test['untrusted_intermediates']: - dump_x509('Intermediate', inter) - for root in test['trusted_certs']: - dump_x509('Trusted', root) - -def main(args = None): + print(test["description"]) + dump_x509("Peer", test["peer_certificate"]) + for inter in test["untrusted_intermediates"]: + dump_x509("Intermediate", inter) + for root in test["trusted_certs"]: + dump_x509("Trusted", root) + + +def main(args=None): if args is None: args = sys.argv opts = optparse.OptionParser() - opts.add_option('--run-only', metavar='REGEX', default=None, - help='Run only tests matching regex') - opts.add_option('--stop-on-first-failure', action='store_true', default=False, - help='Exit immediately on first failure') - - opts.add_option('--verbose', default=False, action='store_true', - help='Print more details') + opts.add_option( + "--run-only", + metavar="REGEX", + default=None, + help="Run only tests matching regex", + ) + opts.add_option( + "--stop-on-first-failure", + action="store_true", + default=False, + help="Exit immediately on first failure", + ) + + opts.add_option( + "--verbose", default=False, action="store_true", help="Print more details" + ) (options, args) = opts.parse_args(args) @@ -163,7 +168,7 @@ def main(args = None): limbo_contents = open(limbo_file).read() limbo_json = json.loads(limbo_contents) - if limbo_json['version'] != 1: + if limbo_json["version"] != 1: print("Unexpected version in %s" % (limbo_file)) return 1 @@ -174,32 +179,32 @@ def main(args = None): unexpected_accept = 0 modified = 0 - for test in limbo_json['testcases']: + for test in limbo_json["testcases"]: if run_only is not None: - if run_only.match(test['id']) is None: + if run_only.match(test["id"]) is None: continue - if test['extended_key_usage'] != [] or test['max_chain_depth'] is not None: + if test["extended_key_usage"] != [] or test["max_chain_depth"] is not None: # we have no way of expressing this here ignored += 1 if options.verbose: - print("IGNR %s" % (test['id'])) + print("IGNR %s" % (test["id"])) continue - if test['id'] in ignored_tests: + if test["id"] in ignored_tests: ignored += 1 continue - expected_to_pass = test['expected_result'] == 'SUCCESS' + expected_to_pass = test["expected_result"] == "SUCCESS" modified_result = False - if test['id'] in tests_that_succeed_unexpectedly: - assert(not expected_to_pass) + if test["id"] in tests_that_succeed_unexpectedly: + assert not expected_to_pass expected_to_pass = True modified_result = True modified += 1 - elif test['id'] in tests_that_fail_unexpectedly: - assert(expected_to_pass) + elif test["id"] in tests_that_fail_unexpectedly: + assert expected_to_pass expected_to_pass = False modified_result = True modified += 1 @@ -207,40 +212,49 @@ def main(args = None): tests += 1 try: - trust_roots = [X509Cert(buf=x) for x in test['trusted_certs']] - intermediates = [X509Cert(buf=x) for x in test['untrusted_intermediates']] - ee_cert = X509Cert(buf=test['peer_certificate']) + trust_roots = [X509Cert(buf=x) for x in test["trusted_certs"]] + intermediates = [X509Cert(buf=x) for x in test["untrusted_intermediates"]] + ee_cert = X509Cert(buf=test["peer_certificate"]) except Exception as e: if not expected_to_pass: success += 1 else: - print("Test %s unexpected failed at parse time: got %s" % (test['id'], e)) + print( + "Test %s unexpected failed at parse time: got %s" % (test["id"], e) + ) if options.verbose: print(test) continue validation_time = 0 - if test['validation_time'] is not None: - validation_time = int(parser.parse(test['validation_time']).timestamp()) + if test["validation_time"] is not None: + validation_time = int(parser.parse(test["validation_time"]).timestamp()) hostname = None - if test['expected_peer_name'] is not None: - if test['expected_peer_name']['kind'] in ['DNS', 'IP']: - hostname = test['expected_peer_name']['value'] + if test["expected_peer_name"] is not None: + if test["expected_peer_name"]["kind"] in ["DNS", "IP"]: + hostname = test["expected_peer_name"]["value"] else: - print("Ignoring peer_name kind %s" % (test['expected_peer_name']['kind'])) - - result = ee_cert.verify(intermediates, trust_roots, required_strength=110, - hostname=hostname, reference_time=validation_time) + print( + "Ignoring peer_name kind %s" % (test["expected_peer_name"]["kind"]) + ) + + result = ee_cert.verify( + intermediates, + trust_roots, + required_strength=110, + hostname=hostname, + reference_time=validation_time, + ) if result == 0: if expected_to_pass: success += 1 if options.verbose: - print(report_success(test['id'], modified_result, 'accepted')) + print(report_success(test["id"], modified_result, "accepted")) else: unexpected_accept += 1 - print(report_failure(test['id'], modified_result, 'accepted')) + print(report_failure(test["id"], modified_result, "accepted")) if options.verbose: describe(test) @@ -250,11 +264,15 @@ def main(args = None): if not expected_to_pass: success += 1 if options.verbose: - print(report_success(test['id'], modified_result, 'rejected')) + print(report_success(test["id"], modified_result, "rejected")) else: unexpected_reject += 1 status_str = X509Cert.validation_status(result) - print(report_failure(test['id'], modified_result, "rejected with '%s'" % (status_str))) + print( + report_failure( + test["id"], modified_result, "rejected with '%s'" % (status_str) + ) + ) if options.verbose: describe(test) if options.stop_on_first_failure: @@ -272,5 +290,6 @@ def main(args = None): return 0 -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/run_tests_under_valgrind.py b/src/scripts/run_tests_under_valgrind.py index 199abcf71ba..52f9b109a0b 100755 --- a/src/scripts/run_tests_under_valgrind.py +++ b/src/scripts/run_tests_under_valgrind.py @@ -9,13 +9,14 @@ """ import multiprocessing -import optparse # pylint: disable=deprecated-module +import optparse # pylint: disable=deprecated-module import subprocess import sys import time from multiprocessing.pool import ThreadPool + def get_concurrency(): def_concurrency = 2 max_concurrency = 16 @@ -25,22 +26,24 @@ def get_concurrency(): except ImportError: return def_concurrency + def available_tests(botan_test): - cmd = [botan_test, '--list-tests'] + cmd = [botan_test, "--list-tests"] tests = subprocess.Popen(cmd, close_fds=True, stdout=subprocess.PIPE).communicate() - return [str(s, encoding='utf8') for s in tests[0].split()] + return [str(s, encoding="utf8") for s in tests[0].split()] + def run_valgrind(options, test): - valgrind_cmd = ['valgrind', '-v', '--error-exitcode=9'] + valgrind_cmd = ["valgrind", "-v", "--error-exitcode=9"] if options.with_leak_check: - valgrind_cmd += ['--leak-check=full', '--show-reachable=yes'] + valgrind_cmd += ["--leak-check=full", "--show-reachable=yes"] if options.track_origins: - valgrind_cmd += ['--track-origins=yes'] + valgrind_cmd += ["--track-origins=yes"] - botan_test_options = ['--test-threads=1', '--run-memory-intensive-tests'] + botan_test_options = ["--test-threads=1", "--run-memory-intensive-tests"] cmd = valgrind_cmd + [options.test_binary] + botan_test_options + test start = time.time() @@ -52,15 +55,19 @@ def run_valgrind(options, test): sys.stdout.flush() if proc.returncode == 0: - return True # success - - print("FAILED: valgrind testing %s failed with error code %d" % (test, proc.returncode)) - print(proc.stdout.decode('utf8')) - print(proc.stderr.decode('utf8')) + return True # success + + print( + "FAILED: valgrind testing %s failed with error code %d" + % (test, proc.returncode) + ) + print(proc.stdout.decode("utf8")) + print(proc.stderr.decode("utf8")) return False + def filter_tests(available, cmdline, options): - skip_tests = sum([x.split(',') for x in options.skip_tests], []) + skip_tests = sum([x.split(",") for x in options.skip_tests], []) to_run = [] @@ -68,7 +75,7 @@ def filter_tests(available, cmdline, options): if test in skip_tests: continue - if test.startswith('pkcs11'): + if test.startswith("pkcs11"): continue if cmdline == [] or test in cmdline: @@ -76,34 +83,61 @@ def filter_tests(available, cmdline, options): return to_run + def split_list(list, n): - return [list[x:x+n] for x in range(0, len(list), n)] + return [list[x : x + n] for x in range(0, len(list), n)] + -def main(args = None): +def main(args=None): if args is None: args = sys.argv parser = optparse.OptionParser() - parser.add_option('--verbose', action='store_true', default=False, help='be noisy') - - parser.add_option('--test-binary', metavar='PATH', default='./botan-test', - help='path to botan-test binary') - - parser.add_option('--jobs', metavar='J', default=get_concurrency(), - help='number of jobs to run in parallel (default %default)') - - parser.add_option('--with-leak-check', action='store_true', default=False, - help='enable full valgrind leak checks') - - parser.add_option('--track-origins', action='store_true', default=False, - help='enable origin tracking') - - parser.add_option('--skip-tests', metavar='TESTS', default=[], action='append', - help='skip the named tests') - - parser.add_option('--bunch', action='store_true', default=False, - help='run several test suites under each valgrind exec') + parser.add_option("--verbose", action="store_true", default=False, help="be noisy") + + parser.add_option( + "--test-binary", + metavar="PATH", + default="./botan-test", + help="path to botan-test binary", + ) + + parser.add_option( + "--jobs", + metavar="J", + default=get_concurrency(), + help="number of jobs to run in parallel (default %default)", + ) + + parser.add_option( + "--with-leak-check", + action="store_true", + default=False, + help="enable full valgrind leak checks", + ) + + parser.add_option( + "--track-origins", + action="store_true", + default=False, + help="enable origin tracking", + ) + + parser.add_option( + "--skip-tests", + metavar="TESTS", + default=[], + action="append", + help="skip the named tests", + ) + + parser.add_option( + "--bunch", + action="store_true", + default=False, + help="run several test suites under each valgrind exec", + ) (options, args) = parser.parse_args(args) @@ -134,5 +168,6 @@ def main(args = None): else: return 0 -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/run_tls_attacker.py b/src/scripts/run_tls_attacker.py index 4d69749f7cc..a581ab32501 100755 --- a/src/scripts/run_tls_attacker.py +++ b/src/scripts/run_tls_attacker.py @@ -12,19 +12,24 @@ import optparse import string + def run_subprocess(cmd): - print("Running '%s'" % (' '.join(cmd))) + print("Running '%s'" % (" ".join(cmd))) proc = subprocess.Popen(cmd, bufsize=-1) proc.communicate() if proc.returncode != 0: - print('Running "%s" failed rc %d' % (' '.join(cmd), proc.returncode)) + print('Running "%s" failed rc %d' % (" ".join(cmd), proc.returncode)) sys.exit(proc.returncode) + def spawn_server(cmd): - print("Spawning '%s'" % (' '.join(cmd))) - return subprocess.Popen(cmd, bufsize=-1)#,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + print("Spawning '%s'" % (" ".join(cmd))) + return subprocess.Popen( + cmd, bufsize=-1 + ) # ,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + def main(args=None): if args is None: @@ -32,12 +37,18 @@ def main(args=None): parser = optparse.OptionParser() - parser.add_option('--type', default='tests', - help='Which TLS-Attacker tests to run (tests, policy, fuzzer)') - parser.add_option('--src-dir', metavar='DIR', default='./src', - help='Specify path to botan sources (default "%default")') - parser.add_option('--verbose', action='store_true', - help='Be noisy') + parser.add_option( + "--type", + default="tests", + help="Which TLS-Attacker tests to run (tests, policy, fuzzer)", + ) + parser.add_option( + "--src-dir", + metavar="DIR", + default="./src", + help='Specify path to botan sources (default "%default")', + ) + parser.add_option("--verbose", action="store_true", help="Be noisy") (options, args) = parser.parse_args(args) @@ -50,7 +61,7 @@ def main(args=None): test_type = options.type src_dir = options.src_dir - if test_type not in ['tests', 'policy', 'fuzzer']: + if test_type not in ["tests", "policy", "fuzzer"]: print("Unknown --type %s" % (options.test_type)) return 1 @@ -62,80 +73,119 @@ def main(args=None): print("Unable to find src dir at %s" % (src_dir)) return 1 - test_data_dir = os.path.join(src_dir, 'tests/data') + test_data_dir = os.path.join(src_dir, "tests/data") - lax_policy_txt = os.path.join(test_data_dir, 'tls-policy/compat.txt') - bsi_policy_txt = os.path.join(test_data_dir, 'tls-policy/bsi.txt') + lax_policy_txt = os.path.join(test_data_dir, "tls-policy/compat.txt") + bsi_policy_txt = os.path.join(test_data_dir, "tls-policy/bsi.txt") - tls_attacker_dir = os.path.join(ci_tools, 'TLS-Attacker') - tls_attacker_jar = os.path.join(tls_attacker_dir, 'TLS-Attacker-1.2.jar') - tls_attacker_testsuites = os.path.join(tls_attacker_dir, 'resources/testsuite') - tls_fuzzer_workflows = os.path.join(tls_attacker_dir, 'resources/fuzzing/workflows') + tls_attacker_dir = os.path.join(ci_tools, "TLS-Attacker") + tls_attacker_jar = os.path.join(tls_attacker_dir, "TLS-Attacker-1.2.jar") + tls_attacker_testsuites = os.path.join(tls_attacker_dir, "resources/testsuite") + tls_fuzzer_workflows = os.path.join(tls_attacker_dir, "resources/fuzzing/workflows") if not os.access(tls_attacker_jar, os.R_OK): print("Unable to find TLS-Attacker jar at %s" % (tls_attacker_jar)) return 1 - rsa_key = tempfile.NamedTemporaryFile(prefix='rsa_key_') - rsa_crt = tempfile.NamedTemporaryFile(prefix='rsa_crt_') - - run_subprocess([cli_exe, 'keygen', '--algo=RSA', '--params=2048', '--output=%s' % (rsa_key.name)]) - run_subprocess([cli_exe, 'gen_self_signed', rsa_key.name, 'localhost', '--output=%s' % (rsa_crt.name)]) - - server_log = 'botan_log.txt' - server_err_log = 'botan_err_log.txt' + rsa_key = tempfile.NamedTemporaryFile(prefix="rsa_key_") + rsa_crt = tempfile.NamedTemporaryFile(prefix="rsa_crt_") + + run_subprocess( + [ + cli_exe, + "keygen", + "--algo=RSA", + "--params=2048", + "--output=%s" % (rsa_key.name), + ] + ) + run_subprocess( + [ + cli_exe, + "gen_self_signed", + rsa_key.name, + "localhost", + "--output=%s" % (rsa_crt.name), + ] + ) + + server_log = "botan_log.txt" + server_err_log = "botan_err_log.txt" tls_port = random.randint(50000, 60000) - botan_server_cmd = [cli_exe, 'tls_server', rsa_crt.name, rsa_key.name, - '--port=%d' % (tls_port), - '--output='+server_log, - '--error-output='+server_err_log] - - java_tls_attacker = ['java', '-jar', tls_attacker_jar, - '-loglevel', 'DEBUG' if options.verbose else 'ERROR'] - tls_attacker_opts = ['-tls_timeout', '300', '-connect', 'localhost:%d' % (tls_port)] - - if test_type == 'tests': + botan_server_cmd = [ + cli_exe, + "tls_server", + rsa_crt.name, + rsa_key.name, + "--port=%d" % (tls_port), + "--output=" + server_log, + "--error-output=" + server_err_log, + ] + + java_tls_attacker = [ + "java", + "-jar", + tls_attacker_jar, + "-loglevel", + "DEBUG" if options.verbose else "ERROR", + ] + tls_attacker_opts = ["-tls_timeout", "300", "-connect", "localhost:%d" % (tls_port)] + + if test_type == "tests": try: - server_process = spawn_server(botan_server_cmd + - ['--policy=%s' % (lax_policy_txt)]) + server_process = spawn_server( + botan_server_cmd + ["--policy=%s" % (lax_policy_txt)] + ) time.sleep(1) - run_subprocess(java_tls_attacker + ['testsuite_server'] + tls_attacker_opts + - ['-folder', tls_attacker_testsuites]) + run_subprocess( + java_tls_attacker + + ["testsuite_server"] + + tls_attacker_opts + + ["-folder", tls_attacker_testsuites] + ) finally: server_process.terminate() - elif test_type == 'policy': + elif test_type == "policy": try: - server_process = spawn_server(botan_server_cmd + - ['--policy=%s' % (bsi_policy_txt)]) + server_process = spawn_server( + botan_server_cmd + ["--policy=%s" % (bsi_policy_txt)] + ) time.sleep(1) - run_subprocess(java_tls_attacker + ['testtls_server'] + tls_attacker_opts + - ['-policy', bsi_policy_txt]) + run_subprocess( + java_tls_attacker + + ["testtls_server"] + + tls_attacker_opts + + ["-policy", bsi_policy_txt] + ) finally: server_process.terminate() - elif test_type == 'fuzzer': - + elif test_type == "fuzzer": template_mapping = { - 'rsa_key': rsa_key.name, - 'rsa_cert': rsa_crt.name, - 'botan_cli': cli_exe, - 'workflow_dir': tls_fuzzer_workflows, - 'fuzz_policy': lax_policy_txt, - 'tls_port': str(tls_port), - 'PORT': '$PORT' # this is a var for TLS-Attacker don't touch it + "rsa_key": rsa_key.name, + "rsa_cert": rsa_crt.name, + "botan_cli": cli_exe, + "workflow_dir": tls_fuzzer_workflows, + "fuzz_policy": lax_policy_txt, + "tls_port": str(tls_port), + "PORT": "$PORT", # this is a var for TLS-Attacker don't touch it } - template_txt = open(os.path.join(src_dir, 'scripts/fuzzer.xml')).read() + template_txt = open(os.path.join(src_dir, "scripts/fuzzer.xml")).read() config = string.Template(template_txt).substitute(template_mapping) - fuzzer_config = tempfile.NamedTemporaryFile(prefix='fuzzer_cfg_', delete=False) - fuzzer_config.write(config.encode('ascii')) + fuzzer_config = tempfile.NamedTemporaryFile(prefix="fuzzer_cfg_", delete=False) + fuzzer_config.write(config.encode("ascii")) fuzzer_config.close() - run_subprocess(java_tls_attacker + ['multi_fuzzer'] + - ['-startup_command_file', fuzzer_config.name]) + run_subprocess( + java_tls_attacker + + ["multi_fuzzer"] + + ["-startup_command_file", fuzzer_config.name] + ) + -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/run_tls_fuzzer.py b/src/scripts/run_tls_fuzzer.py index a5ca0535a52..f7a2487b794 100755 --- a/src/scripts/run_tls_fuzzer.py +++ b/src/scripts/run_tls_fuzzer.py @@ -10,38 +10,40 @@ import os import time + def script_is_disabled(script_name): - if script_name.find('tls13') >= 0: + if script_name.find("tls13") >= 0: return True - if script_name.find('sslv2') >= 0: + if script_name.find("sslv2") >= 0: return True disabled = { - 'test-SSLv3-padding.py', - 'test-serverhello-random.py', # assumes support for SSLv2 hello + "test-SSLv3-padding.py", + "test-serverhello-random.py", # assumes support for SSLv2 hello } if script_name in disabled: return True slow = { - 'test-bleichenbacher-workaround.py', - 'test-client-compatibility.py', - 'test-dhe-key-share-random.py', - 'test-dhe-no-shared-secret-padding.py', - 'test-ecdhe-padded-shared-secret.py', - 'test-ecdhe-rsa-key-share-random.py', - 'test-fuzzed-plaintext.py', - 'test-invalid-client-hello-w-record-overflow.py', - 'test-invalid-client-hello.py', - 'test-large-hello.py', + "test-bleichenbacher-workaround.py", + "test-client-compatibility.py", + "test-dhe-key-share-random.py", + "test-dhe-no-shared-secret-padding.py", + "test-ecdhe-padded-shared-secret.py", + "test-ecdhe-rsa-key-share-random.py", + "test-fuzzed-plaintext.py", + "test-invalid-client-hello-w-record-overflow.py", + "test-invalid-client-hello.py", + "test-large-hello.py", } if script_name in slow: return True return False -def main(args = None): + +def main(args=None): if args is None: args = sys.argv[1:] @@ -50,17 +52,17 @@ def main(args = None): # TODO generate key and spawn the server on some random port in tmp dir # TODO support running tls_server binary under valgrind - parser.add_argument('--verbose', action='store_true', default=False) - parser.add_argument('tls-fuzzer-dir') + parser.add_argument("--verbose", action="store_true", default=False) + parser.add_argument("tls-fuzzer-dir") args = vars(parser.parse_args(args)) - tlsfuzzer_dir = args['tls-fuzzer-dir'] + tlsfuzzer_dir = args["tls-fuzzer-dir"] if not os.access(tlsfuzzer_dir, os.X_OK): raise Exception("Unable to read TLS fuzzer dir") - tls_scripts_dir = os.path.join(tlsfuzzer_dir, 'scripts') + tls_scripts_dir = os.path.join(tlsfuzzer_dir, "scripts") if not os.access(tlsfuzzer_dir, os.X_OK): raise Exception("Unable to read TLS fuzzer scripts dir") @@ -70,19 +72,21 @@ def main(args = None): for script in scripts: if script_is_disabled(script): - logging.debug('Skipping %s' % (script)) + logging.debug("Skipping %s" % (script)) continue - procs[script] = subprocess.Popen([sys.executable, os.path.join(tls_scripts_dir, script)], - cwd=tlsfuzzer_dir, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + procs[script] = subprocess.Popen( + [sys.executable, os.path.join(tls_scripts_dir, script)], + cwd=tlsfuzzer_dir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) results = {} while len(results) != len(procs): - time.sleep(.5) - for (script, proc) in procs.items(): - + time.sleep(0.5) + for script, proc in procs.items(): if script in results: continue @@ -96,5 +100,6 @@ def main(args = None): sys.stdout.flush() return 0 -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/test_all_configs.py b/src/scripts/test_all_configs.py index 012e287f1d6..ccec5ef9229 100755 --- a/src/scripts/test_all_configs.py +++ b/src/scripts/test_all_configs.py @@ -11,46 +11,52 @@ Botan is released under the Simplified BSD License (see license.txt) """ -import optparse # pylint: disable=deprecated-module +import optparse # pylint: disable=deprecated-module import sys import subprocess + def get_module_list(configure_py): - configure = subprocess.Popen([configure_py, '--list-modules'], stdout=subprocess.PIPE) + configure = subprocess.Popen( + [configure_py, "--list-modules"], stdout=subprocess.PIPE + ) (stdout, _) = configure.communicate() if configure.returncode != 0: raise Exception("Running configure.py --list-modules failed") - modules = [s.decode('ascii') for s in stdout.split()] - modules.remove('tpm') # can't test - modules.remove('jitter_rng') - modules.remove('esdm_rng') - modules.remove('tpm2') - modules.remove('tpm2_crypto_backend') - modules.remove('tpm2_ecc') - modules.remove('tpm2_rsa') - modules.remove('base') # can't remove + modules = [s.decode("ascii") for s in stdout.split()] + modules.remove("tpm") # can't test + modules.remove("jitter_rng") + modules.remove("esdm_rng") + modules.remove("tpm2") + modules.remove("tpm2_crypto_backend") + modules.remove("tpm2_ecc") + modules.remove("tpm2_rsa") + modules.remove("base") # can't remove return modules + def get_concurrency(): def_concurrency = 2 try: import multiprocessing + return max(def_concurrency, multiprocessing.cpu_count()) except ImportError: return def_concurrency + def try_to_run(cmdline): - print("Running %s ... " % (' '.join(cmdline))) + print("Running %s ... " % (" ".join(cmdline))) sys.stdout.flush() cmd = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = cmd.communicate() - failed = (cmd.returncode != 0) + failed = cmd.returncode != 0 if failed: print("FAILURE") @@ -60,20 +66,26 @@ def try_to_run(cmdline): return not failed + def run_test_build(configure_py, modules, include, jobs, run_tests): - config = [configure_py, '--without-documentation', '--compiler-cache=ccache', '--maintainer-mode'] + config = [ + configure_py, + "--without-documentation", + "--compiler-cache=ccache", + "--maintainer-mode", + ] if include: - config.append('--minimized') + config.append("--minimized") if modules: - config.append('--enable-modules=' + ','.join(modules)) + config.append("--enable-modules=" + ",".join(modules)) else: - config.append('--disable-modules=' + ','.join(modules)) + config.append("--disable-modules=" + ",".join(modules)) if try_to_run(config) is False: return False - if try_to_run(['make', '-j', str(jobs)]) is False: + if try_to_run(["make", "-j", str(jobs)]) is False: return False if run_tests is False: @@ -82,45 +94,61 @@ def run_test_build(configure_py, modules, include, jobs, run_tests): # Flaky test causing errors when running tests tests_to_skip = [] - cmdline = ['./botan-test', '--test-threads=%d' % (jobs)] + cmdline = ["./botan-test", "--test-threads=%d" % (jobs)] if len(tests_to_skip) > 0: - cmdline.append('--skip-tests=%s' % (','.join(tests_to_skip))) + cmdline.append("--skip-tests=%s" % (",".join(tests_to_skip))) return try_to_run(cmdline) -def main(args): +def main(args): # TODO take configure.py and botan-test paths via options parser = optparse.OptionParser() - parser.add_option('--run-tests', default=False, action='store_true') - parser.add_option('--jobs', default=get_concurrency(), - help="jobs to run (default %default)") + parser.add_option("--run-tests", default=False, action="store_true") + parser.add_option( + "--jobs", default=get_concurrency(), help="jobs to run (default %default)" + ) (options, args) = parser.parse_args(args) run_tests = options.run_tests jobs = int(options.jobs) - configure_py = './configure.py' + configure_py = "./configure.py" modules = get_module_list(configure_py) - cant_disable = ['block', 'hash', 'hex', 'mac', 'modes', 'rng', 'stream', 'utils', 'cpuid'] - always_include = ['thread_utils', 'sha2_64'] + cant_disable = [ + "block", + "hash", + "hex", + "mac", + "modes", + "rng", + "stream", + "utils", + "cpuid", + ] + always_include = ["thread_utils", "sha2_64"] fails = 0 failed = [] for module in sorted(modules): if (module in always_include) or (module in cant_disable): - continue # already testing it + continue # already testing it extra = [] - if module == 'auto_rng': - extra.append('system_rng') - if run_test_build(configure_py, [module] + always_include + extra, True, jobs, run_tests) is False: + if module == "auto_rng": + extra.append("system_rng") + if ( + run_test_build( + configure_py, [module] + always_include + extra, True, jobs, run_tests + ) + is False + ): failed.append(module) fails += 1 @@ -132,11 +160,12 @@ def main(args): fails += 1 if len(failed) > 0: - print("Failed building with %s" % (' '.join(failed))) + print("Failed building with %s" % (" ".join(failed))) else: print("All configurations ok") return fails -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main(sys.argv)) diff --git a/src/scripts/test_cli.py b/src/scripts/test_cli.py index 91f61c85be4..1024da9099c 100755 --- a/src/scripts/test_cli.py +++ b/src/scripts/test_cli.py @@ -12,7 +12,7 @@ import json import logging import multiprocessing -import optparse # pylint: disable=deprecated-module +import optparse # pylint: disable=deprecated-module import os import platform import random @@ -33,20 +33,23 @@ # pylint: disable=global-statement,unused-argument CLI_PATH = None -ASYNC_TIMEOUT = 15 # seconds -TEST_DATA_DIR = '.' +ASYNC_TIMEOUT = 15 # seconds +TEST_DATA_DIR = "." ONLINE_TESTS = False TESTS_RUN = 0 TESTS_FAILED = 0 + def run_socket_tests(): # Some of the socket tests fail on FreeBSD CI, for reasons unknown. # Connecting to the server port fails. Possibly a local firewall? return platform.system().lower() != "freebsd" + def run_online_tests(): return ONLINE_TESTS + class TestLogHandler(logging.StreamHandler): def emit(self, record): # Do the default stuff first @@ -55,6 +58,7 @@ def emit(self, record): global TESTS_FAILED TESTS_FAILED += 1 + def setup_logging(options): if options.verbose: log_level = logging.DEBUG @@ -64,10 +68,11 @@ def setup_logging(options): log_level = logging.INFO lh = TestLogHandler(sys.stdout) - lh.setFormatter(logging.Formatter('%(levelname) 7s: %(message)s')) + lh.setFormatter(logging.Formatter("%(levelname) 7s: %(message)s")) logging.getLogger().addHandler(lh) logging.getLogger().setLevel(log_level) + def port_for(service): # use ports in range 63000-63100 for tests, which will hopefully # avoid conflicts with local services @@ -75,11 +80,11 @@ def port_for(service): base_port = 63000 port_assignments = { - 'tls_server': 0, - 'tls_http_server': 1, - 'tls_proxy': 2, - 'tls_proxy_backend': 3, - 'roughtime': 4, + "tls_server": 0, + "tls_http_server": 1, + "tls_proxy": 2, + "tls_proxy_backend": 3, + "roughtime": 4, } if service in port_assignments: @@ -105,7 +110,7 @@ class AsyncTestProcess: def __init__(self, name): self._name = name self._proc = None - self._stdout = b'' + self._stdout = b"" self._all_clear = False def all_clear(self): @@ -117,18 +122,23 @@ def returncode(self): @property def stdout(self): - return self._stdout.decode('utf-8') + return self._stdout.decode("utf-8") - async def _launch(self, cmd, start_sentinel = None): + async def _launch(self, cmd, start_sentinel=None): """Launch the process and wait for it to reach a defined state. If provided this listens on the process' stdout until a given sentinel string was printed. This is useful to wait for a server to be ready. """ - logging.debug("Executing: '%s'", ' '.join(cmd)) + logging.debug("Executing: '%s'", " ".join(cmd)) try: - self._proc = await asyncio.create_subprocess_exec(*cmd, stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) + self._proc = await asyncio.create_subprocess_exec( + *cmd, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) if start_sentinel: await self._read_stdout_until(start_sentinel) except: @@ -147,7 +157,9 @@ async def _read_stdout_until(self, needle): """ try: - self._stdout += await asyncio.wait_for(self._proc.stdout.readuntil(needle), timeout=ASYNC_TIMEOUT) + self._stdout += await asyncio.wait_for( + self._proc.stdout.readuntil(needle), timeout=ASYNC_TIMEOUT + ) except asyncio.IncompleteReadError as e: logging.error("%s did not report back as expected", self._name) self._stdout += e.partial @@ -156,7 +168,9 @@ async def _read_stdout_until(self, needle): logging.error("%s ran into a timeout before reporting back", self._name) raise - async def _close_stdin_read_stdout_to_eof_and_wait_for_termination(self): # pylint: disable=invalid-name + async def _close_stdin_read_stdout_to_eof_and_wait_for_termination( + self, + ): # pylint: disable=invalid-name """Gracefully signal the process to terminate by closing its stdin. If the process does not terminate in time, this function will log an @@ -164,7 +178,9 @@ async def _close_stdin_read_stdout_to_eof_and_wait_for_termination(self): # pyli """ self._proc.stdin.close() try: - self._stdout += await asyncio.wait_for(self._proc.stdout.read(), timeout=ASYNC_TIMEOUT) + self._stdout += await asyncio.wait_for( + self._proc.stdout.read(), timeout=ASYNC_TIMEOUT + ) except asyncio.TimeoutError: logging.error("%s did not close their stdout as expected", self._name) raise @@ -185,36 +201,73 @@ async def _finalize(self): finally: (final_stdout, final_stderr) = await self._proc.communicate() self._stdout += final_stdout - logging.debug("%s finished with return code: %d", self._name, self._proc.returncode) - logging.debug("%s said (stdout): %s", self._name, self._stdout.decode('utf-8')) + logging.debug( + "%s finished with return code: %d", self._name, self._proc.returncode + ) + logging.debug( + "%s said (stdout): %s", self._name, self._stdout.decode("utf-8") + ) if final_stderr: - logging.log(logging.ERROR if not self._all_clear else logging.DEBUG, - "%s said (stderr): %s", self._name, final_stderr.decode('utf-8')) + logging.log( + logging.ERROR if not self._all_clear else logging.DEBUG, + "%s said (stderr): %s", + self._name, + final_stderr.decode("utf-8"), + ) + class ServerCertificateSuite: """Generates a temporary self-signed certificate chain for testing TLS servers.""" def __init__(self, tmp_dir, ecdsa_algo, hash_algo): - tmp_subdir = tempfile.mkdtemp(prefix='botan_cli_', dir=tmp_dir) - self.private_key = os.path.join(tmp_subdir, 'priv.pem') - self.ca_cert = os.path.join(tmp_subdir, 'ca.crt') - crt_req = os.path.join(tmp_subdir, 'crt.req') - self.cert = os.path.join(tmp_subdir, 'server.crt') - - test_cli("keygen", ["--algo=ECDSA", f"--params={ecdsa_algo}", f"--output={self.private_key}"], "") - test_cli("gen_self_signed", [self.private_key, "CA", "--ca", "--country=VT", "--dns=ca.example", f"--hash={hash_algo}", f"--output={self.ca_cert}"], "") - test_cli("cert_verify", self.ca_cert, "Certificate did not validate - Cannot establish trust") - test_cli("gen_pkcs10", [f"{self.private_key}", "localhost", f"--output={crt_req}"]) - test_cli("sign_cert", [self.ca_cert, self.private_key, crt_req, f"--output={self.cert}"]) - - -def test_cli(cmd, cmd_options, - expected_output=None, - cmd_input=None, - expected_stderr=None, - use_drbg=True, - extra_env=None, - timeout=None): + tmp_subdir = tempfile.mkdtemp(prefix="botan_cli_", dir=tmp_dir) + self.private_key = os.path.join(tmp_subdir, "priv.pem") + self.ca_cert = os.path.join(tmp_subdir, "ca.crt") + crt_req = os.path.join(tmp_subdir, "crt.req") + self.cert = os.path.join(tmp_subdir, "server.crt") + + test_cli( + "keygen", + ["--algo=ECDSA", f"--params={ecdsa_algo}", f"--output={self.private_key}"], + "", + ) + test_cli( + "gen_self_signed", + [ + self.private_key, + "CA", + "--ca", + "--country=VT", + "--dns=ca.example", + f"--hash={hash_algo}", + f"--output={self.ca_cert}", + ], + "", + ) + test_cli( + "cert_verify", + self.ca_cert, + "Certificate did not validate - Cannot establish trust", + ) + test_cli( + "gen_pkcs10", [f"{self.private_key}", "localhost", f"--output={crt_req}"] + ) + test_cli( + "sign_cert", + [self.ca_cert, self.private_key, crt_req, f"--output={self.cert}"], + ) + + +def test_cli( + cmd, + cmd_options, + expected_output=None, + cmd_input=None, + expected_stderr=None, + use_drbg=True, + extra_env=None, + timeout=None, +): global TESTS_RUN TESTS_RUN += 1 @@ -222,19 +275,19 @@ def test_cli(cmd, cmd_options, opt_list = [] if isinstance(cmd_options, str): - opt_list = cmd_options.split(' ') + opt_list = cmd_options.split(" ") elif isinstance(cmd_options, list): opt_list = cmd_options if use_drbg: fixed_drbg_seed = "802" * 32 - drbg_options = ['--rng-type=drbg', '--drbg-seed=' + fixed_drbg_seed] + drbg_options = ["--rng-type=drbg", "--drbg-seed=" + fixed_drbg_seed] else: drbg_options = [] cmdline = [CLI_PATH, cmd] + drbg_options + opt_list - logging.debug("Executing '%s'", ' '.join([CLI_PATH, cmd] + opt_list)) + logging.debug("Executing '%s'", " ".join([CLI_PATH, cmd] + opt_list)) stdout = None stderr = None @@ -244,64 +297,91 @@ def test_cli(cmd, cmd_options, proc_env = None if extra_env: proc_env = os.environ - for (k,v) in extra_env.items(): + for k, v in extra_env.items(): proc_env[k] = v - proc = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=proc_env) + proc = subprocess.Popen( + cmdline, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=proc_env + ) (stdout, stderr) = proc.communicate(timeout=timeout) else: - proc = subprocess.Popen(cmdline, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + proc = subprocess.Popen( + cmdline, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) (stdout, stderr) = proc.communicate(cmd_input.encode(), timeout=timeout) except subprocess.TimeoutExpired: logging.error("Reached timeout of %d seconds for command %s", timeout, cmdline) proc.kill() (stdout, stderr) = proc.communicate() - stdout = stdout.decode('ascii').strip() - stderr = stderr.decode('ascii').strip() + stdout = stdout.decode("ascii").strip() + stderr = stderr.decode("ascii").strip() if "\r\n" in stdout: stdout = stdout.replace("\r\n", "\n") if stderr: if expected_stderr is None: - logging.error("Got output on stderr %s (stdout was %s) for command %s", stderr, stdout, cmdline, stack_info=True) + logging.error( + "Got output on stderr %s (stdout was %s) for command %s", + stderr, + stdout, + cmdline, + stack_info=True, + ) else: if stderr != expected_stderr: - logging.error("Got output on stderr %s which did not match expected value %s", stderr, expected_stderr, stack_info=True) + logging.error( + "Got output on stderr %s which did not match expected value %s", + stderr, + expected_stderr, + stack_info=True, + ) else: if expected_stderr is not None: - logging.error('Expected output on stderr but got nothing', stack_info=True) + logging.error("Expected output on stderr but got nothing", stack_info=True) if expected_output is not None: if stdout != expected_output: - logging.error("Got unexpected output running cmd %s %s", cmd, cmd_options, stack_info=True) - logging.info("Output lengths %d vs expected %d", len(stdout), len(expected_output)) + logging.error( + "Got unexpected output running cmd %s %s", + cmd, + cmd_options, + stack_info=True, + ) + logging.info( + "Output lengths %d vs expected %d", len(stdout), len(expected_output) + ) logging.info("Got %s", stdout) logging.info("Exp %s", expected_output) return stdout + def check_for_command(cmd): - cmdline = [CLI_PATH, 'has_command', cmd] + cmdline = [CLI_PATH, "has_command", cmd] proc = subprocess.Popen(cmdline) proc.communicate() return proc.returncode == 0 + def cli_config_tests(_tmp_dir): prefix = test_cli("config", "prefix") cflags = test_cli("config", "cflags") ldflags = test_cli("config", "ldflags") libs = test_cli("config", "libs") - if platform.system() == 'Windows': - if len(prefix) < 4 or prefix[1] != ':' or prefix[2] != '\\': + if platform.system() == "Windows": + if len(prefix) < 4 or prefix[1] != ":" or prefix[2] != "\\": logging.error("Bad prefix %s", prefix) if not ldflags.endswith(("-L%s\\lib" % (prefix))): logging.error("Bad ldflags %s", ldflags) else: - if len(prefix) < 4 or prefix[0] != '/': + if len(prefix) < 4 or prefix[0] != "/": logging.error("Bad prefix %s", prefix) if not ldflags.endswith(("-L%s/lib" % (prefix))): logging.error("Bad ldflags %s", ldflags) @@ -310,6 +390,7 @@ def cli_config_tests(_tmp_dir): if "-lbotan-3" not in libs: logging.error("Bad libs %s", libs) + def cli_help_tests(_tmp_dir): output = test_cli("help", None, None) @@ -317,51 +398,64 @@ def cli_help_tests(_tmp_dir): if len(output) < 500: logging.error("Help output seems very short") + def cli_version_tests(_tmp_dir): output = test_cli("version", None, None) - version_re = re.compile(r'[0-9]\.[0-9]+\.[0-9](\-[a-z]+[0-9]+)?') + version_re = re.compile(r"[0-9]\.[0-9]+\.[0-9](\-[a-z]+[0-9]+)?") if not version_re.match(output): logging.error("Unexpected short version output %s", output) output = test_cli("version", ["--full"], None, None) - version_full_re = re.compile(r'Botan [0-9]\.[0-9]+\.[0-9](\-[a-z]+[0-9]+)?( UNSAFE .* BUILD)? \(.*\)$') + version_full_re = re.compile( + r"Botan [0-9]\.[0-9]+\.[0-9](\-[a-z]+[0-9]+)?( UNSAFE .* BUILD)? \(.*\)$" + ) if not version_full_re.match(output): logging.error("Unexpected long version output %s", output) + def cli_is_prime_tests(_tmp_dir): test_cli("is_prime", "5", "5 is probably prime") test_cli("is_prime", "9", "9 is composite") - test_cli("is_prime", "548950623407687320763", "548950623407687320763 is probably prime") + test_cli( + "is_prime", "548950623407687320763", "548950623407687320763 is probably prime" + ) + def cli_gen_prime_tests(_tmp_dir): test_cli("gen_prime", "64", "15568813029901363163") test_cli("gen_prime", "128", "287193909494025008847286845478788766073") + def cli_cycle_counter(_tmp_dir): output = test_cli("cpu_clock", None, None) - if output.startswith('No CPU cycle counter on this machine'): + if output.startswith("No CPU cycle counter on this machine"): return - have_clock_re = re.compile(r'Estimated CPU clock [0-9\.]+ (M|G)Hz') + have_clock_re = re.compile(r"Estimated CPU clock [0-9\.]+ (M|G)Hz") if have_clock_re.match(output): return - logging.error('Unexpected output from cpu_clock: %s', output) + logging.error("Unexpected output from cpu_clock: %s", output) + def cli_entropy_tests(_tmp_dir): output = test_cli("entropy", ["all"], None) - status_re = re.compile('Polling [a-z0-9_]+ gathered [0-9]+ bytes in [0-9]+ outputs with estimated entropy [0-9]+') - unavail_re = re.compile('Source [a-z0-9_]+ is unavailable') - comp_re = re.compile('Sample from [a-z0-9_]+ was .* compressed from [0-9]+ bytes to [0-9]+ bytes') - output_re = re.compile(r'[A-F0-9]+(...)?') + status_re = re.compile( + "Polling [a-z0-9_]+ gathered [0-9]+ bytes in [0-9]+ outputs with estimated entropy [0-9]+" + ) + unavail_re = re.compile("Source [a-z0-9_]+ is unavailable") + comp_re = re.compile( + "Sample from [a-z0-9_]+ was .* compressed from [0-9]+ bytes to [0-9]+ bytes" + ) + output_re = re.compile(r"[A-F0-9]+(...)?") status_next = True - for line in output.split('\n'): + for line in output.split("\n"): if comp_re.match(line): continue @@ -371,30 +465,37 @@ def cli_entropy_tests(_tmp_dir): elif unavail_re.match(line) is not None: pass else: - logging.error('Unexpected status line %s', line) + logging.error("Unexpected status line %s", line) status_next = False else: if output_re.match(line) is None: - logging.error('Unexpected sample line %s', line) + logging.error("Unexpected sample line %s", line) status_next = True + def cli_factor_tests(_tmp_dir): test_cli("factor", "97", "97: 97") test_cli("factor", "9753893489562389", "9753893489562389: 21433 455087644733") - test_cli("factor", "12019502040659149507", "12019502040659149507: 3298628633 3643787579") + test_cli( + "factor", "12019502040659149507", "12019502040659149507: 3298628633 3643787579" + ) + def cli_mod_inverse_tests(_tmp_dir): test_cli("mod_inverse", "97 802", "339") test_cli("mod_inverse", "98 802", "No modular inverse exists") + def cli_base64_tests(_tmp_dir): test_cli("base64_enc", "-", "YmVlcyE=", "bees!") test_cli("base64_dec", "-", "bees!", "YmVlcyE=") + def cli_base32_tests(_tmp_dir): test_cli("base32_enc", "-", "MJSWK4ZB", "bees!") test_cli("base32_dec", "-", "bees!", "MJSWK4ZB") + def cli_base58_tests(_tmp_dir): test_cli("base58_enc", "-", "C6sRAr4", "bees!") test_cli("base58_dec", "-", "bees!", "C6sRAr4") @@ -402,56 +503,93 @@ def cli_base58_tests(_tmp_dir): test_cli("base58_enc", ["--check", "-"], "Cjv15cdjaBc", "F00F") test_cli("base58_dec", ["--check", "-"], "F00F", "Cjv15cdjaBc") + def cli_hex_tests(_tmp_dir): test_cli("hex_enc", "-", "6265657321", "bees!") test_cli("hex_dec", "-", "bees!", "6265657321") -def cli_hash_tests(_tmp_dir): - test_cli("hash", "--algo=SHA-256", - "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855 -", "") - test_cli("hash", "--algo=SHA-256", - "BA7816BF8F01CFEA414140DE5DAE2223B00361A396177A9CB410FF61F20015AD -", "abc") - - test_cli("hash", ["--algo=SHA-256", "--format=base64"], - "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0= -", "abc") - - test_cli("hash", ["--algo=SHA-224", "--format=base58", "--no-fsname"], - "MuGc8HkSVyJjfMjPM5UQikPToBTzNucEghcGLe", "abc") +def cli_hash_tests(_tmp_dir): + test_cli( + "hash", + "--algo=SHA-256", + "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855 -", + "", + ) + + test_cli( + "hash", + "--algo=SHA-256", + "BA7816BF8F01CFEA414140DE5DAE2223B00361A396177A9CB410FF61F20015AD -", + "abc", + ) + + test_cli( + "hash", + ["--algo=SHA-256", "--format=base64"], + "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0= -", + "abc", + ) + + test_cli( + "hash", + ["--algo=SHA-224", "--format=base58", "--no-fsname"], + "MuGc8HkSVyJjfMjPM5UQikPToBTzNucEghcGLe", + "abc", + ) + + test_cli( + "hash", + ["--algo=SHA-224", "--format=base58check", "--no-fsname"], + "3MmfMqgrhemdVa9bDAGfooukbviWtKMBx2xauL2RsyAe", + "abc", + ) - test_cli("hash", ["--algo=SHA-224", "--format=base58check", "--no-fsname"], - "3MmfMqgrhemdVa9bDAGfooukbviWtKMBx2xauL2RsyAe", "abc") def cli_hmac_tests(tmp_dir): - key_file = os.path.join(tmp_dir, 'hmac.key') + key_file = os.path.join(tmp_dir, "hmac.key") test_cli("rng", ["64", "--output=%s" % (key_file)], "") - test_cli("hmac", ["--no-fsname", "--hash=SHA-384", key_file, key_file], - "E3A8529377030B28A7DBDFC50DDEC8E4ECEFB6EA850D95EB785938CD3E3AFEF9EF8B08AF219C1496633193468AB755CB") + test_cli( + "hmac", + ["--no-fsname", "--hash=SHA-384", key_file, key_file], + "E3A8529377030B28A7DBDFC50DDEC8E4ECEFB6EA850D95EB785938CD3E3AFEF9EF8B08AF219C1496633193468AB755CB", + ) + def cli_bcrypt_tests(_tmp_dir): - test_cli("gen_bcrypt", "--work-factor=4 s3kr1t", - "$2a$04$0.8G7o08XYwvBBWA3l0WUujtwoGZgGDzVSN8fNkNqXikcK4A3lHPS") + test_cli( + "gen_bcrypt", + "--work-factor=4 s3kr1t", + "$2a$04$0.8G7o08XYwvBBWA3l0WUujtwoGZgGDzVSN8fNkNqXikcK4A3lHPS", + ) + + test_cli( + "check_bcrypt", + "s3kr1t $2a$04$gHX4Qg7pDSJuXiPXnmt8leyb.FFzX1Bv4rXwIj2cPSakJ8zNnhIka", + "Password is valid", + ) - test_cli("check_bcrypt", "s3kr1t $2a$04$gHX4Qg7pDSJuXiPXnmt8leyb.FFzX1Bv4rXwIj2cPSakJ8zNnhIka", - "Password is valid") + test_cli( + "check_bcrypt", + "santa $2a$04$gHX4Qg7pDSJuXiPXnmt8leyb.FFzX1Bv4rXwIj2cPSakJ8zNnhIka", + "Password is NOT valid", + ) - test_cli("check_bcrypt", "santa $2a$04$gHX4Qg7pDSJuXiPXnmt8leyb.FFzX1Bv4rXwIj2cPSakJ8zNnhIka", - "Password is NOT valid") def cli_argon2_tests(_tmp_dir): password = "s3kr1t" expected = "$argon2id$v=19$m=8,t=1,p=1$2A+I9q2+ZayxDDYC5n2YWw$/Lhx+Jbtlpw+Kxpskfv7+AKhBL/5ebalTJkVC1O5+1E" - test_cli("gen_argon2", ['--mem=8', password], expected) - test_cli("gen_argon2", ['--mem=8', '--t=1', password], expected) - test_cli("gen_argon2", ['--mem=8', '--t=1', '--p=1', password], expected) + test_cli("gen_argon2", ["--mem=8", password], expected) + test_cli("gen_argon2", ["--mem=8", "--t=1", password], expected) + test_cli("gen_argon2", ["--mem=8", "--t=1", "--p=1", password], expected) test_cli("check_argon2", [password, expected], "Password is valid") test_cli("check_argon2", ["guessing", expected], "Password is NOT valid") -def cli_gen_dl_group_tests(_tmp_dir): +def cli_gen_dl_group_tests(_tmp_dir): pem = """-----BEGIN X9.42 DH PARAMETERS----- MIIBJAKBgwTw7LQiLkXJsrgMVQxTPlWaQlYz/raZ+5RtIZe4YluQgRQGPFADLZ/t TOYzuIzZJFOcdKtEtrVkxZRGSkjZwKFKLUD6fzSjoC2M2EHktK/y5HsvxBxL4tKr @@ -478,31 +616,43 @@ def cli_gen_dl_group_tests(_tmp_dir): def cli_key_tests(tmp_dir): - pem = """-----BEGIN PRIVATE KEY----- MIGEAgEAMBAGByqGSM49AgEGBSuBBAAKBG0wawIBAQQg2A+I9q2+ZayxDDYC5n2Y W8Bn/zBm4D3mwS5qMwADRDehRANCAATwnDFqsjXL9SD/Rr1Vy4pb79PswXdQNZBN mlLtJ5JvZ0/p6zP3x+Y9yPIrAR8L/acG5ItSrAKXzzuqQQZMv4aN -----END PRIVATE KEY-----""" - priv_key = os.path.join(tmp_dir, 'priv.pem') - pub_key = os.path.join(tmp_dir, 'pub.pem') - pub_der_key = os.path.join(tmp_dir, 'pub.der') - enc_pem = os.path.join(tmp_dir, 'priv_enc.pem') - enc_der = os.path.join(tmp_dir, 'priv_enc.der') - ca_cert = os.path.join(tmp_dir, 'ca.crt') - crt_req = os.path.join(tmp_dir, 'crt.req') - user_cert = os.path.join(tmp_dir, 'user.crt') + priv_key = os.path.join(tmp_dir, "priv.pem") + pub_key = os.path.join(tmp_dir, "pub.pem") + pub_der_key = os.path.join(tmp_dir, "pub.der") + enc_pem = os.path.join(tmp_dir, "priv_enc.pem") + enc_der = os.path.join(tmp_dir, "priv_enc.der") + ca_cert = os.path.join(tmp_dir, "ca.crt") + crt_req = os.path.join(tmp_dir, "crt.req") + user_cert = os.path.join(tmp_dir, "user.crt") test_cli("keygen", ["--algo=ECDSA", "--params=secp256k1"], pem) - test_cli("keygen", ["--algo=ECDSA", "--params=secp256r1", "--output=" + priv_key], "") + test_cli( + "keygen", ["--algo=ECDSA", "--params=secp256r1", "--output=" + priv_key], "" + ) test_cli("pkcs8", "--pub-out --output=%s %s" % (pub_key, priv_key), "") - test_cli("pkcs8", "--pub-out --der-out --output=%s %s" % (pub_der_key, priv_key), "") - - test_cli("pkcs8", "--pass-out=foof --cipher=AES-128/CBC --der-out --output=%s %s" % (enc_der, priv_key), "") - test_cli("pkcs8", "--pass-out=foof --pbkdf=Scrypt --output=%s %s" % (enc_pem, priv_key), "") + test_cli( + "pkcs8", "--pub-out --der-out --output=%s %s" % (pub_der_key, priv_key), "" + ) + + test_cli( + "pkcs8", + "--pass-out=foof --cipher=AES-128/CBC --der-out --output=%s %s" + % (enc_der, priv_key), + "", + ) + test_cli( + "pkcs8", + "--pass-out=foof --pbkdf=Scrypt --output=%s %s" % (enc_pem, priv_key), + "", + ) dec_pem = test_cli("pkcs8", ["--pass-in=foof", enc_pem], None) dec_der = test_cli("pkcs8", ["--pass-in=foof", enc_der], None) @@ -510,146 +660,253 @@ def cli_key_tests(tmp_dir): if dec_pem != dec_der: logging.error("Problem decrypting PKCS8 key") - test_cli("fingerprint", ['--no-fsname', pub_key], - "83:FC:67:87:30:C7:0C:9C:54:9A:E7:A1:FA:25:83:4C:77:A4:43:16:33:6D:47:3C:CE:4B:91:62:30:97:62:D4") - - test_cli("fingerprint", ['--no-fsname', pub_der_key], - "83:FC:67:87:30:C7:0C:9C:54:9A:E7:A1:FA:25:83:4C:77:A4:43:16:33:6D:47:3C:CE:4B:91:62:30:97:62:D4") - - test_cli("fingerprint", ['--no-fsname', pub_key, pub_der_key], - "83:FC:67:87:30:C7:0C:9C:54:9A:E7:A1:FA:25:83:4C:77:A4:43:16:33:6D:47:3C:CE:4B:91:62:30:97:62:D4\n" - "83:FC:67:87:30:C7:0C:9C:54:9A:E7:A1:FA:25:83:4C:77:A4:43:16:33:6D:47:3C:CE:4B:91:62:30:97:62:D4") - - test_cli("fingerprint", [pub_der_key], - pub_der_key + - ": 83:FC:67:87:30:C7:0C:9C:54:9A:E7:A1:FA:25:83:4C:77:A4:43:16:33:6D:47:3C:CE:4B:91:62:30:97:62:D4") - - test_cli("fingerprint", ['-'], - "83:FC:67:87:30:C7:0C:9C:54:9A:E7:A1:FA:25:83:4C:77:A4:43:16:33:6D:47:3C:CE:4B:91:62:30:97:62:D4", - open(pub_key, 'rb').read().decode()) + test_cli( + "fingerprint", + ["--no-fsname", pub_key], + "83:FC:67:87:30:C7:0C:9C:54:9A:E7:A1:FA:25:83:4C:77:A4:43:16:33:6D:47:3C:CE:4B:91:62:30:97:62:D4", + ) + + test_cli( + "fingerprint", + ["--no-fsname", pub_der_key], + "83:FC:67:87:30:C7:0C:9C:54:9A:E7:A1:FA:25:83:4C:77:A4:43:16:33:6D:47:3C:CE:4B:91:62:30:97:62:D4", + ) + + test_cli( + "fingerprint", + ["--no-fsname", pub_key, pub_der_key], + "83:FC:67:87:30:C7:0C:9C:54:9A:E7:A1:FA:25:83:4C:77:A4:43:16:33:6D:47:3C:CE:4B:91:62:30:97:62:D4\n" + "83:FC:67:87:30:C7:0C:9C:54:9A:E7:A1:FA:25:83:4C:77:A4:43:16:33:6D:47:3C:CE:4B:91:62:30:97:62:D4", + ) + + test_cli( + "fingerprint", + [pub_der_key], + pub_der_key + + ": 83:FC:67:87:30:C7:0C:9C:54:9A:E7:A1:FA:25:83:4C:77:A4:43:16:33:6D:47:3C:CE:4B:91:62:30:97:62:D4", + ) + + test_cli( + "fingerprint", + ["-"], + "83:FC:67:87:30:C7:0C:9C:54:9A:E7:A1:FA:25:83:4C:77:A4:43:16:33:6D:47:3C:CE:4B:91:62:30:97:62:D4", + open(pub_key, "rb").read().decode(), + ) valid_sig = "nI4mI1ec14Y7nYUWs2edysAVvkob0TWpmGh5rrYWDA+/W9Fj0ZM21qJw8qa3/avAOIVBO6hoMEVmfJYXlS+ReA==" test_cli("sign", "%s %s" % (priv_key, pub_key), valid_sig) - test_cli("verify", [pub_key, pub_key, '-'], - "Signature is valid", valid_sig) - - test_cli("verify", [pub_key, pub_key, '-'], - "Signature is invalid", - valid_sig.replace("G", "H")) - - test_cli("gen_self_signed", - [priv_key, "CA", "--ca", "--country=VT", - "--dns=ca.example", "--hash=SHA-384", "--output="+ca_cert], - "") - - test_cli("cert_verify", ca_cert, "Certificate did not validate - Cannot establish trust") - - cert_info = test_cli("cert_info", ['--fingerprint', ca_cert], None) + test_cli("verify", [pub_key, pub_key, "-"], "Signature is valid", valid_sig) + + test_cli( + "verify", + [pub_key, pub_key, "-"], + "Signature is invalid", + valid_sig.replace("G", "H"), + ) + + test_cli( + "gen_self_signed", + [ + priv_key, + "CA", + "--ca", + "--country=VT", + "--dns=ca.example", + "--hash=SHA-384", + "--output=" + ca_cert, + ], + "", + ) + + test_cli( + "cert_verify", ca_cert, "Certificate did not validate - Cannot establish trust" + ) + + cert_info = test_cli("cert_info", ["--fingerprint", ca_cert], None) if cert_info.find('Subject: CN="CA",C="VT"') < 0: - logging.error('Unexpected output for cert_info command %s', cert_info) - if cert_info.find('Subject keyid: 69DD911C9EEE3400C67CBC3F3056CBE711BD56AF9495013F') < 0: - logging.error('Unexpected output for cert_info command %s', cert_info) + logging.error("Unexpected output for cert_info command %s", cert_info) + if ( + cert_info.find( + "Subject keyid: 69DD911C9EEE3400C67CBC3F3056CBE711BD56AF9495013F" + ) + < 0 + ): + logging.error("Unexpected output for cert_info command %s", cert_info) test_cli("gen_pkcs10", "%s User --output=%s" % (priv_key, crt_req)) - test_cli("sign_cert", "%s %s %s --output=%s" % (ca_cert, priv_key, crt_req, user_cert)) + test_cli( + "sign_cert", "%s %s %s --output=%s" % (ca_cert, priv_key, crt_req, user_cert) + ) - test_cli("cert_verify", [user_cert, ca_cert], - "Certificate passes validation checks") + test_cli( + "cert_verify", [user_cert, ca_cert], "Certificate passes validation checks" + ) + + test_cli( + "cert_verify", + user_cert, + "Certificate did not validate - Certificate issuer not found", + ) - test_cli("cert_verify", user_cert, - "Certificate did not validate - Certificate issuer not found") def cli_xmss_sign_tests(tmp_dir): - priv_key = os.path.join(tmp_dir, 'priv.pem') - pub_key = os.path.join(tmp_dir, 'pub.pem') - pub_key2 = os.path.join(tmp_dir, 'pub2.pem') - msg = os.path.join(tmp_dir, 'input') - sig1 = os.path.join(tmp_dir, 'sig1') - sig2 = os.path.join(tmp_dir, 'sig2') - root_crt = os.path.join(tmp_dir, 'root.crt') - int_csr = os.path.join(tmp_dir, 'int.csr') - int_crt = os.path.join(tmp_dir, 'int.crt') - - test_cli("rng", ['--output=%s' % (msg)], "") - test_cli("hash", ["--no-fsname", msg], "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855") + priv_key = os.path.join(tmp_dir, "priv.pem") + pub_key = os.path.join(tmp_dir, "pub.pem") + pub_key2 = os.path.join(tmp_dir, "pub2.pem") + msg = os.path.join(tmp_dir, "input") + sig1 = os.path.join(tmp_dir, "sig1") + sig2 = os.path.join(tmp_dir, "sig2") + root_crt = os.path.join(tmp_dir, "root.crt") + int_csr = os.path.join(tmp_dir, "int.csr") + int_crt = os.path.join(tmp_dir, "int.crt") + + test_cli("rng", ["--output=%s" % (msg)], "") + test_cli( + "hash", + ["--no-fsname", msg], + "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855", + ) test_cli("keygen", ["--algo=XMSS", "--output=%s" % (priv_key)], "") - test_cli("hash", ["--no-fsname", priv_key], "1F040283F0D7D2156B06B7BE03FA5861035FF3BCC059671DB288162C04A94CED") + test_cli( + "hash", + ["--no-fsname", priv_key], + "1F040283F0D7D2156B06B7BE03FA5861035FF3BCC059671DB288162C04A94CED", + ) test_cli("pkcs8", "--pub-out --output=%s %s" % (pub_key, priv_key), "") - test_cli("fingerprint", ['--no-fsname', pub_key], - "6F:C4:08:CB:C3:61:CC:49:8A:25:90:3B:2F:D4:4D:B8:7F:2F:27:06:8C:8F:01:E0:01:DB:42:1F:B4:09:09:D9") + test_cli( + "fingerprint", + ["--no-fsname", pub_key], + "6F:C4:08:CB:C3:61:CC:49:8A:25:90:3B:2F:D4:4D:B8:7F:2F:27:06:8C:8F:01:E0:01:DB:42:1F:B4:09:09:D9", + ) # verify the key is updated after each signature: test_cli("sign", [priv_key, msg, "--output=%s" % (sig1)], "") test_cli("verify", [pub_key, msg, sig1], "Signature is valid") - test_cli("hash", ["--no-fsname", sig1], "9DEBA79CE9FDC4966D7BA7B05ABEC54E3C11BB1C2C2732F7658820F2CAE47646") - test_cli("hash", ["--no-fsname", priv_key], "A71507087530C85E9CF971CF3A305890B07B51519C405A2B3D0037C64D5802B1") + test_cli( + "hash", + ["--no-fsname", sig1], + "9DEBA79CE9FDC4966D7BA7B05ABEC54E3C11BB1C2C2732F7658820F2CAE47646", + ) + test_cli( + "hash", + ["--no-fsname", priv_key], + "A71507087530C85E9CF971CF3A305890B07B51519C405A2B3D0037C64D5802B1", + ) test_cli("sign", [priv_key, msg, "--output=%s" % (sig2)], "") test_cli("verify", [pub_key, msg, sig2], "Signature is valid") - test_cli("hash", ["--no-fsname", sig2], "803EC5D6BECDFB9DC676EE2EDFEFE3D71EE924343A2ED9D2D7BFF0A9D97D704E") - test_cli("hash", ["--no-fsname", priv_key], "D581F5BFDA65669A825165C7A9CF17D6D5C5DF349004BCB7416DCD1A5C0349A0") + test_cli( + "hash", + ["--no-fsname", sig2], + "803EC5D6BECDFB9DC676EE2EDFEFE3D71EE924343A2ED9D2D7BFF0A9D97D704E", + ) + test_cli( + "hash", + ["--no-fsname", priv_key], + "D581F5BFDA65669A825165C7A9CF17D6D5C5DF349004BCB7416DCD1A5C0349A0", + ) # private key updates, public key is unchanged: test_cli("pkcs8", "--pub-out --output=%s %s" % (pub_key2, priv_key), "") - test_cli("fingerprint", ['--no-fsname', pub_key2], - "6F:C4:08:CB:C3:61:CC:49:8A:25:90:3B:2F:D4:4D:B8:7F:2F:27:06:8C:8F:01:E0:01:DB:42:1F:B4:09:09:D9") + test_cli( + "fingerprint", + ["--no-fsname", pub_key2], + "6F:C4:08:CB:C3:61:CC:49:8A:25:90:3B:2F:D4:4D:B8:7F:2F:27:06:8C:8F:01:E0:01:DB:42:1F:B4:09:09:D9", + ) # verify that key is updated when creating a self-signed certificate - test_cli("gen_self_signed", - [priv_key, "Root", "--ca", "--path-limit=2", "--output="+root_crt], "") - test_cli("hash", ["--no-fsname", priv_key], "ACFD94CDF5D0674EE5489039CF70850A1FFF95480A94E8C6C6FD2BF006909D07") + test_cli( + "gen_self_signed", + [priv_key, "Root", "--ca", "--path-limit=2", "--output=" + root_crt], + "", + ) + test_cli( + "hash", + ["--no-fsname", priv_key], + "ACFD94CDF5D0674EE5489039CF70850A1FFF95480A94E8C6C6FD2BF006909D07", + ) # verify that key is updated after signing a certificate request test_cli("gen_pkcs10", "%s Intermediate --ca --output=%s" % (priv_key, int_csr)) - test_cli("hash", ["--no-fsname", priv_key], "BE6F8F868DB495D95F73B50A370A218225253048E2F1C7C3E286568FDE203700") + test_cli( + "hash", + ["--no-fsname", priv_key], + "BE6F8F868DB495D95F73B50A370A218225253048E2F1C7C3E286568FDE203700", + ) # verify that key is updated after issuing a certificate - test_cli("sign_cert", "%s %s %s --output=%s" % (root_crt, priv_key, int_csr, int_crt)) - test_cli("hash", ["--no-fsname", priv_key], "8D3B736D8A708C342F9263163E0E3BAFE4132F74AE53A8EDF78074422CF80496") + test_cli( + "sign_cert", "%s %s %s --output=%s" % (root_crt, priv_key, int_csr, int_crt) + ) + test_cli( + "hash", + ["--no-fsname", priv_key], + "8D3B736D8A708C342F9263163E0E3BAFE4132F74AE53A8EDF78074422CF80496", + ) + + test_cli( + "cert_verify", + "%s %s" % (int_crt, root_crt), + "Certificate passes validation checks", + ) - test_cli("cert_verify", "%s %s" % (int_crt, root_crt), "Certificate passes validation checks") def cli_pbkdf_tune_tests(_tmp_dir): if not check_for_command("pbkdf_tune"): return - expected = re.compile(r'For (default|[1-9][0-9]*) ms selected Scrypt\([0-9]+,[0-9]+,[0-9]+\) using [0-9]+ MiB') + expected = re.compile( + r"For (default|[1-9][0-9]*) ms selected Scrypt\([0-9]+,[0-9]+,[0-9]+\) using [0-9]+ MiB" + ) - output = test_cli("pbkdf_tune", ["--tune-msec=1", "--check", "1", "10", "50", "default"], None).split('\n') + output = test_cli( + "pbkdf_tune", ["--tune-msec=1", "--check", "1", "10", "50", "default"], None + ).split("\n") for line in output: if expected.match(line) is None: logging.error("Unexpected line '%s'", line) - expected_pbkdf2 = re.compile(r'For (default|[1-9][0-9]*) ms selected PBKDF2\(HMAC\(SHA-256\),[0-9]+\)') + expected_pbkdf2 = re.compile( + r"For (default|[1-9][0-9]*) ms selected PBKDF2\(HMAC\(SHA-256\),[0-9]+\)" + ) - output = test_cli("pbkdf_tune", ["--algo=PBKDF2(SHA-256)", "--check", "1", "10", "50", "default"], None).split('\n') + output = test_cli( + "pbkdf_tune", + ["--algo=PBKDF2(SHA-256)", "--check", "1", "10", "50", "default"], + None, + ).split("\n") for line in output: if expected_pbkdf2.match(line) is None: logging.error("Unexpected line '%s'", line) - expected_argon2 = re.compile(r'For (default|[1-9][0-9]*) ms selected Argon2id\([0-9]+,[0-9]+,[0-9]+\)') + expected_argon2 = re.compile( + r"For (default|[1-9][0-9]*) ms selected Argon2id\([0-9]+,[0-9]+,[0-9]+\)" + ) - output = test_cli("pbkdf_tune", ["--algo=Argon2id", "--check", "1", "10", "50", "default"], None).split('\n') + output = test_cli( + "pbkdf_tune", ["--algo=Argon2id", "--check", "1", "10", "50", "default"], None + ).split("\n") for line in output: if expected_argon2.match(line) is None: logging.error("Unexpected line '%s'", line) + def cli_psk_db_tests(tmp_dir): if not check_for_command("psk_get"): return - psk_db = os.path.join(tmp_dir, 'psk.db') - db_key1 = "909"*32 - db_key2 = "451"*32 + psk_db = os.path.join(tmp_dir, "psk.db") + db_key1 = "909" * 32 + db_key2 = "451" * 32 test_cli("psk_set", [psk_db, db_key1, "name", "F00FEE"], "") test_cli("psk_set", [psk_db, db_key2, "name", "C00FEE11"], "") @@ -661,15 +918,15 @@ def cli_psk_db_tests(tmp_dir): test_cli("psk_list", [psk_db, db_key1], "name\nname2") test_cli("psk_list", [psk_db, db_key2], "name") -def cli_compress_tests(tmp_dir): +def cli_compress_tests(tmp_dir): if not check_for_command("compress"): return - input_file = os.path.join(tmp_dir, 'input.txt') - output_file = os.path.join(tmp_dir, 'input.txt.gz') + input_file = os.path.join(tmp_dir, "input.txt") + output_file = os.path.join(tmp_dir, "input.txt.gz") - with open(input_file, 'w', encoding='utf8') as f: + with open(input_file, "w", encoding="utf8") as f: f.write("hi there") f.close() @@ -678,7 +935,7 @@ def cli_compress_tests(tmp_dir): if not os.access(output_file, os.R_OK): logging.error("Compression did not created expected output file") - output_hdr = open(output_file, 'rb').read(2) + output_hdr = open(output_file, "rb").read(2) if output_hdr[0] != 0x1F or output_hdr[1] != 0x8B: logging.error("Did not see expected gzip header") @@ -690,84 +947,105 @@ def cli_compress_tests(tmp_dir): if not os.access(input_file, os.R_OK): logging.error("Decompression did not created expected output file") - recovered = open(input_file, encoding='utf8').read() + recovered = open(input_file, encoding="utf8").read() if recovered != "hi there": logging.error("Decompression did not recover original input") + def cli_rng_tests(_tmp_dir): test_cli("rng", "10", "D80F88F6ADBE65ACB10C") test_cli("rng", "16", "D80F88F6ADBE65ACB10C3602E67D985B") test_cli("rng", "10 6", "D80F88F6ADBE65ACB10C\n1B119CC068AF") - test_cli("rng", ['--format=base64', '10'], "2A+I9q2+ZayxDA==") - test_cli("rng", ['--format=base58', '10'], "D93XRyVfxqs7oR") - test_cli("rng", ['--format=base58check', '10'], "2NS1jYUq92TyGFVnhVLa") + test_cli("rng", ["--format=base64", "10"], "2A+I9q2+ZayxDA==") + test_cli("rng", ["--format=base58", "10"], "D93XRyVfxqs7oR") + test_cli("rng", ["--format=base58check", "10"], "2NS1jYUq92TyGFVnhVLa") - hex_10 = re.compile('[A-F0-9]{20}') + hex_10 = re.compile("[A-F0-9]{20}") - rngs = ['system', 'auto', 'entropy'] + rngs = ["system", "auto", "entropy"] # execute ESDM tests only on Linux - if 'BOTAN_BUILD_WITH_ESDM' in os.environ: - rngs += ['esdm-full', 'esdm-pr'] + if "BOTAN_BUILD_WITH_ESDM" in os.environ: + rngs += ["esdm-full", "esdm-pr"] for rng in rngs: - output = test_cli("rng", ["10", '--%s' % (rng)], use_drbg=False) + output = test_cli("rng", ["10", "--%s" % (rng)], use_drbg=False) if output == "D80F88F6ADBE65ACB10C": - logging.error('RNG produced DRBG output') + logging.error("RNG produced DRBG output") if hex_10.match(output) is None: - logging.error('Unexpected RNG output %s', output) + logging.error("Unexpected RNG output %s", output) - has_rdrand = test_cli("cpuid", []).find(' rdrand ') > 0 + has_rdrand = test_cli("cpuid", []).find(" rdrand ") > 0 if has_rdrand: - output = test_cli("rng", ["10", '--rdrand'], use_drbg=False) + output = test_cli("rng", ["10", "--rdrand"], use_drbg=False) if output == "D80F88F6ADBE65ACB10C": - logging.error('RDRAND produced DRBG output') + logging.error("RDRAND produced DRBG output") if hex_10.match(output) is None: - logging.error('Unexpected RNG output %s', output) + logging.error("Unexpected RNG output %s", output) + def cli_roughtime_check_tests(tmp_dir): if not check_for_command("roughtime_check"): return - chain = os.path.join(tmp_dir, 'roughtime-chain') + chain = os.path.join(tmp_dir, "roughtime-chain") - with open(chain, 'w', encoding='utf8') as f: - f.write("""\ + with open(chain, "w", encoding="utf8") as f: + f.write( + """\ ed25519 bbT+RPS7zKX6w71ssPibzmwWqU9ffRV5oj2OresSmhE= eu9yhsJfVfguVSqGZdE8WKIxaBBM0ZG3Vmuc+IyZmG2YVmrIktUByDdwIFw6F4rZqmSFsBO85ljoVPz5bVPCOw== BQAAAEAAAABAAAAApAAAADwBAABTSUcAUEFUSFNSRVBDRVJUSU5EWBnGOEajOwPA6G7oL47seBP4C7eEpr57H43C2/fK/kMA0UGZVUdf4KNX8oxOK6JIcsbVk8qhghTwA70qtwpYmQkDAAAABAAAAAwAAABSQURJTUlEUFJPT1RAQg8AJrA8tEqPBQAqisiuAxgy2Pj7UJAiWbCdzGz1xcCnja3T+AqhC8fwpeIwW4GPy/vEb/awXW2DgSLKJfzWIAz+2lsR7t4UjNPvAgAAAEAAAABTSUcAREVMRes9Ch4X0HIw5KdOTB8xK4VDFSJBD/G9t7Et/CU7UW61OiTBXYYQTG2JekWZmGa0OHX1JPGG+APkpbsNw0BKUgYDAAAAIAAAACgAAABQVUJLTUlOVE1BWFR/9BWjpsWTQ1f6iUJea3EfZ1MkX3ftJiV3ABqNLpncFwAAAAAAAAAA//////////8AAAAA ed25519 gD63hSj3ScS+wuOeGrubXlq35N1c5Lby/S+T7MNTjxo= uLeTON9D+2HqJMzK6sYWLNDEdtBl9t/9yw1cVAOm0/sONH5Oqdq9dVPkC9syjuWbglCiCPVF+FbOtcxCkrgMmA== BQAAAEAAAABAAAAApAAAADwBAABTSUcAUEFUSFNSRVBDRVJUSU5EWOw1jl0uSiBEH9HE8/6r7zxoSc01f48vw+UzH8+VJoPelnvVJBj4lnH8uRLh5Aw0i4Du7XM1dp2u0r/I5PzhMQoDAAAABAAAAAwAAABSQURJTUlEUFJPT1RAQg8AUBo+tEqPBQC47l77to7ESFTVhlw1SC74P5ssx6gpuJ6eP+1916GuUiySGE/x3Fp0c3otUGAdsRQou5p9PDTeane/YEeVq4/8AgAAAEAAAABTSUcAREVMRe5T1ml8wHyWAcEtHP/U5Rg/jFXTEXOSglngSa4aI/CECVdy4ZNWeP6vv+2//ZW7lQsrWo7ZkXpvm9BdBONRSQIDAAAAIAAAACgAAABQVUJLTUlOVE1BWFQpXlenV0OfVisvp9jDHXLw8vymZVK9Pgw9k6Edf8ZEhUgSGEc5jwUASHLvZE2PBQAAAAAA ed25519 etPaaIxcBMY1oUeGpwvPMCJMwlRVNxv51KK/tktoJTQ= U53wX99JzZwy4BXa9C6R04bPu4yqFB5w5/wTgG8Mw5wm+VLrY70ECxJ9ZHnpdHVHaLEU3aeLnQFZyZPRAEOCyw== BQAAAEAAAABAAAAApAAAADwBAABTSUcAUEFUSFNSRVBDRVJUSU5EWMh3mPWCCbOlX8xDWbU9qdfKoReJX/XLsivom8bJJYmcC7T03tyXrtWUheEJweHtg4qMgSyifQS1MjHJSy1jPAsDAAAABAAAAAwAAABSQURJTUlEUFJPT1RAQg8Akxw/tEqPBQBfOsOuciR7jiAW5itQ39y8yVr/ZJmgMwvTjqaU4/wA05ZqG4RqoLdvDXh5bCNySL6LrrnBNSAHwn5COt0CItNuAgAAAEAAAABTSUcAREVMRVP3BIOzsZmuxqMi+ScIBPyKtzFfK7ZlPFNP0JrNwln2QYtAcQFIKywDdNAAL+n8i3dz1p99K50FJjCkCl2J6AMDAAAAIAAAACgAAABQVUJLTUlOVE1BWFQKC/kZVdjiNT2NCSGfnpot4eqipyMFsyMjiIQmqqqXqQCAa245jwUAAGCgA56PBQAAAAAA ed25519 AW5uAoTSTDfG5NfY1bTh08GUnOqlRb+HVhbJ3ODJvsE= IcZcXFuaLKYYhWcK3sT/6PrVeXMmabCRbf9hvVfkMkqEW1PFL++ZnHJ1/m+G8azITxvktwsfP1YAOOxWdbf9XQ== BQAAAEAAAABAAAAApAAAADwBAABTSUcAUEFUSFNSRVBDRVJUSU5EWL5DAl8GPNUQ/mSXl0tI4N9yZAO+PiXTodJOTDL+WU/x26iqgyyQRikSSocRMzAEVLDGasdyW19mVC6H/6vfXggDAAAABAAAAAwAAABSQURJTUlEUFJPT1RAQg8Av/JAtEqPBQBIP346SHhCdDfughzeH+uYSbxngDYxqHzBDtZt0obUKrzxfRWzD1oR61B1reLvoPVCKSfzEngi/g1NSQjTrzNMAgAAAEAAAABTSUcAREVMRTQLLplQv0rN4p77Bo59qT8bbquV6MKSwILI/Tw2LLGo9noaZegUFmM+rNu1d1AVOEVQ01j6/2xDmBvp0d6MZgEDAAAAIAAAACgAAABQVUJLTUlOVE1BWFS4a1dYoIB5u/zkbR3sIteuhVrQkszzj+Gng9ywo6O9VgAAAAAAAAAA//////////8AAAAA ed25519 cj8GsiNlRkqiDElAeNMSBBMwrAl15hYPgX50+GWX/lA= Tsy82BBU2xxVqNe1ip11OyEGoKWhKoSggWjBmDTSBmKbTs7bPPCEidYc5TQ23sQUWe62G35fQOVU28q+Eq5uhQ== BQAAAEAAAABAAAAApAAAADwBAABTSUcAUEFUSFNSRVBDRVJUSU5EWDAmi7zgXAqLgQXVfbjeqnUZRiXCZI64QIoAKFL83CQHbyXgB4cNwHfQ9mSg0hYxTp1M8QxOuzusnUpk05DIRwwDAAAABAAAAAwAAABSQURJTUlEUFJPT1RAQg8AcOBCtEqPBQBhsr1mKOxxCf4VDFzAtYB4Nhs332AN1LrJU/8+VqktzfPd2R7awJHEVEWugvSvOrr+9d332mQObAkYfKfDtbSFAgAAAEAAAABTSUcAREVMRUjnhDvkIjFzTEYtgHOfMpRHtnNZj4P31RFtapkwzGjOtc93pYDd7zqQCw2AVcfbSnPqa8k26z96Q9fVRzq0pw8DAAAAIAAAACgAAABQVUJLTUlOVE1BWFR7qp2oerjpbN8Y23nUGARIlsgkodW4owH29ZKhxDMn8AAAAAAAAAAA//////////8AAAAA -""") +""" + ) - test_cli("roughtime_check", chain, """\ + test_cli( + "roughtime_check", + chain, + """\ 1: UTC 2019-08-04T13:38:17 (+-1000000us) 2: UTC 2019-08-04T13:38:17 (+-1000000us) 3: UTC 2019-08-04T13:38:17 (+-1000000us) 4: UTC 2019-08-04T13:38:18 (+-1000000us) - 5: UTC 2019-08-04T13:38:18 (+-1000000us)""") - - with open(chain, 'w', encoding='utf8') as f: - f.write("ed25519 bbT+RPS7zKX6w71ssPibzmwWqU9ffRV5oj2OresSmhE= eu9yhsJfVfguVSqGZdE8WKIxaBBM0ZG3Vmuc+IyZmG2YVmrIktUByDdwIFw6F4rZqmSFsBO85ljoVPz5bVPCOw== BQAAAEAAAABAAAAApAAAADwBAABTSUcAUEFUSFNSRVBDRVJUSU5EWBnGOEajOwPA6G7oL47seBP4C7eEpr57H43C2/fK/kMA0UGZVUdf4KNX8oxOK6JIcsbVk8qhghTwA70qtwpYmQkDAAAABAAAAAwAAABSQURJTUlEUFJPT1RAQg8AJrA8tEqPBQAqisiuAxgy2Pj7UJAiWbCdzGz1xcCnja3T+AqhC8fwpeIwW4GPy/vEb/awXW2DgSLKJfzWIAz+2lsR7t4UjNPvAgAAAEAAAABTSUcAREVMRes9Ch4X0HIw5KdOTB8xK4VDFSJBD/G9t7Et/CU7UW61OiTBXYYQTG2JekWZmGa0OHX1JPGG+APkpbsNw0BKUgYDAAAAIAAAACgAAABQVUJLTUlOVE1BWFR/9BWjpsWTQ1f6iUJea3EfZ1MkX3ftJiV3ABqNLpncFwAAAAAAAAAA//////////8AAAAA") - test_cli("roughtime_check", [chain, "--raw-time"], "1: UTC 1564925897781286 (+-1000000us)") + 5: UTC 2019-08-04T13:38:18 (+-1000000us)""", + ) + + with open(chain, "w", encoding="utf8") as f: + f.write( + "ed25519 bbT+RPS7zKX6w71ssPibzmwWqU9ffRV5oj2OresSmhE= eu9yhsJfVfguVSqGZdE8WKIxaBBM0ZG3Vmuc+IyZmG2YVmrIktUByDdwIFw6F4rZqmSFsBO85ljoVPz5bVPCOw== BQAAAEAAAABAAAAApAAAADwBAABTSUcAUEFUSFNSRVBDRVJUSU5EWBnGOEajOwPA6G7oL47seBP4C7eEpr57H43C2/fK/kMA0UGZVUdf4KNX8oxOK6JIcsbVk8qhghTwA70qtwpYmQkDAAAABAAAAAwAAABSQURJTUlEUFJPT1RAQg8AJrA8tEqPBQAqisiuAxgy2Pj7UJAiWbCdzGz1xcCnja3T+AqhC8fwpeIwW4GPy/vEb/awXW2DgSLKJfzWIAz+2lsR7t4UjNPvAgAAAEAAAABTSUcAREVMRes9Ch4X0HIw5KdOTB8xK4VDFSJBD/G9t7Et/CU7UW61OiTBXYYQTG2JekWZmGa0OHX1JPGG+APkpbsNw0BKUgYDAAAAIAAAACgAAABQVUJLTUlOVE1BWFR/9BWjpsWTQ1f6iUJea3EfZ1MkX3ftJiV3ABqNLpncFwAAAAAAAAAA//////////8AAAAA" + ) + test_cli( + "roughtime_check", + [chain, "--raw-time"], + "1: UTC 1564925897781286 (+-1000000us)", + ) + + with open(chain, "w", encoding="utf8") as f: + f.write( + "ed25519 cbT+RPS7zKX6w71ssPibzmwWqU9ffRV5oj2OresSmhE= eu9yhsJfVfguVSqGZdE8WKIxaBBM0ZG3Vmuc+IyZmG2YVmrIktUByDdwIFw6F4rZqmSFsBO85ljoVPz5bVPCOw== BQAAAEAAAABAAAAApAAAADwBAABTSUcAUEFUSFNSRVBDRVJUSU5EWBnGOEajOwPA6G7oL47seBP4C7eEpr57H43C2/fK/kMA0UGZVUdf4KNX8oxOK6JIcsbVk8qhghTwA70qtwpYmQkDAAAABAAAAAwAAABSQURJTUlEUFJPT1RAQg8AJrA8tEqPBQAqisiuAxgy2Pj7UJAiWbCdzGz1xcCnja3T+AqhC8fwpeIwW4GPy/vEb/awXW2DgSLKJfzWIAz+2lsR7t4UjNPvAgAAAEAAAABTSUcAREVMRes9Ch4X0HIw5KdOTB8xK4VDFSJBD/G9t7Et/CU7UW61OiTBXYYQTG2JekWZmGa0OHX1JPGG+APkpbsNw0BKUgYDAAAAIAAAACgAAABQVUJLTUlOVE1BWFR/9BWjpsWTQ1f6iUJea3EfZ1MkX3ftJiV3ABqNLpncFwAAAAAAAAAA//////////8AAAAA" + ) + test_cli( + "roughtime_check", + chain, + expected_stderr="Error: Roughtime: Invalid signature or public key", + ) - with open(chain, 'w', encoding='utf8') as f: - f.write("ed25519 cbT+RPS7zKX6w71ssPibzmwWqU9ffRV5oj2OresSmhE= eu9yhsJfVfguVSqGZdE8WKIxaBBM0ZG3Vmuc+IyZmG2YVmrIktUByDdwIFw6F4rZqmSFsBO85ljoVPz5bVPCOw== BQAAAEAAAABAAAAApAAAADwBAABTSUcAUEFUSFNSRVBDRVJUSU5EWBnGOEajOwPA6G7oL47seBP4C7eEpr57H43C2/fK/kMA0UGZVUdf4KNX8oxOK6JIcsbVk8qhghTwA70qtwpYmQkDAAAABAAAAAwAAABSQURJTUlEUFJPT1RAQg8AJrA8tEqPBQAqisiuAxgy2Pj7UJAiWbCdzGz1xcCnja3T+AqhC8fwpeIwW4GPy/vEb/awXW2DgSLKJfzWIAz+2lsR7t4UjNPvAgAAAEAAAABTSUcAREVMRes9Ch4X0HIw5KdOTB8xK4VDFSJBD/G9t7Et/CU7UW61OiTBXYYQTG2JekWZmGa0OHX1JPGG+APkpbsNw0BKUgYDAAAAIAAAACgAAABQVUJLTUlOVE1BWFR/9BWjpsWTQ1f6iUJea3EfZ1MkX3ftJiV3ABqNLpncFwAAAAAAAAAA//////////8AAAAA") - test_cli("roughtime_check", chain, expected_stderr='Error: Roughtime: Invalid signature or public key') def cli_roughtime_tests(tmp_dir): if not check_for_command("roughtime"): return - server_port = port_for('roughtime') - chain_file = os.path.join(tmp_dir, 'roughtime-chain') - ecosystem = os.path.join(tmp_dir, 'ecosystem') + server_port = port_for("roughtime") + chain_file = os.path.join(tmp_dir, "roughtime-chain") + ecosystem = os.path.join(tmp_dir, "ecosystem") def run_udp_server(): sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - server_address = ('127.0.0.1', server_port) + server_address = ("127.0.0.1", server_port) sock.bind(server_address) while True: @@ -809,68 +1087,92 @@ def run_udp_server(): server_request = request[0] server_response = response[0] - test_cli("roughtime", [], expected_stderr='Please specify either --servers-file or --host and --pubkey') - - with open(ecosystem, 'w', encoding='utf8') as f: - f.write("Cloudflare-Roughtime ed25519 gD63hSj4ScS+wuOeGrubXlq35N1c5Lby/S+T7MNTjxo= udp 127.0.0.1:" + str(server_port)) - - test_cli("roughtime", [ - "--check-local-clock=0", - "--chain-file=", - "--servers-file=" + ecosystem] - , expected_stderr='ERROR: Public key does not match!') - - with open(ecosystem, 'w', encoding='utf8') as f: - f.write("Cloudflare-Roughtime ed25519 gD63hSj3ScS+wuOeGrubXlq35N1c5Lby/S+T7MNTjxo= udp 127.0.0.1:" + str(server_port)) - - test_cli("roughtime", [ - "--chain-file=", - "--servers-file=" + ecosystem] - , expected_stderr='ERROR: Local clock mismatch') - - test_cli("roughtime", [ - "--check-local-clock=0", - "--chain-file=" + chain_file, - "--servers-file=" + ecosystem] - , "Cloudflare-Roughtime : UTC 2019-09-12T08:00:11 (+-1000000us)") - - with open(chain_file, 'r', encoding='utf8') as f: + test_cli( + "roughtime", + [], + expected_stderr="Please specify either --servers-file or --host and --pubkey", + ) + + with open(ecosystem, "w", encoding="utf8") as f: + f.write( + "Cloudflare-Roughtime ed25519 gD63hSj4ScS+wuOeGrubXlq35N1c5Lby/S+T7MNTjxo= udp 127.0.0.1:" + + str(server_port) + ) + + test_cli( + "roughtime", + ["--check-local-clock=0", "--chain-file=", "--servers-file=" + ecosystem], + expected_stderr="ERROR: Public key does not match!", + ) + + with open(ecosystem, "w", encoding="utf8") as f: + f.write( + "Cloudflare-Roughtime ed25519 gD63hSj3ScS+wuOeGrubXlq35N1c5Lby/S+T7MNTjxo= udp 127.0.0.1:" + + str(server_port) + ) + + test_cli( + "roughtime", + ["--chain-file=", "--servers-file=" + ecosystem], + expected_stderr="ERROR: Local clock mismatch", + ) + + test_cli( + "roughtime", + [ + "--check-local-clock=0", + "--chain-file=" + chain_file, + "--servers-file=" + ecosystem, + ], + "Cloudflare-Roughtime : UTC 2019-09-12T08:00:11 (+-1000000us)", + ) + + with open(chain_file, "r", encoding="utf8") as f: read_data = f.read() if read_data != chain[0]: logging.error("unexpected chain") server_request = request[1] server_response = response[1] - test_cli("roughtime", [ - "--check-local-clock=0", - "--chain-file=" + chain_file, - "--host=127.0.0.1:" + str(server_port), - "--pubkey=gD63hSj3ScS+wuOeGrubXlq35N1c5Lby/S+T7MNTjxo=", - "--raw-time"] - , "UTC 1568275214691000 (+-1000000us)") - - with open(chain_file, 'r', encoding='utf8') as f: + test_cli( + "roughtime", + [ + "--check-local-clock=0", + "--chain-file=" + chain_file, + "--host=127.0.0.1:" + str(server_port), + "--pubkey=gD63hSj3ScS+wuOeGrubXlq35N1c5Lby/S+T7MNTjxo=", + "--raw-time", + ], + "UTC 1568275214691000 (+-1000000us)", + ) + + with open(chain_file, "r", encoding="utf8") as f: read_data = f.read() if read_data != chain[1]: logging.error("unexpected chain") server_request = request[2] server_response = response[2] - test_cli("roughtime", [ - "--check-local-clock=0", - "--chain-file=" + chain_file, - "--host=127.0.0.1:" + str(server_port), - "--pubkey=gD63hSj3ScS+wuOeGrubXlq35N1c5Lby/S+T7MNTjxo=", - "--max-chain-size=2"] - , "UTC 2019-09-12T08:00:42 (+-1000000us)") - - with open(chain_file, 'r', encoding='utf8') as f: + test_cli( + "roughtime", + [ + "--check-local-clock=0", + "--chain-file=" + chain_file, + "--host=127.0.0.1:" + str(server_port), + "--pubkey=gD63hSj3ScS+wuOeGrubXlq35N1c5Lby/S+T7MNTjxo=", + "--max-chain-size=2", + ], + "UTC 2019-09-12T08:00:42 (+-1000000us)", + ) + + with open(chain_file, "r", encoding="utf8") as f: read_data = f.read() if read_data != chain[2]: logging.error("unexpected chain") + def cli_zfec_tests(tmp_dir): - input_file = os.path.join(tmp_dir, 'input.bin') + input_file = os.path.join(tmp_dir, "input.bin") exp_hash = "B49BCD978052C2C05A2D9ACE9863D150E3FA5765FCDF91AC47B5EAD54BFEE24E" @@ -880,15 +1182,22 @@ def cli_zfec_tests(tmp_dir): k = 3 n = 5 - test_cli("fec_encode", ["--output-dir=%s" % (tmp_dir), - "--prefix=%s" % (prefix), - str(k), str(n), input_file]) + test_cli( + "fec_encode", + [ + "--output-dir=%s" % (tmp_dir), + "--prefix=%s" % (prefix), + str(k), + str(n), + input_file, + ], + ) - info_re = re.compile('FEC share [0-9]/%d with %d needed for recovery' % (n, k)) + info_re = re.compile("FEC share [0-9]/%d with %d needed for recovery" % (n, k)) share_files = [] for share in range(1, n + 1): - expected_share = os.path.join(tmp_dir, '%s.%d_%d.fec' % (prefix, share, n)) + expected_share = os.path.join(tmp_dir, "%s.%d_%d.fec" % (prefix, share, n)) share_files.append(expected_share) info_out = test_cli("fec_info", expected_share) if info_re.match(info_out) is None: @@ -897,13 +1206,19 @@ def cli_zfec_tests(tmp_dir): k_shares = n - k # Insufficient shares: - test_cli("fec_decode", share_files[(k_shares + 1):], None, None, - "At least %d shares are required for recovery" % (k)) - - output_file = os.path.join(tmp_dir, 'output.bin') + test_cli( + "fec_decode", + share_files[(k_shares + 1) :], + None, + None, + "At least %d shares are required for recovery" % (k), + ) + + output_file = os.path.join(tmp_dir, "output.bin") test_cli("fec_decode", share_files[k_shares:] + ["--output=%s" % (output_file)]) test_cli("hash", ["--no-fsname", output_file], exp_hash) + def cli_pk_workfactor_tests(_tmp_dir): test_cli("pk_workfactor", "1024", "80") test_cli("pk_workfactor", "2048", "111") @@ -911,25 +1226,23 @@ def cli_pk_workfactor_tests(_tmp_dir): test_cli("pk_workfactor", ["--type=dl", "512"], "58") test_cli("pk_workfactor", ["--type=dl_exp", "512"], "192") -def cli_dl_group_info_tests(_tmp_dir): - dl_output = re.compile('(P|G) = [A-F0-9]+') +def cli_dl_group_info_tests(_tmp_dir): + dl_output = re.compile("(P|G) = [A-F0-9]+") for bits in [1024, 1536, 2048, 3072, 4096, 6144, 8192]: output = test_cli("dl_group_info", "modp/ietf/%d" % (bits)) - lines = output.split('\n') + lines = output.split("\n") if len(lines) != 2: - logging.error('Unexpected output from dl_group_info') + logging.error("Unexpected output from dl_group_info") for line in lines: if not dl_output.match(line): - logging.error('Unexpected output from dl_group_info') - + logging.error("Unexpected output from dl_group_info") def cli_ec_group_info_tests(_tmp_dir): - secp256r1_info = """P = 0xFFFFFFFF00000001000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFF A = 0xFFFFFFFF00000001000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFC B = 0x5AC635D8AA3A93E7B3EBBD55769886BC651D06B0CC53B0F63BCE3C3E27D2604B @@ -943,76 +1256,124 @@ def cli_ec_group_info_tests(_tmp_dir): test_cli("ec_group_info", "secp256r1", secp256r1_info) test_cli("ec_group_info", "--pem secp256r1", secp256r1_pem) + def cli_cpuid_tests(_tmp_dir): def read_flags(cli_output): - if not cpuid_output.startswith('CPUID flags:'): + if not cpuid_output.startswith("CPUID flags:"): logging.error('Unexpected cpuid output "%s"', cpuid_output) - return cli_output[13:].split(' ') if cli_output[13:] != '' else [] + return cli_output[13:].split(" ") if cli_output[13:] != "" else [] cpuid_output = test_cli("cpuid", []) - flag_re = re.compile('[a-z0-9_]+') + flag_re = re.compile("[a-z0-9_]+") flags = read_flags(cpuid_output) for flag in flags: - if flag != '' and flag_re.match(flag) is None: + if flag != "" and flag_re.match(flag) is None: logging.error('Unexpected CPUID flag name "%s"', flag) - env = {'BOTAN_CLEAR_CPUID': flag} + env = {"BOTAN_CLEAR_CPUID": flag} cpuid_output = test_cli("cpuid", [], None, None, None, True, env) mod_flags = read_flags(cpuid_output) for f in mod_flags: if f == flag: - logging.error('Clearing CPUID %s did not disable it', flag) + logging.error("Clearing CPUID %s did not disable it", flag) if f not in flags: - logging.error('Clearing CPUID %s caused flag %s to appear', flag, f) + logging.error("Clearing CPUID %s caused flag %s to appear", flag, f) + def cli_cc_enc_tests(_tmp_dir): test_cli("cc_encrypt", ["8028028028028029", "pass"], "4308989841607208") test_cli("cc_decrypt", ["4308989841607208", "pass"], "8028028028028027") + def cli_cert_issuance_tests(tmp_dir, algos=None): - root_key = os.path.join(tmp_dir, 'root.key') - root_crt = os.path.join(tmp_dir, 'root.crt') - int_key = os.path.join(tmp_dir, 'int.key') - int_crt = os.path.join(tmp_dir, 'int.crt') - int_csr = os.path.join(tmp_dir, 'int.csr') - leaf_key = os.path.join(tmp_dir, 'leaf.key') - leaf_crt = os.path.join(tmp_dir, 'leaf.crt') - leaf_csr = os.path.join(tmp_dir, 'leaf.csr') + root_key = os.path.join(tmp_dir, "root.key") + root_crt = os.path.join(tmp_dir, "root.crt") + int_key = os.path.join(tmp_dir, "int.key") + int_crt = os.path.join(tmp_dir, "int.crt") + int_csr = os.path.join(tmp_dir, "int.csr") + leaf_key = os.path.join(tmp_dir, "leaf.key") + leaf_crt = os.path.join(tmp_dir, "leaf.crt") + leaf_csr = os.path.join(tmp_dir, "leaf.csr") if algos is None: - algos = [("RSA", "2048"),("RSA", "2048"),("RSA", "2048")] - - test_cli("keygen", ["--algo=%s" % algos[0][0], "--params=%s" % algos[0][1], "--output=" + root_key], "") - test_cli("keygen", ["--algo=%s" % algos[1][0], "--params=%s" % algos[1][1], "--output=" + int_key], "") - test_cli("keygen", ["--algo=%s" % algos[2][0], "--params=%s" % algos[2][1], "--output=" + leaf_key], "") - - test_cli("gen_self_signed", - [root_key, "Root", "--ca", "--path-limit=2", "--output="+root_crt], "") + algos = [("RSA", "2048"), ("RSA", "2048"), ("RSA", "2048")] + + test_cli( + "keygen", + [ + "--algo=%s" % algos[0][0], + "--params=%s" % algos[0][1], + "--output=" + root_key, + ], + "", + ) + test_cli( + "keygen", + ["--algo=%s" % algos[1][0], "--params=%s" % algos[1][1], "--output=" + int_key], + "", + ) + test_cli( + "keygen", + [ + "--algo=%s" % algos[2][0], + "--params=%s" % algos[2][1], + "--output=" + leaf_key, + ], + "", + ) + + test_cli( + "gen_self_signed", + [root_key, "Root", "--ca", "--path-limit=2", "--output=" + root_crt], + "", + ) test_cli("gen_pkcs10", "%s Intermediate --ca --output=%s" % (int_key, int_csr)) - test_cli("sign_cert", "%s %s %s --output=%s" % (root_crt, root_key, int_csr, int_crt)) + test_cli( + "sign_cert", "%s %s %s --output=%s" % (root_crt, root_key, int_csr, int_crt) + ) test_cli("gen_pkcs10", "%s Leaf --output=%s" % (leaf_key, leaf_csr)) - test_cli("sign_cert", "%s %s %s --output=%s" % (int_crt, int_key, leaf_csr, leaf_crt)) + test_cli( + "sign_cert", "%s %s %s --output=%s" % (int_crt, int_key, leaf_csr, leaf_crt) + ) + + test_cli( + "cert_verify", + "%s %s %s" % (leaf_crt, int_crt, root_crt), + "Certificate passes validation checks", + ) - test_cli("cert_verify", "%s %s %s" % (leaf_crt, int_crt, root_crt), "Certificate passes validation checks") def cli_cert_issuance_alternative_algos_tests(tmp_dir): - for i, algo in enumerate([[("Dilithium", "Dilithium-8x7-AES-r3"), ("Dilithium", "Dilithium-8x7-AES-r3"), ("Dilithium", "Dilithium-8x7-AES-r3")], - [("ECDSA", "secp256r1"), ("ECDSA", "secp384r1"), ("ECDSA", "secp256r1")], - [("Dilithium", "Dilithium-6x5-r3"), ("ECDSA", "secp256r1"), ("RSA", "2048")]]): + for i, algo in enumerate( + [ + [ + ("Dilithium", "Dilithium-8x7-AES-r3"), + ("Dilithium", "Dilithium-8x7-AES-r3"), + ("Dilithium", "Dilithium-8x7-AES-r3"), + ], + [("ECDSA", "secp256r1"), ("ECDSA", "secp384r1"), ("ECDSA", "secp256r1")], + [ + ("Dilithium", "Dilithium-6x5-r3"), + ("ECDSA", "secp256r1"), + ("RSA", "2048"), + ], + ] + ): sub_tmp_dir = os.path.join(tmp_dir, str(i)) os.mkdir(sub_tmp_dir) cli_cert_issuance_tests(sub_tmp_dir, algo) + def cli_marvin_tests(tmp_dir): if not check_for_command("marvin_test"): return - rsa_key = os.path.join(tmp_dir, 'rsa.pem') - data_dir = os.path.join(tmp_dir, 'testcases') + rsa_key = os.path.join(tmp_dir, "rsa.pem") + data_dir = os.path.join(tmp_dir, "testcases") test_cli("keygen", ["--algo=RSA", "--params=1024", "--output=" + rsa_key], "") @@ -1027,15 +1388,15 @@ def cli_marvin_tests(tmp_dir): output_file = os.path.join(data_dir, "invalid%d" % i) ctext = bytes([i] * 128) - with open(output_file, 'bw') as out: + with open(output_file, "bw") as out: out.write(ctext) output = test_cli("marvin_test", [rsa_key, data_dir, "--runs=%d" % (runs)]) first_line = True total_lines = 0 - for line in output.split('\n'): - res = line.split(',') + for line in output.split("\n"): + res = line.split(",") if len(res) != test_inputs: logging.error("Unexpected output from MARVIN test: %s", line) @@ -1053,41 +1414,61 @@ def cli_marvin_tests(tmp_dir): if total_lines != runs + 1: logging.error("Unexpected number of lines from MARVIN test") -def cli_timing_test_tests(_tmp_dir): - timing_tests = ["bleichenbacher", "manger", - "ecdsa", "ecc_mul", "inverse_mod", "pow_mod", - "lucky13sec3", "lucky13sec4sha1", - "lucky13sec4sha256", "lucky13sec4sha384"] +def cli_timing_test_tests(_tmp_dir): + timing_tests = [ + "bleichenbacher", + "manger", + "ecdsa", + "ecc_mul", + "inverse_mod", + "pow_mod", + "lucky13sec3", + "lucky13sec4sha1", + "lucky13sec4sha256", + "lucky13sec4sha384", + ] - output_re = re.compile('[0-9]+;[0-9];[0-9]+') + output_re = re.compile("[0-9]+;[0-9];[0-9]+") for suite in timing_tests: - output = test_cli("timing_test", [suite, "--measurement-runs=16", "--warmup-runs=3", "--test-data-dir=%s" % TEST_DATA_DIR], None).split('\n') + output = test_cli( + "timing_test", + [ + suite, + "--measurement-runs=16", + "--warmup-runs=3", + "--test-data-dir=%s" % TEST_DATA_DIR, + ], + None, + ).split("\n") for line in output: if output_re.match(line) is None: logging.error("Unexpected output in timing_test %s: %s", suite, line) + def cli_tls_ciphersuite_tests(_tmp_dir): - policies = ['default', 'suiteb_128', 'suiteb_192', 'strict', 'all'] + policies = ["default", "suiteb_128", "suiteb_192", "strict", "all"] - versions = ['tls1.2'] + versions = ["tls1.2"] - ciphersuite_re = re.compile('^[A-Z0-9_]+$') + ciphersuite_re = re.compile("^[A-Z0-9_]+$") for policy in policies: for version in versions: - - if version != 'tls1.2' and policy != 'all': + if version != "tls1.2" and policy != "all": continue - output = test_cli("tls_ciphers", ["--version=" + version, "--policy=" + policy], None).split('\n') + output = test_cli( + "tls_ciphers", ["--version=" + version, "--policy=" + policy], None + ).split("\n") for line in output: if ciphersuite_re.match(line) is None: logging.error("Unexpected ciphersuite line %s", line) + def cli_asn1_tests(_tmp_dir): input_pem = """-----BEGIN BLOB----- MCACAQUTBnN0cmluZzEGAQH/AgFjBAUAAAAAAAMEAP///w== @@ -1105,12 +1486,25 @@ def cli_asn1_tests(_tmp_dir): test_cli("asn1print", "--pem -", expected, input_pem) - test_cli("oid_info", "RSA", "The string 'RSA' is associated with OID 1.2.840.113549.1.1.1") - test_cli("oid_info", "1.2.840.113549.1.1.1", "OID 1.2.840.113549.1.1.1 is associated with RSA") + test_cli( + "oid_info", + "RSA", + "The string 'RSA' is associated with OID 1.2.840.113549.1.1.1", + ) + test_cli( + "oid_info", + "1.2.840.113549.1.1.1", + "OID 1.2.840.113549.1.1.1 is associated with RSA", + ) test_cli("oid_info", "1.2.3.4", "OID 1.2.3.4 is not recognized") + def cli_tls_socket_tests(tmp_dir): - if not run_socket_tests() or not check_for_command("tls_client") or not check_for_command("tls_server"): + if ( + not run_socket_tests() + or not check_for_command("tls_client") + or not check_for_command("tls_server") + ): return psk = "FEEDFACECAFEBEEF" @@ -1131,18 +1525,28 @@ def __init__(self, name, protocol_version, policy, **kwargs): configs = [ # Explicitly testing x448-based key exchange against ourselves, as Bogo test # don't cover that. Better than nothing... - TestConfig("x448", "1.2", "allow_tls12=true\nallow_tls13=false\nkey_exchange_groups=x448"), - TestConfig("x448", "1.3", "allow_tls12=false\nallow_tls13=true\nkey_exchange_groups=x448"), - + TestConfig( + "x448", + "1.2", + "allow_tls12=true\nallow_tls13=false\nkey_exchange_groups=x448", + ), + TestConfig( + "x448", + "1.3", + "allow_tls12=false\nallow_tls13=true\nkey_exchange_groups=x448", + ), # Regression test: TLS 1.3 server hit an assertion when no certificate # chain was found. Here, we provoke this by requiring # an RSA-based certificate (server uses ECDSA). - TestConfig("No server cert", "1.3", "allow_tls12=false\nallow_tls13=true\nsignature_methods=RSA\n", - stdout_regex='Alert: handshake_failure', expect_error=True), - + TestConfig( + "No server cert", + "1.3", + "allow_tls12=false\nallow_tls13=true\nsignature_methods=RSA\n", + stdout_regex="Alert: handshake_failure", + expect_error=True, + ), TestConfig("TLS 1.3", "1.3", "allow_tls12=false\nallow_tls13=true\n"), TestConfig("TLS 1.2", "1.2", "allow_tls12=true\nallow_tls13=false\n"), - # At the moment, TLS 1.2 does not implement record_size_limit. # Therefore, clients can offer it only with TLS 1.2 being disabled. # Otherwise, a server negotiating TLS 1.2 and using the record_size_limit @@ -1150,18 +1554,38 @@ def __init__(self, name, protocol_version, policy, **kwargs): # # TODO: Remove this crutch after implementing record_size_limit for TLS 1.2 # and extend the test to use it for both TLS 1.2 and 1.3. - TestConfig("Record size limit", "1.3", "allow_tls12=false\nallow_tls13=true\nrecord_size_limit=64\n"), - - TestConfig("PSK TLS 1.2", "1.2", "allow_tls12=true\nallow_tls13=false\nkey_exchange_methods=ECDHE_PSK\n", - psk=psk, psk_identity=psk_identity, - stdout_regex=f'Handshake complete, TLS v1\\.2.*\nUtilized PSK identity: {psk_identity}.*'), - - TestConfig("PSK TLS 1.3", "1.3", "allow_tls12=false\nallow_tls13=true\nkey_exchange_methods=ECDHE_PSK\n", - psk=psk, psk_identity=psk_identity, psk_prf=psk_prf, - stdout_regex=f'Handshake complete, TLS v1\\.3.*\nUtilized PSK identity: {psk_identity}.*'), - - TestConfig("Kyber KEM", "1.3", "allow_tls12=false\nallow_tls13=true\nkey_exchange_groups=ML-KEM-768"), - TestConfig("Hybrid PQ/T", "1.3", "allow_tls12=false\nallow_tls13=true\nkey_exchange_groups=x25519/ML-KEM-768"), + TestConfig( + "Record size limit", + "1.3", + "allow_tls12=false\nallow_tls13=true\nrecord_size_limit=64\n", + ), + TestConfig( + "PSK TLS 1.2", + "1.2", + "allow_tls12=true\nallow_tls13=false\nkey_exchange_methods=ECDHE_PSK\n", + psk=psk, + psk_identity=psk_identity, + stdout_regex=f"Handshake complete, TLS v1\\.2.*\nUtilized PSK identity: {psk_identity}.*", + ), + TestConfig( + "PSK TLS 1.3", + "1.3", + "allow_tls12=false\nallow_tls13=true\nkey_exchange_methods=ECDHE_PSK\n", + psk=psk, + psk_identity=psk_identity, + psk_prf=psk_prf, + stdout_regex=f"Handshake complete, TLS v1\\.3.*\nUtilized PSK identity: {psk_identity}.*", + ), + TestConfig( + "Kyber KEM", + "1.3", + "allow_tls12=false\nallow_tls13=true\nkey_exchange_groups=ML-KEM-768", + ), + TestConfig( + "Hybrid PQ/T", + "1.3", + "allow_tls12=false\nallow_tls13=true\nkey_exchange_groups=x25519/ML-KEM-768", + ), ] class TestServer(AsyncTestProcess): @@ -1174,24 +1598,33 @@ def __init__(self, tmp_dir, port, psk, psk_identity, psk_prf, clients=0): self.clients = clients self.cert_suite = ServerCertificateSuite(tmp_dir, "secp256r1", "SHA-384") - self.policy = os.path.join(tmp_dir, 'test_server_policy.txt') + self.policy = os.path.join(tmp_dir, "test_server_policy.txt") - with open(self.policy, 'w', encoding='utf8') as f: - f.write('key_exchange_methods = ECDH DH ECDHE_PSK\n') - f.write("key_exchange_groups = x25519 x448 secp256r1 ffdhe/ietf/2048 ML-KEM-768 x25519/ML-KEM-768") + with open(self.policy, "w", encoding="utf8") as f: + f.write("key_exchange_methods = ECDH DH ECDHE_PSK\n") + f.write( + "key_exchange_groups = x25519 x448 secp256r1 ffdhe/ietf/2048 ML-KEM-768 x25519/ML-KEM-768" + ) @property def ca_cert(self): return self.cert_suite.ca_cert async def __aenter__(self): - server_cmd = [CLI_PATH, "tls_server", f"--max-clients={self.clients}", - f"--port={self.port}", f"--policy={self.policy}", - f"--psk={self.psk}", f"--psk-identity={self.psk_identity}", - f"--psk-prf={self.psk_prf}", - self.cert_suite.cert, self.cert_suite.private_key] - - await self._launch(server_cmd, b'Listening for new connections') + server_cmd = [ + CLI_PATH, + "tls_server", + f"--max-clients={self.clients}", + f"--port={self.port}", + f"--policy={self.policy}", + f"--psk={self.psk}", + f"--psk-identity={self.psk_identity}", + f"--psk-prf={self.psk_prf}", + self.cert_suite.cert, + self.cert_suite.private_key, + ] + + await self._launch(server_cmd, b"Listening for new connections") return self @@ -1199,17 +1632,20 @@ async def __aexit__(self, *_): await self._finalize() class TestClient(AsyncTestProcess): - client_message = b'Client message %d with extra stuff to test record_size_limit: %s\n' % (random.randint(0, 2**128), b'oO' * 64) + client_message = ( + b"Client message %d with extra stuff to test record_size_limit: %s\n" + % (random.randint(0, 2**128), b"oO" * 64) + ) def __init__(self, tmp_dir, server_port, ca, config): super().__init__("Client") self.tmp_dir = tmp_dir self.port = server_port self.ca = ca - self.policy = os.path.join(tmp_dir, 'test_client_policy.txt') + self.policy = os.path.join(tmp_dir, "test_client_policy.txt") self.config = config - with open(self.policy, 'w', encoding='utf8') as f: + with open(self.policy, "w", encoding="utf8") as f: f.write(self.config.policy) async def perform_message_ping_pong(self): @@ -1221,33 +1657,56 @@ async def perform_message_ping_pong(self): await self._read_stdout_until(TestClient.client_message) # close the client and expect to read stdout until EOF - retcode = await self._close_stdin_read_stdout_to_eof_and_wait_for_termination() + retcode = ( + await self._close_stdin_read_stdout_to_eof_and_wait_for_termination() + ) if retcode != 0: - raise Exception(f'Client failed with error ({self.config.name}): {retcode}') + raise Exception( + f"Client failed with error ({self.config.name}): {retcode}" + ) self._check_stdout_regex() async def expect_handshake_error(self): - retcode = await self._close_stdin_read_stdout_to_eof_and_wait_for_termination() + retcode = ( + await self._close_stdin_read_stdout_to_eof_and_wait_for_termination() + ) if retcode == 0: - raise Exception(f"Expected an error, but tls_client finished with success ({self.config.name})") + raise Exception( + f"Expected an error, but tls_client finished with success ({self.config.name})" + ) self._check_stdout_regex() def _check_stdout_regex(self): if self.config.stdout_regex: match = re.search(self.config.stdout_regex, self.stdout) if not match: - raise Exception(f"Client log did not match expected regex ({self.config.name}): {self.config.stdout_regex}") + raise Exception( + f"Client log did not match expected regex ({self.config.name}): {self.config.stdout_regex}" + ) async def __aenter__(self): - client_cmd = [CLI_PATH, "tls_client", 'localhost', f'--port={self.port}', f'--trusted-cas={self.ca}', - f'--tls-version={self.config.protocol_version}', f'--policy={self.policy}'] + client_cmd = [ + CLI_PATH, + "tls_client", + "localhost", + f"--port={self.port}", + f"--trusted-cas={self.ca}", + f"--tls-version={self.config.protocol_version}", + f"--policy={self.policy}", + ] if self.config.psk: - client_cmd += [f'--psk={self.config.psk}', f'--psk-identity={self.config.psk_identity}'] + client_cmd += [ + f"--psk={self.config.psk}", + f"--psk-identity={self.config.psk_identity}", + ] if self.config.psk_prf: - client_cmd += [f'--psk-prf={self.config.psk_prf}'] + client_cmd += [f"--psk-prf={self.config.psk_prf}"] - await self._launch(client_cmd, b'Handshake complete' if not self.config.expect_error else None) + await self._launch( + client_cmd, + b"Handshake complete" if not self.config.expect_error else None, + ) return self @@ -1255,11 +1714,19 @@ async def __aexit__(self, *_): await self._finalize() async def run_async_test(): - async with TestServer(tmp_dir, port_for('tls_server'), psk, psk_identity, psk_prf, len(configs)) as server: + async with TestServer( + tmp_dir, port_for("tls_server"), psk, psk_identity, psk_prf, len(configs) + ) as server: errors = 0 for tls_config in configs: - logging.debug("Running test for %s in TLS %s mode", tls_config.name, tls_config.protocol_version) - async with TestClient(tmp_dir, server.port, server.ca_cert, tls_config) as client: + logging.debug( + "Running test for %s in TLS %s mode", + tls_config.name, + tls_config.protocol_version, + ) + async with TestClient( + tmp_dir, server.port, server.ca_cert, tls_config + ) as client: try: if tls_config.expect_error: await client.expect_handshake_error() @@ -1274,8 +1741,13 @@ async def run_async_test(): asyncio.run(run_async_test()) + def cli_tls_online_pqc_hybrid_tests(tmp_dir): - if not run_socket_tests() or not run_online_tests() or not check_for_command("tls_client"): + if ( + not run_socket_tests() + or not run_online_tests() + or not check_for_command("tls_client") + ): return class TestConfig: @@ -1289,15 +1761,25 @@ def __init__(self, host, kex_algo, port=443, ca=None): self.ca_file = None def setup(self, tmp_dir): - self.policy_file = tempfile.NamedTemporaryFile(dir=tmp_dir, mode="w+", encoding="utf-8") - self.policy_file.write('\n'.join(["allow_tls13 = true", - "allow_tls12 = false", - "key_exchange_methods = HYBRID KEM", - f"key_exchange_groups = {self.kex_algo}"])) + self.policy_file = tempfile.NamedTemporaryFile( + dir=tmp_dir, mode="w+", encoding="utf-8" + ) + self.policy_file.write( + "\n".join( + [ + "allow_tls13 = true", + "allow_tls12 = false", + "key_exchange_methods = HYBRID KEM", + f"key_exchange_groups = {self.kex_algo}", + ] + ) + ) self.policy_file.flush() if self.ca: - self.ca_file = tempfile.NamedTemporaryFile(dir=tmp_dir, mode="w+", encoding="utf-8") + self.ca_file = tempfile.NamedTemporaryFile( + dir=tmp_dir, mode="w+", encoding="utf-8" + ) self.ca_file.write(self.ca) self.ca_file.flush() @@ -1324,7 +1806,7 @@ def get_oqs_resource(resource): def get_oqs_ports(): try: - return json.loads(get_oqs_resource("/assignments.json"))['ecdsap256'] + return json.loads(get_oqs_resource("/assignments.json"))["ecdsap256"] except Exception: return None @@ -1341,31 +1823,135 @@ def get_oqs_rootca(): if oqsp and oqs_test_ca: # src/scripts/test_cli.py --run-online-tests ./botan pqc_hybrid_tests test_cfg += [ - TestConfig("test.openquantumsafe.org", "x25519/ML-KEM-768", port=oqsp['X25519MLKEM768'], ca=oqs_test_ca), - TestConfig("test.openquantumsafe.org", "secp256r1/ML-KEM-768", port=oqsp['SecP256r1MLKEM768'], ca=oqs_test_ca), - TestConfig("test.openquantumsafe.org", "ML-KEM-512", port=oqsp['mlkem512'], ca=oqs_test_ca), - TestConfig("test.openquantumsafe.org", "ML-KEM-768", port=oqsp['mlkem768'], ca=oqs_test_ca), - TestConfig("test.openquantumsafe.org", "ML-KEM-1024", port=oqsp['mlkem1024'], ca=oqs_test_ca), - + TestConfig( + "test.openquantumsafe.org", + "x25519/ML-KEM-768", + port=oqsp["X25519MLKEM768"], + ca=oqs_test_ca, + ), + TestConfig( + "test.openquantumsafe.org", + "secp256r1/ML-KEM-768", + port=oqsp["SecP256r1MLKEM768"], + ca=oqs_test_ca, + ), + TestConfig( + "test.openquantumsafe.org", + "ML-KEM-512", + port=oqsp["mlkem512"], + ca=oqs_test_ca, + ), + TestConfig( + "test.openquantumsafe.org", + "ML-KEM-768", + port=oqsp["mlkem768"], + ca=oqs_test_ca, + ), + TestConfig( + "test.openquantumsafe.org", + "ML-KEM-1024", + port=oqsp["mlkem1024"], + ca=oqs_test_ca, + ), # We track OQS's code point allocations for FrodoKEM and hybrids thereof. # All are defined in TLS's private code point section (0xFE00 - 0xFFFF) # and may change in the future. - TestConfig("test.openquantumsafe.org", "eFrodoKEM-640-SHAKE", port=oqsp['frodo640shake'], ca=oqs_test_ca), - TestConfig("test.openquantumsafe.org", "eFrodoKEM-976-SHAKE", port=oqsp['frodo976shake'], ca=oqs_test_ca), - TestConfig("test.openquantumsafe.org", "eFrodoKEM-1344-SHAKE", port=oqsp['frodo1344shake'], ca=oqs_test_ca), - TestConfig("test.openquantumsafe.org", "eFrodoKEM-640-AES", port=oqsp['frodo640aes'], ca=oqs_test_ca), - TestConfig("test.openquantumsafe.org", "eFrodoKEM-976-AES", port=oqsp['frodo976aes'], ca=oqs_test_ca), - TestConfig("test.openquantumsafe.org", "eFrodoKEM-1344-AES", port=oqsp['frodo1344aes'], ca=oqs_test_ca), - TestConfig("test.openquantumsafe.org", "x25519/eFrodoKEM-640-SHAKE", port=oqsp['x25519_frodo640shake'], ca=oqs_test_ca), - TestConfig("test.openquantumsafe.org", "x25519/eFrodoKEM-640-AES", port=oqsp['x25519_frodo640aes'], ca=oqs_test_ca), - TestConfig("test.openquantumsafe.org", "x448/eFrodoKEM-976-SHAKE", port=oqsp['x448_frodo976shake'], ca=oqs_test_ca), - TestConfig("test.openquantumsafe.org", "x448/eFrodoKEM-976-AES", port=oqsp['x448_frodo976aes'], ca=oqs_test_ca), - TestConfig("test.openquantumsafe.org", "secp256r1/eFrodoKEM-640-SHAKE", port=oqsp['p256_frodo640shake'], ca=oqs_test_ca), - TestConfig("test.openquantumsafe.org", "secp256r1/eFrodoKEM-640-AES", port=oqsp['p256_frodo640aes'], ca=oqs_test_ca), - TestConfig("test.openquantumsafe.org", "secp384r1/eFrodoKEM-976-SHAKE", port=oqsp['p384_frodo976shake'], ca=oqs_test_ca), - TestConfig("test.openquantumsafe.org", "secp384r1/eFrodoKEM-976-AES", port=oqsp['p384_frodo976aes'], ca=oqs_test_ca), - TestConfig("test.openquantumsafe.org", "secp521r1/eFrodoKEM-1344-SHAKE", port=oqsp['p521_frodo1344shake'], ca=oqs_test_ca), - TestConfig("test.openquantumsafe.org", "secp521r1/eFrodoKEM-1344-AES", port=oqsp['p521_frodo1344aes'], ca=oqs_test_ca), + TestConfig( + "test.openquantumsafe.org", + "eFrodoKEM-640-SHAKE", + port=oqsp["frodo640shake"], + ca=oqs_test_ca, + ), + TestConfig( + "test.openquantumsafe.org", + "eFrodoKEM-976-SHAKE", + port=oqsp["frodo976shake"], + ca=oqs_test_ca, + ), + TestConfig( + "test.openquantumsafe.org", + "eFrodoKEM-1344-SHAKE", + port=oqsp["frodo1344shake"], + ca=oqs_test_ca, + ), + TestConfig( + "test.openquantumsafe.org", + "eFrodoKEM-640-AES", + port=oqsp["frodo640aes"], + ca=oqs_test_ca, + ), + TestConfig( + "test.openquantumsafe.org", + "eFrodoKEM-976-AES", + port=oqsp["frodo976aes"], + ca=oqs_test_ca, + ), + TestConfig( + "test.openquantumsafe.org", + "eFrodoKEM-1344-AES", + port=oqsp["frodo1344aes"], + ca=oqs_test_ca, + ), + TestConfig( + "test.openquantumsafe.org", + "x25519/eFrodoKEM-640-SHAKE", + port=oqsp["x25519_frodo640shake"], + ca=oqs_test_ca, + ), + TestConfig( + "test.openquantumsafe.org", + "x25519/eFrodoKEM-640-AES", + port=oqsp["x25519_frodo640aes"], + ca=oqs_test_ca, + ), + TestConfig( + "test.openquantumsafe.org", + "x448/eFrodoKEM-976-SHAKE", + port=oqsp["x448_frodo976shake"], + ca=oqs_test_ca, + ), + TestConfig( + "test.openquantumsafe.org", + "x448/eFrodoKEM-976-AES", + port=oqsp["x448_frodo976aes"], + ca=oqs_test_ca, + ), + TestConfig( + "test.openquantumsafe.org", + "secp256r1/eFrodoKEM-640-SHAKE", + port=oqsp["p256_frodo640shake"], + ca=oqs_test_ca, + ), + TestConfig( + "test.openquantumsafe.org", + "secp256r1/eFrodoKEM-640-AES", + port=oqsp["p256_frodo640aes"], + ca=oqs_test_ca, + ), + TestConfig( + "test.openquantumsafe.org", + "secp384r1/eFrodoKEM-976-SHAKE", + port=oqsp["p384_frodo976shake"], + ca=oqs_test_ca, + ), + TestConfig( + "test.openquantumsafe.org", + "secp384r1/eFrodoKEM-976-AES", + port=oqsp["p384_frodo976aes"], + ca=oqs_test_ca, + ), + TestConfig( + "test.openquantumsafe.org", + "secp521r1/eFrodoKEM-1344-SHAKE", + port=oqsp["p521_frodo1344shake"], + ca=oqs_test_ca, + ), + TestConfig( + "test.openquantumsafe.org", + "secp521r1/eFrodoKEM-1344-AES", + port=oqsp["p521_frodo1344aes"], + ca=oqs_test_ca, + ), ] else: logging.info("failed to pull OQS port assignment, skipping OQS...") @@ -1374,14 +1960,19 @@ def get_oqs_rootca(): cfg.setup(tmp_dir) stdout = cfg.run() if "Handshake complete" not in stdout: - logging.error('Failed to complete handshake (%s with %s): %s', cfg.host, cfg.kex_algo, stdout) + logging.error( + "Failed to complete handshake (%s with %s): %s", + cfg.host, + cfg.kex_algo, + stdout, + ) def cli_tls_http_server_tests(tmp_dir): if not run_socket_tests() or not check_for_command("tls_http_server"): return - server_port = port_for('tls_http_server') + server_port = port_for("tls_http_server") class BotanHttpServer(AsyncTestProcess): def __init__(self, tmp_dir, port, clients=0): @@ -1395,10 +1986,16 @@ def ca_cert(self): return self.cert_suite.ca_cert async def __aenter__(self): - server_cmd = [CLI_PATH, 'tls_http_server', f'--port={self.port}', f'--max-clients={self.clients}', - self.cert_suite.cert, self.cert_suite.private_key] + server_cmd = [ + CLI_PATH, + "tls_http_server", + f"--port={self.port}", + f"--max-clients={self.clients}", + self.cert_suite.cert, + self.cert_suite.private_key, + ] - await self._launch(server_cmd, b'Listening for new connections') + await self._launch(server_cmd, b"Listening for new connections") return self @@ -1412,35 +2009,40 @@ async def run_async_test(): context.minimum_version = tls_version context.maximum_version = tls_version - conn = HTTPSConnection('localhost', port=server_port, context=context) + conn = HTTPSConnection("localhost", port=server_port, context=context) conn.request("GET", "/", headers={"Connection": "close"}) resp = conn.getresponse() if resp.status != 200: - logging.error('Unexpected response status %d', resp.status) + logging.error("Unexpected response status %d", resp.status) body = str(resp.read()) - if body.find('TLS negotiation with Botan 3.') < 0: - logging.error('Unexpected response body %s', body) + if body.find("TLS negotiation with Botan 3.") < 0: + logging.error("Unexpected response body %s", body) conn.request("POST", "/logout", headers={"Connection": "close"}) resp = conn.getresponse() if resp.status != 405: - logging.error('Unexpected response status %d', resp.status) + logging.error("Unexpected response status %d", resp.status) asyncio.run(run_async_test()) + def cli_tls_proxy_tests(tmp_dir): # This was disabled in GH #3845 due to flakiness, then thought possibly # fixed and enabled again in Gh #4178. However the test still occasionally # fails. Disable it again pending diagnosis... - if not run_socket_tests() or platform.system() == 'Windows' or not check_for_command("tls_proxy"): + if ( + not run_socket_tests() + or platform.system() == "Windows" + or not check_for_command("tls_proxy") + ): return - server_port = port_for('tls_proxy_backend') - proxy_port = port_for('tls_proxy') + server_port = port_for("tls_proxy_backend") + proxy_port = port_for("tls_proxy") max_clients = 4 server_response = binascii.hexlify(os.urandom(32)) @@ -1459,10 +2061,18 @@ def ca_cert(self): return self.cert_suite.ca_cert async def __aenter__(self): - proxy_cmd = [CLI_PATH, 'tls_proxy', str(proxy_port), '127.0.0.1', str(server_port), - self.cert_suite.cert, self.cert_suite.private_key, f'--max-clients={self.clients}'] - - await self._launch(proxy_cmd, b'Listening for new connections') + proxy_cmd = [ + CLI_PATH, + "tls_proxy", + str(proxy_port), + "127.0.0.1", + str(server_port), + self.cert_suite.cert, + self.cert_suite.private_key, + f"--max-clients={self.clients}", + ] + + await self._launch(proxy_cmd, b"Listening for new connections") return self @@ -1471,15 +2081,15 @@ async def __aexit__(self, *_): def run_http_server(): class Handler(BaseHTTPRequestHandler): - def log_message(self, _fmt, *_args): # pylint: disable=arguments-differ + def log_message(self, _fmt, *_args): # pylint: disable=arguments-differ pass # muzzle log output - def do_GET(self): # pylint: disable=invalid-name + def do_GET(self): # pylint: disable=invalid-name self.send_response(200) self.end_headers() self.wfile.write(server_response) - httpd = HTTPServer(('', server_port), Handler) + httpd = HTTPServer(("", server_port), Handler) httpd.serve_forever() http_thread = threading.Thread(target=run_http_server) @@ -1498,36 +2108,40 @@ async def run_async_test(): context.minimum_version = ssl.TLSVersion.TLSv1_2 context.maximum_version = ssl.TLSVersion.TLSv1_2 - conn = HTTPSConnection('localhost', port=proxy_port, context=context, timeout=20) + conn = HTTPSConnection( + "localhost", port=proxy_port, context=context, timeout=20 + ) conn.request("GET", "/") resp = conn.getresponse() if resp.status != 200: - logging.error('Unexpected response status %d', resp.status) + logging.error("Unexpected response status %d", resp.status) body = resp.read() if body != server_response: - logging.error('Unexpected response from server %s', body) + logging.error("Unexpected response from server %s", body) asyncio.run(run_async_test()) + def cli_trust_root_tests(tmp_dir): - pem_file = os.path.join(tmp_dir, 'pems') - dn_file = os.path.join(tmp_dir, 'dns') + pem_file = os.path.join(tmp_dir, "pems") + dn_file = os.path.join(tmp_dir, "dns") - test_cli("trust_roots", ['--dn-only', '--output=%s' % (dn_file)], "") + test_cli("trust_roots", ["--dn-only", "--output=%s" % (dn_file)], "") - dn_re = re.compile('(.+=\".+\")(,.+=\".+\")?') + dn_re = re.compile('(.+=".+")(,.+=".+")?') - for line in open(dn_file, encoding='utf8'): + for line in open(dn_file, encoding="utf8"): if dn_re.match(line) is None: logging.error("Unexpected DN line %s", line) - test_cli("trust_roots", ['--output=%s' % (pem_file)], "") + test_cli("trust_roots", ["--output=%s" % (pem_file)], "") + def cli_tss_tests(tmp_dir): - data_file = os.path.join(tmp_dir, 'data') + data_file = os.path.join(tmp_dir, "data") exp_hash = "53B3C59276AE30EA7FD882268E80FD96AD80CC9FEB15F9FB940E7C4B5CF80B9E" @@ -1537,11 +2151,15 @@ def cli_tss_tests(tmp_dir): m = 3 n = 5 - test_cli("tss_split", [str(m), str(n), data_file, "--share-prefix=%s/split" % (tmp_dir)], "") + test_cli( + "tss_split", + [str(m), str(n), data_file, "--share-prefix=%s/split" % (tmp_dir)], + "", + ) share_files = [] - for i in range(1, n+1): + for i in range(1, n + 1): share = os.path.join(tmp_dir, "split%d.tss" % (i)) if not os.access(share, os.R_OK): logging.error("Failed to create expected split file %s", share) @@ -1560,23 +2178,34 @@ def cli_tss_tests(tmp_dir): test_cli("hash", ["--no-fsname", rec3], exp_hash) rec2 = os.path.join(tmp_dir, "recovered_2") - test_cli("tss_recover", share_files[3:] + ["--output=%s" % (rec2)], "", None, - "Error: Insufficient shares to do TSS reconstruction") + test_cli( + "tss_recover", + share_files[3:] + ["--output=%s" % (rec2)], + "", + None, + "Error: Insufficient shares to do TSS reconstruction", + ) def cli_pk_encrypt_tests(tmp_dir): - input_file = os.path.join(tmp_dir, 'input') - ctext_file = os.path.join(tmp_dir, 'ctext') - recovered_file = os.path.join(tmp_dir, 'recovered') - rsa_priv_key = os.path.join(tmp_dir, 'rsa.priv') - rsa_pub_key = os.path.join(tmp_dir, 'rsa.pub') + input_file = os.path.join(tmp_dir, "input") + ctext_file = os.path.join(tmp_dir, "ctext") + recovered_file = os.path.join(tmp_dir, "recovered") + rsa_priv_key = os.path.join(tmp_dir, "rsa.priv") + rsa_pub_key = os.path.join(tmp_dir, "rsa.pub") - test_cli("keygen", ["--algo=RSA", "--params=2048", "--output=%s" % (rsa_priv_key)], "") + test_cli( + "keygen", ["--algo=RSA", "--params=2048", "--output=%s" % (rsa_priv_key)], "" + ) key_hash = "D1621B7D1272545F8CCC220BC7F6F5BAF0150303B19299F0C5B79C095B3CDFC0" test_cli("hash", ["--no-fsname", "--algo=SHA-256", rsa_priv_key], key_hash) - test_cli("pkcs8", ["--pub-out", "%s/rsa.priv" % (tmp_dir), "--output=%s" % (rsa_pub_key)], "") + test_cli( + "pkcs8", + ["--pub-out", "%s/rsa.priv" % (tmp_dir), "--output=%s" % (rsa_pub_key)], + "", + ) # Generate a random input file test_cli("rng", ["10", "16", "32", "--output=%s" % (input_file)], "") @@ -1592,87 +2221,128 @@ def cli_pk_encrypt_tests(tmp_dir): test_cli("hash", ["--no-fsname", "--algo=SHA-256", ctext_file], ctext_hash) # Decrypt and verify plaintext is recovered - test_cli("pk_decrypt", [rsa_priv_key, ctext_file, "--output=%s" % (recovered_file)], "") + test_cli( + "pk_decrypt", [rsa_priv_key, ctext_file, "--output=%s" % (recovered_file)], "" + ) test_cli("hash", ["--no-fsname", "--algo=SHA-256", recovered_file], rng_output_hash) + def cli_uuid_tests(_tmp_dir): test_cli("uuid", [], "D80F88F6-ADBE-45AC-B10C-3602E67D985B") - uuid_re = re.compile(r'[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}') + uuid_re = re.compile( + r"[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}" + ) output = test_cli("uuid", []) if uuid_re.match(output) is None: - logging.error('Bad uuid output %s', output) + logging.error("Bad uuid output %s", output) -def cli_tls_client_hello_tests(_tmp_dir): +def cli_tls_client_hello_tests(_tmp_dir): chellos = [ # TLS 1.2 - ("6073536D3FA201A37C1F3944F6DCDD5A83FAA67DF4B1C9CBE4FA4399FDE7673C", "16030100cf010000cb03035b3cf2457b864d7bef2a4b1f84fc3ced2b68d9551f3455ffdd305af277a91bb200003a16b816b716ba16b9cca9cca8c02cc030c02bc02fc0adc0acc024c00ac028c014c023c009c027c013ccaa009f009ec09fc09e006b003900670033010000680000000e000c000009676d61696c2e636f6d000500050100000000000a001a0018001d0017001a0018001b0019001c01000101010201030104000b00020100000d00140012080508040806050106010401050306030403001600000017000000230000ff01000100"), - + ( + "6073536D3FA201A37C1F3944F6DCDD5A83FAA67DF4B1C9CBE4FA4399FDE7673C", + "16030100cf010000cb03035b3cf2457b864d7bef2a4b1f84fc3ced2b68d9551f3455ffdd305af277a91bb200003a16b816b716ba16b9cca9cca8c02cc030c02bc02fc0adc0acc024c00ac028c014c023c009c027c013ccaa009f009ec09fc09e006b003900670033010000680000000e000c000009676d61696c2e636f6d000500050100000000000a001a0018001d0017001a0018001b0019001c01000101010201030104000b00020100000d00140012080508040806050106010401050306030403001600000017000000230000ff01000100", + ), # TLS 1.3 - ("4D8BB87026C6AEB1356234A01BD62C7DEFB3FEA298B8C50900F5D3F3ADDAADEB", "1603010106010001020303657033B5C89B0356097C9D43B3917BC0D743E34CB118E1DD3FC806EC9CED2FB120657033B589AD50CADDC8CBA0B805A9841DB3A4F92334C1A44EE968DD4B2983450018130313021301CCA9CCA8C02CC030C02BC02FCCAA009F009E010000A10000000E000C0000096C6F63616C686F7374000A001A0018001D0017001A0018001B0019001C01000101010201030104003300260024001D002002CBD31A5D5754EFD5C8F5152E27302681278A710A22B04403EF9EF0F5F95C1E002B00050403040303000D00140012080508040806050106010401050306030403002D00020101000500050100000000FF01000100002300000017000000160000000B00020100"), + ( + "4D8BB87026C6AEB1356234A01BD62C7DEFB3FEA298B8C50900F5D3F3ADDAADEB", + "1603010106010001020303657033B5C89B0356097C9D43B3917BC0D743E34CB118E1DD3FC806EC9CED2FB120657033B589AD50CADDC8CBA0B805A9841DB3A4F92334C1A44EE968DD4B2983450018130313021301CCA9CCA8C02CC030C02BC02FCCAA009F009E010000A10000000E000C0000096C6F63616C686F7374000A001A0018001D0017001A0018001B0019001C01000101010201030104003300260024001D002002CBD31A5D5754EFD5C8F5152E27302681278A710A22B04403EF9EF0F5F95C1E002B00050403040303000D00140012080508040806050106010401050306030403002D00020101000500050100000000FF01000100002300000017000000160000000B00020100", + ), ] for output_hash, chello in chellos: output = test_cli("tls_client_hello", ["--hex", "-"], None, chello) test_cli("hash", ["--no-fsname", "--algo=SHA-256", "-"], output_hash, output) + def cli_speed_pk_tests(_tmp_dir): msec = 1 - pk_algos = ["ECDSA", "ECDH", "SM2", "ECKCDSA", "ECGDSA", "GOST-34.10", - "DH", "DSA", "ElGamal", "Ed25519", "Ed448", "X25519", "X448", - "RSA", "RSA_keygen", "XMSS", "Kyber", "Dilithium", "SLH-DSA"] + pk_algos = [ + "ECDSA", + "ECDH", + "SM2", + "ECKCDSA", + "ECGDSA", + "GOST-34.10", + "DH", + "DSA", + "ElGamal", + "Ed25519", + "Ed448", + "X25519", + "X448", + "RSA", + "RSA_keygen", + "XMSS", + "Kyber", + "Dilithium", + "SLH-DSA", + ] - output = test_cli("speed", ["--msec=%d" % (msec)] + pk_algos, None).split('\n') + output = test_cli("speed", ["--msec=%d" % (msec)] + pk_algos, None).split("\n") # ECDSA-secp256r1 106 keygen/sec; 9.35 ms/op 37489733 cycles/op (1 op in 9 ms) - format_re = re.compile(r'^.* [0-9]+ ([A-Za-z0-9 ]+)/sec; [0-9]+\.[0-9]+ ms/op .*\([0-9]+ (op|ops) in [0-9\.]+ ms\)') + format_re = re.compile( + r"^.* [0-9]+ ([A-Za-z0-9 ]+)/sec; [0-9]+\.[0-9]+ ms/op .*\([0-9]+ (op|ops) in [0-9\.]+ ms\)" + ) for line in output: if format_re.match(line) is None: logging.error("Unexpected line %s", line) + def cli_speed_pbkdf_tests(_tmp_dir): msec = 1 - pbkdf_ops = ['bcrypt', 'passhash9', 'argon2'] + pbkdf_ops = ["bcrypt", "passhash9", "argon2"] - format_re = re.compile(r'^.* [0-9]+ /sec; [0-9]+\.[0-9]+ ms/op .*\([0-9]+ (op|ops) in [0-9]+(\.[0-9]+)? ms\)') + format_re = re.compile( + r"^.* [0-9]+ /sec; [0-9]+\.[0-9]+ ms/op .*\([0-9]+ (op|ops) in [0-9]+(\.[0-9]+)? ms\)" + ) for op in pbkdf_ops: - output = test_cli("speed", ["--msec=%d" % (msec), op], None).split('\n') + output = test_cli("speed", ["--msec=%d" % (msec), op], None).split("\n") for line in output: if format_re.match(line) is None: logging.error("Unexpected line %s", line) + def cli_speed_table_tests(_tmp_dir): msec = 1 - version_re = re.compile(r'Botan [0-9]\.[0-9]+\.[0-9](\-[a-z]+[0-9]+)?( UNSAFE .* BUILD)? \(.*\)$') - cpuid_re = re.compile(r'^CPUID: [a-z_0-9 ]*$') - format_re = re.compile(r'^.* buffer size [0-9]+ bytes: [0-9]+\.[0-9]+ MiB\/sec .*\([0-9]+\.[0-9]+ MiB in [0-9]+\.[0-9]+ ms\)') - tbl_hdr_re = re.compile(r'^algo +operation +1024 bytes$') - tbl_val_re = re.compile(r'^.* +(encrypt|decrypt) +[0-9]+(\.[0-9]{2})$') + version_re = re.compile( + r"Botan [0-9]\.[0-9]+\.[0-9](\-[a-z]+[0-9]+)?( UNSAFE .* BUILD)? \(.*\)$" + ) + cpuid_re = re.compile(r"^CPUID: [a-z_0-9 ]*$") + format_re = re.compile( + r"^.* buffer size [0-9]+ bytes: [0-9]+\.[0-9]+ MiB\/sec .*\([0-9]+\.[0-9]+ MiB in [0-9]+\.[0-9]+ ms\)" + ) + tbl_hdr_re = re.compile(r"^algo +operation +1024 bytes$") + tbl_val_re = re.compile(r"^.* +(encrypt|decrypt) +[0-9]+(\.[0-9]{2})$") - output = test_cli("speed", ["--format=table", "--msec=%d" % (msec), "AES-128"], None).split('\n') + output = test_cli( + "speed", ["--format=table", "--msec=%d" % (msec), "AES-128"], None + ).split("\n") if len(output) != 11: - logging.error('Unexpected number of lines from table output') + logging.error("Unexpected number of lines from table output") if version_re.match(output[0]) is None: logging.error("Unexpected version line %s", output[0]) - if output[1] != '': + if output[1] != "": if cpuid_re.match(output[1]) is None: logging.error("Unexpected cpuid line %s", output[1]) - elif output[2] != '': + elif output[2] != "": logging.error("Expected newline got %s", output[2]) if format_re.match(output[3]) is None: logging.error("Unexpected line %s", output[3]) if format_re.match(output[4]) is None: logging.error("Unexpected line %s", output[4]) - if output[5] != '': + if output[5] != "": logging.error("Expected newline got %s", output[5]) if tbl_hdr_re.match(output[6]) is None: @@ -1681,86 +2351,138 @@ def cli_speed_table_tests(_tmp_dir): logging.error("Unexpected table header %s", output[7]) if tbl_val_re.match(output[8]) is None: logging.error("Unexpected table header %s", output[8]) - if output[9] != '': + if output[9] != "": logging.error("Expected newline got %s", output[9]) - if output[10].find('results are the number of 1000s bytes processed per second') < 0: + if ( + output[10].find("results are the number of 1000s bytes processed per second") + < 0 + ): logging.error("Unexpected trailing message got %s", output[10]) + def cli_speed_invalid_option_tests(_tmp_dir): speed_usage = "Usage: speed --msec=500 --format=default --time-unit=ms --ecc-groups= --buf-size=1024 --clear-cpuid= --cpu-clock-speed=0 --cpu-clock-ratio=1.0 *algos" - test_cli("speed", ["--buf-size=0", "--msec=1", "AES-128"], - expected_stderr="Usage error: Cannot have a zero-sized buffer\n%s" % (speed_usage)) - - test_cli("speed", ["--buf-size=F00F", "--msec=1", "AES-128"], - expected_stderr="Usage error: Invalid integer value 'F00F' for option buf-size\n%s" % (speed_usage)) - - test_cli("speed", ["--buf-size=90000000", "--msec=1", "AES-128"], - expected_stderr="Usage error: Specified buffer size is too large\n%s" % (speed_usage)) + test_cli( + "speed", + ["--buf-size=0", "--msec=1", "AES-128"], + expected_stderr="Usage error: Cannot have a zero-sized buffer\n%s" + % (speed_usage), + ) + + test_cli( + "speed", + ["--buf-size=F00F", "--msec=1", "AES-128"], + expected_stderr="Usage error: Invalid integer value 'F00F' for option buf-size\n%s" + % (speed_usage), + ) + + test_cli( + "speed", + ["--buf-size=90000000", "--msec=1", "AES-128"], + expected_stderr="Usage error: Specified buffer size is too large\n%s" + % (speed_usage), + ) + + test_cli( + "speed", + ["--clear-cpuid=goku", "--msec=1", "AES-128"], + expected_stderr="Warning don't know CPUID flag 'goku'", + ) - test_cli("speed", ["--clear-cpuid=goku", "--msec=1", "AES-128"], - expected_stderr="Warning don't know CPUID flag 'goku'") def cli_speed_math_tests(_tmp_dir): msec = 1 # these all have a common output format - math_ops = ['mp_mul', 'mp_div', 'mp_div10', 'modexp', 'random_prime', 'inverse_mod', - 'rfc3394', 'fpe_fe1', 'ecdsa_recovery', 'ecc', 'bn_redc', 'primality_test'] + math_ops = [ + "mp_mul", + "mp_div", + "mp_div10", + "modexp", + "random_prime", + "inverse_mod", + "rfc3394", + "fpe_fe1", + "ecdsa_recovery", + "ecc", + "bn_redc", + "primality_test", + ] - format_re = re.compile(r'^.* [0-9]+ /sec; [0-9]+\.[0-9]+ ms/op .*\([0-9]+ (op|ops) in [0-9]+(\.[0-9]+)? ms\)') + format_re = re.compile( + r"^.* [0-9]+ /sec; [0-9]+\.[0-9]+ ms/op .*\([0-9]+ (op|ops) in [0-9]+(\.[0-9]+)? ms\)" + ) for op in math_ops: - output = test_cli("speed", ["--msec=%d" % (msec), op], None).split('\n') + output = test_cli("speed", ["--msec=%d" % (msec), op], None).split("\n") for line in output: if format_re.match(line) is None: logging.error("Unexpected line %s", line) -def cli_speed_tests(_tmp_dir): +def cli_speed_tests(_tmp_dir): msec = 1 - output = test_cli("speed", ["--msec=%d" % (msec), "--buf-size=64,512", "AES-128"], None).split('\n') + output = test_cli( + "speed", ["--msec=%d" % (msec), "--buf-size=64,512", "AES-128"], None + ).split("\n") if len(output) % 4 != 0: logging.error("Unexpected number of lines for AES-128 speed test") - format_re = re.compile(r'^.* .* buffer size [0-9]+ bytes: [0-9]+\.[0-9]+ MiB\/sec .*\([0-9]+\.[0-9]+ MiB in [0-9]+\.[0-9]+ ms\)') + format_re = re.compile( + r"^.* .* buffer size [0-9]+ bytes: [0-9]+\.[0-9]+ MiB\/sec .*\([0-9]+\.[0-9]+ MiB in [0-9]+\.[0-9]+ ms\)" + ) for line in output: if format_re.match(line) is None: logging.error("Unexpected line %s", line) - output = test_cli("speed", ["--msec=%d" % (msec), "ChaCha20", "SHA-256", "HMAC(SHA-256)"], None).split('\n') + output = test_cli( + "speed", ["--msec=%d" % (msec), "ChaCha20", "SHA-256", "HMAC(SHA-256)"], None + ).split("\n") - format_re = re.compile(r'^.* buffer size [0-9]+ bytes: [0-9]+\.[0-9]+ MiB\/sec .*\([0-9]+\.[0-9]+ MiB in [0-9]+\.[0-9]+ ms\)') + format_re = re.compile( + r"^.* buffer size [0-9]+ bytes: [0-9]+\.[0-9]+ MiB\/sec .*\([0-9]+\.[0-9]+ MiB in [0-9]+\.[0-9]+ ms\)" + ) for line in output: if format_re.match(line) is None: logging.error("Unexpected line %s", line) - output = test_cli("speed", ["--msec=%d" % (msec), "AES-128/GCM"], None).split('\n') - format_re_ks = re.compile(r'^AES-128/GCM\(16\).* [0-9]+ key schedule/sec; [0-9]+\.[0-9]+ ms/op .*\([0-9]+ (op|ops) in [0-9\.]+ ms\)') - format_re_cipher = re.compile(r'^AES-128/GCM\(16\) .* buffer size [0-9]+ bytes: [0-9]+\.[0-9]+ MiB\/sec .*\([0-9]+\.[0-9]+ MiB in [0-9]+\.[0-9]+ ms\)') + output = test_cli("speed", ["--msec=%d" % (msec), "AES-128/GCM"], None).split("\n") + format_re_ks = re.compile( + r"^AES-128/GCM\(16\).* [0-9]+ key schedule/sec; [0-9]+\.[0-9]+ ms/op .*\([0-9]+ (op|ops) in [0-9\.]+ ms\)" + ) + format_re_cipher = re.compile( + r"^AES-128/GCM\(16\) .* buffer size [0-9]+ bytes: [0-9]+\.[0-9]+ MiB\/sec .*\([0-9]+\.[0-9]+ MiB in [0-9]+\.[0-9]+ ms\)" + ) for line in output: if format_re_ks.match(line) is None: if format_re_cipher.match(line) is None: - logging.error('Unexpected line %s', line) + logging.error("Unexpected line %s", line) - output = test_cli("speed", ["--msec=%d" % (msec), "scrypt"], None).split('\n') + output = test_cli("speed", ["--msec=%d" % (msec), "scrypt"], None).split("\n") - format_re = re.compile(r'^scrypt-[0-9]+-[0-9]+-[0-9]+ \([0-9]+ MiB\) [0-9]+ /sec; [0-9]+\.[0-9]+ ms/op .*\([0-9]+ (op|ops) in [0-9\.]+ ms\)') + format_re = re.compile( + r"^scrypt-[0-9]+-[0-9]+-[0-9]+ \([0-9]+ MiB\) [0-9]+ /sec; [0-9]+\.[0-9]+ ms/op .*\([0-9]+ (op|ops) in [0-9\.]+ ms\)" + ) for line in output: if format_re.match(line) is None: logging.error("Unexpected line %s", line) - output = test_cli("speed", ["--msec=%d" % (msec), "RNG"], None).split('\n') + output = test_cli("speed", ["--msec=%d" % (msec), "RNG"], None).split("\n") # ChaCha_RNG generate buffer size 1024 bytes: 954.431 MiB/sec 4.01 cycles/byte (477.22 MiB in 500.00 ms) - format_re = re.compile(r'^.* generate buffer size [0-9]+ bytes: [0-9]+\.[0-9]+ MiB/sec .*\([0-9]+\.[0-9]+ MiB in [0-9]+\.[0-9]+ ms') + format_re = re.compile( + r"^.* generate buffer size [0-9]+ bytes: [0-9]+\.[0-9]+ MiB/sec .*\([0-9]+\.[0-9]+ MiB in [0-9]+\.[0-9]+ ms" + ) for line in output: if format_re.match(line) is None: logging.error("Unexpected line %s", line) - output = test_cli("speed", ["--msec=%d" % (msec), "zfec"], None).split('\n') - format_re = re.compile(r'^zfec [0-9]+/[0-9]+ (encode|decode) buffer size [0-9]+ bytes: [0-9]+\.[0-9]+ MiB/sec .*\([0-9]+\.[0-9]+ MiB in [0-9]+\.[0-9]+ ms') + output = test_cli("speed", ["--msec=%d" % (msec), "zfec"], None).split("\n") + format_re = re.compile( + r"^zfec [0-9]+/[0-9]+ (encode|decode) buffer size [0-9]+ bytes: [0-9]+\.[0-9]+ MiB/sec .*\([0-9]+\.[0-9]+ MiB in [0-9]+\.[0-9]+ ms" + ) for line in output: if format_re.match(line) is None: logging.error("Unexpected line %s", line) @@ -1771,17 +2493,18 @@ def cli_speed_tests(_tmp_dir): if len(json_blob) < 2: logging.error("Unexpected size for JSON output") - if 'version' not in json_blob[0]: + if "version" not in json_blob[0]: logging.error("Didn't find version header in first JSON object") for b in json_blob[1:]: - for field in ['algo', 'op', 'events', 'bps', 'buf_size', 'nanos']: + for field in ["algo", "op", "events", "bps", "buf_size", "nanos"]: if field not in b: - logging.error('Missing field %s in JSON record %s', field, b) + logging.error("Missing field %s in JSON record %s", field, b) + def run_test(fn_name, fn): start = time.time() - tmp_dir = tempfile.mkdtemp(prefix='botan_cli_') + tmp_dir = tempfile.mkdtemp(prefix="botan_cli_") try: fn(tmp_dir) except Exception as e: @@ -1790,21 +2513,23 @@ def run_test(fn_name, fn): shutil.rmtree(tmp_dir) end = time.time() - logging.info("Ran %s in %.02f sec", fn_name, end-start) + logging.info("Ran %s in %.02f sec", fn_name, end - start) + def main(args=None): if args is None: args = sys.argv parser = optparse.OptionParser( - formatter=optparse.IndentedHelpFormatter(max_help_position=50)) + formatter=optparse.IndentedHelpFormatter(max_help_position=50) + ) - parser.add_option('--verbose', action='store_true', default=False) - parser.add_option('--quiet', action='store_true', default=False) - parser.add_option('--threads', action='store', type='int', default=0) - parser.add_option('--run-slow-tests', action='store_true', default=False) - parser.add_option('--run-online-tests', action='store_true', default=False) - parser.add_option('--test-data-dir', default='.') + parser.add_option("--verbose", action="store_true", default=False) + parser.add_option("--quiet", action="store_true", default=False) + parser.add_option("--threads", action="store", type="int", default=0) + parser.add_option("--run-slow-tests", action="store_true", default=False) + parser.add_option("--run-online-tests", action="store_true", default=False) + parser.add_option("--test-data-dir", default=".") (options, args) = parser.parse_args(args) @@ -1826,7 +2551,7 @@ def main(args=None): CLI_PATH = args[1] global TEST_DATA_DIR - TEST_DATA_DIR = os.path.join(options.test_data_dir, 'src/tests/data/timing/') + TEST_DATA_DIR = os.path.join(options.test_data_dir, "src/tests/data/timing/") test_regex = None if len(args) == 3: @@ -1893,7 +2618,7 @@ def main(args=None): cli_uuid_tests, cli_version_tests, cli_zfec_tests, - ] + ] test_fns = [] @@ -1928,12 +2653,15 @@ def main(args=None): end_time = time.time() - print("Ran %d tests with %d failures in %.02f seconds" % ( - TESTS_RUN, TESTS_FAILED, end_time - start_time)) + print( + "Ran %d tests with %d failures in %.02f seconds" + % (TESTS_RUN, TESTS_FAILED, end_time - start_time) + ) if TESTS_FAILED > 0: return 1 return 0 -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/test_cli_crypt.py b/src/scripts/test_cli_crypt.py index 325fca68f67..79fe22d6227 100755 --- a/src/scripts/test_cli_crypt.py +++ b/src/scripts/test_cli_crypt.py @@ -19,6 +19,7 @@ import multiprocessing from multiprocessing.pool import ThreadPool + class VecDocument: def __init__(self, filepath): self.data = OrderedDict() @@ -31,14 +32,14 @@ def __init__(self, filepath): PATTERN_GROUPHEADER = r"^\[(.+)\]$" PATTERN_KEYVALUE = r"^\s*([a-zA-Z]+)\s*=(.*)$" - with open(filepath, 'r') as f: + with open(filepath, "r") as f: # Append one empty line to simplify parsing lines = f.read().splitlines() + ["\n"] for line in lines: line = line.strip() if line.startswith("#"): - pass # Skip + pass # Skip elif line == "": current_testcase_number += 1 elif re.match(PATTERN_GROUPHEADER, line): @@ -67,9 +68,11 @@ def __init__(self, filepath): def get_data(self): return self.data + TESTS_RUN = 0 TESTS_FAILED = 0 + class TestLogHandler(logging.StreamHandler, object): def emit(self, record): # Do the default stuff first @@ -78,6 +81,7 @@ def emit(self, record): global TESTS_FAILED TESTS_FAILED += 1 + def setup_logging(options): if options.verbose: log_level = logging.DEBUG @@ -87,17 +91,18 @@ def setup_logging(options): log_level = logging.INFO lh = TestLogHandler(sys.stdout) - lh.setFormatter(logging.Formatter('%(levelname) 7s: %(message)s')) + lh.setFormatter(logging.Formatter("%(levelname) 7s: %(message)s")) logging.getLogger().addHandler(lh) logging.getLogger().setLevel(log_level) + def test_cipher_kat(cli_binary, data): - iv = data['Nonce'] if 'Nonce' in data else '' - key = data['Key'] - plaintext = data['In'].lower() - ciphertext = data['Out'].lower() - algorithm = data['Algorithm'] - direction = data['Direction'] + iv = data["Nonce"] if "Nonce" in data else "" + key = data["Key"] + plaintext = data["In"].lower() + ciphertext = data["Out"].lower() + algorithm = data["Algorithm"] + direction = data["Direction"] cmd = [ cli_binary, @@ -105,30 +110,36 @@ def test_cipher_kat(cli_binary, data): "--cipher=%s" % algorithm, "--nonce=%s" % iv, "--key=%s" % key, - "-"] + "-", + ] - if 'AD' in data: - cmd += ['--ad=%s' % (data['AD'])] + if "AD" in data: + cmd += ["--ad=%s" % (data["AD"])] if direction == "decrypt": - cmd += ['--decrypt'] + cmd += ["--decrypt"] if direction == "decrypt": invalue = ciphertext else: invalue = plaintext - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE) + p = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE + ) (stdout_raw, stderr_raw) = p.communicate(input=binascii.unhexlify(invalue)) output = binascii.hexlify(stdout_raw).decode("UTF-8").lower() stderr = stderr_raw.decode("UTF-8") - if stderr != '': + if stderr != "": logging.error("Unexpected stderr output %s" % (stderr)) expected = plaintext if direction == "decrypt" else ciphertext if expected != output: - logging.error("For test %s got %s expected %s" % (data['testname'], output, expected)) + logging.error( + "For test %s got %s expected %s" % (data["testname"], output, expected) + ) + def get_testdata(document, max_tests): out = [] @@ -136,18 +147,19 @@ def get_testdata(document, max_tests): testcase_number = 0 for testcase in document[algorithm]: testcase_number += 1 - for direction in ['encrypt', 'decrypt']: + for direction in ["encrypt", "decrypt"]: testname = "{} no {:0>3} ({})".format( - algorithm.lower(), testcase_number, direction) + algorithm.lower(), testcase_number, direction + ) testname = re.sub("[^a-z0-9-]", "_", testname) testname = re.sub("_+", "_", testname) testname = testname.strip("_") - test = {'testname': testname} + test = {"testname": testname} for key in testcase: value = testcase[key] test[key] = value - test['Algorithm'] = algorithm - test['Direction'] = direction + test["Algorithm"] = algorithm + test["Direction"] = direction out.append(test) @@ -155,17 +167,18 @@ def get_testdata(document, max_tests): break return out + def main(args=None): if args is None: args = sys.argv parser = argparse.ArgumentParser(description="") - parser.add_argument('cli_binary', help='path to the botan cli binary') - parser.add_argument('--threads', type=int, default=0, metavar="T") - parser.add_argument('--verbose', action='store_true', default=False) - parser.add_argument('--quiet', action='store_true', default=False) - parser.add_argument('--run-slow-tests', action='store_true', default=False) - parser.add_argument('--test-data-dir', default='.') + parser.add_argument("cli_binary", help="path to the botan cli binary") + parser.add_argument("--threads", type=int, default=0, metavar="T") + parser.add_argument("--verbose", action="store_true", default=False) + parser.add_argument("--quiet", action="store_true", default=False) + parser.add_argument("--run-slow-tests", action="store_true", default=False) + parser.add_argument("--test-data-dir", default=".") args = parser.parse_args() setup_logging(args) @@ -177,18 +190,18 @@ def main(args=None): if threads == 0: threads = multiprocessing.cpu_count() - test_data_dir = os.path.join(args.test_data_dir, 'src', 'tests', 'data') + test_data_dir = os.path.join(args.test_data_dir, "src", "tests", "data") mode_test_data = [ - os.path.join(test_data_dir, 'aead', 'ccm.vec'), - os.path.join(test_data_dir, 'aead', 'chacha20poly1305.vec'), - os.path.join(test_data_dir, 'aead', 'eax.vec'), - os.path.join(test_data_dir, 'aead', 'gcm.vec'), - os.path.join(test_data_dir, 'aead', 'ocb.vec'), - os.path.join(test_data_dir, 'modes', 'cbc.vec'), - os.path.join(test_data_dir, 'modes', 'cfb.vec'), - os.path.join(test_data_dir, 'modes', 'ctr.vec'), - os.path.join(test_data_dir, 'modes', 'xts.vec'), + os.path.join(test_data_dir, "aead", "ccm.vec"), + os.path.join(test_data_dir, "aead", "chacha20poly1305.vec"), + os.path.join(test_data_dir, "aead", "eax.vec"), + os.path.join(test_data_dir, "aead", "gcm.vec"), + os.path.join(test_data_dir, "aead", "ocb.vec"), + os.path.join(test_data_dir, "modes", "cbc.vec"), + os.path.join(test_data_dir, "modes", "cfb.vec"), + os.path.join(test_data_dir, "modes", "ctr.vec"), + os.path.join(test_data_dir, "modes", "xts.vec"), ] kats = [] @@ -212,12 +225,15 @@ def main(args=None): end_time = time.time() - print("Ran %d tests with %d failures in %.02f seconds" % ( - len(kats), TESTS_FAILED, end_time - start_time)) + print( + "Ran %d tests with %d failures in %.02f seconds" + % (len(kats), TESTS_FAILED, end_time - start_time) + ) if TESTS_FAILED > 0: return 1 return 0 -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/test_fuzzers.py b/src/scripts/test_fuzzers.py index b1f32bec487..4212bf5de0f 100755 --- a/src/scripts/test_fuzzers.py +++ b/src/scripts/test_fuzzers.py @@ -6,58 +6,89 @@ import sys import os import subprocess -import optparse # pylint: disable=deprecated-module +import optparse # pylint: disable=deprecated-module import stat import multiprocessing import time import tempfile + def run_fuzzer_gdb(args): (fuzzer_bin, corpus_file) = args - gdb_proc = subprocess.Popen(['gdb', '--quiet', '--return-child-result', fuzzer_bin], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - close_fds=True) + gdb_proc = subprocess.Popen( + ["gdb", "--quiet", "--return-child-result", fuzzer_bin], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=True, + ) - gdb_commands = ('run < %s\nbt\nquit\n' % (corpus_file)).encode('ascii') + gdb_commands = ("run < %s\nbt\nquit\n" % (corpus_file)).encode("ascii") (stdout, stderr) = gdb_proc.communicate(gdb_commands) if gdb_proc.returncode == 0: - return (0, '', '') + return (0, "", "") + + return ( + corpus_file, + gdb_proc.returncode, + stdout.decode("ascii"), + stderr.decode("ascii"), + ) - return (corpus_file, gdb_proc.returncode, stdout.decode('ascii'), stderr.decode('ascii')) def run_fuzzer(args): (fuzzer_bin, corpus_file) = args - corpus_fd = open(corpus_file, 'rb') - fuzzer_proc = subprocess.Popen([fuzzer_bin], stdin=corpus_fd, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) + corpus_fd = open(corpus_file, "rb") + fuzzer_proc = subprocess.Popen( + [fuzzer_bin], + stdin=corpus_fd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=True, + ) (stdout, stderr) = fuzzer_proc.communicate() corpus_fd.close() - return (corpus_file, fuzzer_proc.returncode, stdout.decode('ascii'), stderr.decode('ascii')) + return ( + corpus_file, + fuzzer_proc.returncode, + stdout.decode("ascii"), + stderr.decode("ascii"), + ) + def run_fuzzer_many_files(fuzzer_bin, corpus_files): - fuzzer_proc = subprocess.Popen([fuzzer_bin] + corpus_files, stdin=None, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) + fuzzer_proc = subprocess.Popen( + [fuzzer_bin] + corpus_files, + stdin=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=True, + ) (stdout, stderr) = fuzzer_proc.communicate() - return (fuzzer_proc.returncode, stdout.decode('ascii'), stderr.decode('ascii')) + return (fuzzer_proc.returncode, stdout.decode("ascii"), stderr.decode("ascii")) + def main(args=None): if args is None: args = sys.argv parser = optparse.OptionParser( - usage='Usage: %prog [options] corpus_dir fuzzers_dir', + usage="Usage: %prog [options] corpus_dir fuzzers_dir", ) - parser.add_option('--gdb', action='store_true', - help='Run under GDB and capture backtraces') + parser.add_option( + "--gdb", action="store_true", help="Run under GDB and capture backtraces" + ) - parser.add_option('--one-at-a-time', action='store_true', default=False, - help='Test one corpus input at a time') + parser.add_option( + "--one-at-a-time", + action="store_true", + default=False, + help="Test one corpus input at a time", + ) (options, args) = parser.parse_args(args) @@ -82,7 +113,7 @@ def main(args=None): fuzzers = set([]) for fuzzer in os.listdir(fuzzer_dir): - if fuzzer.endswith('.zip'): + if fuzzer.endswith(".zip"): continue fuzzers.add(fuzzer) @@ -92,7 +123,7 @@ def main(args=None): if not stat.S_ISDIR(os.stat(os.path.join(corpus_dir, corpus)).st_mode): continue - if corpus == '.git': + if corpus == ".git": continue corpii.add(corpus) @@ -113,14 +144,17 @@ def main(args=None): if options.one_at_a_time: pool = multiprocessing.Pool(multiprocessing.cpu_count() * 2) - chunk_size = 32 # arbitrary + chunk_size = 32 # arbitrary run_fuzzer_func = run_fuzzer_gdb if options.gdb else run_fuzzer for fuzzer in sorted(list(fuzzers_with_corpus)): fuzzer_bin = os.path.join(fuzzer_dir, fuzzer) corpus_subdir = os.path.join(corpus_dir, fuzzer) - corpus_files = [os.path.join(corpus_subdir, fsname) for fsname in sorted(list(os.listdir(corpus_subdir)))] + corpus_files = [ + os.path.join(corpus_subdir, fsname) + for fsname in sorted(list(os.listdir(corpus_subdir))) + ] # We have to do this hack because multiprocessing's Pool.map doesn't support # passing any initial arguments, just the single iterable @@ -132,33 +166,44 @@ def main(args=None): (corpus_file, retcode, stdout, stderr) = result if retcode != 0: - print("Fuzzer %s crashed with input %s returncode %d" % (fuzzer, corpus_file, retcode)) + print( + "Fuzzer %s crashed with input %s returncode %d" + % (fuzzer, corpus_file, retcode) + ) crash_count += 1 if stdout: - print("Fuzzer %s produced stdout on input %s:\n%s" % (fuzzer, corpus_file, stdout)) + print( + "Fuzzer %s produced stdout on input %s:\n%s" + % (fuzzer, corpus_file, stdout) + ) stdout_count += 1 if stderr: - print("Fuzzer %s produced stderr on input %s:\n%s" % (fuzzer, corpus_file, stderr)) + print( + "Fuzzer %s produced stderr on input %s:\n%s" + % (fuzzer, corpus_file, stderr) + ) stderr_count += 1 duration = time.time() - start - print("Tested fuzzer %s with %d test cases, %d crashes in %.02f seconds" % ( - fuzzer, len(corpus_files), crash_count, duration)) + print( + "Tested fuzzer %s with %d test cases, %d crashes in %.02f seconds" + % (fuzzer, len(corpus_files), crash_count, duration) + ) crash_count = 0 sys.stdout.flush() else: # Generate a random corpus for fuzzers without a corpus - random_corpus_dir = tempfile.mkdtemp(prefix='fuzzer_corpus_') + random_corpus_dir = tempfile.mkdtemp(prefix="fuzzer_corpus_") - slow_fuzzers = ['invert', 'ecc_p521', 'pow_mod'] + slow_fuzzers = ["invert", "ecc_p521", "pow_mod"] random_corpus_size = 1000 random_corpus_size_for_slow_fuzzers = 100 for i in range(random_corpus_size): random_input = os.urandom(i) - fd = open(os.path.join(random_corpus_dir, 'input_%d' % (i)), 'wb') + fd = open(os.path.join(random_corpus_dir, "input_%d" % (i)), "wb") fd.write(random_input) fd.close() @@ -170,7 +215,10 @@ def main(args=None): else: corpus_subdir = random_corpus_dir - corpus_files = [os.path.join(corpus_subdir, fsname) for fsname in sorted(list(os.listdir(corpus_subdir)))] + corpus_files = [ + os.path.join(corpus_subdir, fsname) + for fsname in sorted(list(os.listdir(corpus_subdir))) + ] if fuzzer in slow_fuzzers: corpus_files = corpus_files[:random_corpus_size_for_slow_fuzzers] @@ -193,14 +241,20 @@ def main(args=None): duration = time.time() - start - print("Tested fuzzer %s with %d test cases, %d crashes in %.02f seconds" % ( - fuzzer, len(corpus_files), crash_count, duration)) + print( + "Tested fuzzer %s with %d test cases, %d crashes in %.02f seconds" + % (fuzzer, len(corpus_files), crash_count, duration) + ) crash_count = 0 if crash_count > 0 or stderr_count > 0 or stdout_count > 0: - print("Ran fuzzer tests, %d crashes %d stdout %d stderr" % (crash_count, stdout_count, stderr_count)) + print( + "Ran fuzzer tests, %d crashes %d stdout %d stderr" + % (crash_count, stdout_count, stderr_count) + ) return 2 return 0 -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/test_python.py b/src/scripts/test_python.py index 02c8b184cfe..3f4bd0d26af 100644 --- a/src/scripts/test_python.py +++ b/src/scripts/test_python.py @@ -21,25 +21,30 @@ if platform.system() == "Windows" and hasattr(os, "add_dll_directory"): os.add_dll_directory(os.getcwd()) -import botan3 as botan # pylint: disable=wrong-import-position +import botan3 as botan # pylint: disable=wrong-import-position + def hex_encode(buf): - return binascii.hexlify(buf).decode('ascii') + return binascii.hexlify(buf).decode("ascii") + def hex_decode(buf): - return binascii.unhexlify(buf.encode('ascii')) + return binascii.unhexlify(buf.encode("ascii")) + # pylint: disable=global-statement ARGS = None + def test_data(relpath): return os.path.join(ARGS.test_data_dir, relpath) + class BotanPythonTests(unittest.TestCase): def test_version(self): version_str = botan.version_string() - self.assertTrue(version_str.startswith('Botan ')) + self.assertTrue(version_str.startswith("Botan ")) self.assertEqual(botan.version_major(), 3) self.assertGreaterEqual(botan.version_minor(), 0) @@ -47,7 +52,6 @@ def test_version(self): self.assertGreaterEqual(botan.ffi_api_version(), 20180713) def test_compare(self): - x = "1234" y = "1234" z = "1233" @@ -72,72 +76,88 @@ def test_block_cipher(self): self.assertEqual(hex_encode(pt), "00112233445566778899aabbccddeeff") def test_kdf(self): + secret = hex_decode("6FD4C3C0F38E5C7A6F83E99CD9BD") + salt = hex_decode("DBB986") + label = hex_decode("") + expected = hex_decode( + "02AEB40A3D4B66FBA540F9D4B20006F2046E0F3A029DEAB201FC692B79EB27CEF7E16069046A" + ) - secret = hex_decode('6FD4C3C0F38E5C7A6F83E99CD9BD') - salt = hex_decode('DBB986') - label = hex_decode('') - expected = hex_decode('02AEB40A3D4B66FBA540F9D4B20006F2046E0F3A029DEAB201FC692B79EB27CEF7E16069046A') - - produced = botan.kdf('KDF2(SHA-1)', secret, 38, salt, label) + produced = botan.kdf("KDF2(SHA-1)", secret, 38, salt, label) self.assertEqual(hex_encode(produced), hex_encode(expected)) def test_pbkdf(self): - - (salt, iterations, pbkdf) = botan.pbkdf('PBKDF2(SHA-1)', '', 32, 10000, hex_decode('0001020304050607')) + (salt, iterations, pbkdf) = botan.pbkdf( + "PBKDF2(SHA-1)", "", 32, 10000, hex_decode("0001020304050607") + ) self.assertEqual(iterations, 10000) - self.assertEqual(hex_encode(pbkdf), - '59b2b1143b4cb1059ec58d9722fb1c72471e0d85c6f7543ba5228526375b0127') + self.assertEqual( + hex_encode(pbkdf), + "59b2b1143b4cb1059ec58d9722fb1c72471e0d85c6f7543ba5228526375b0127", + ) - (salt, iterations, pbkdf) = botan.pbkdf_timed('PBKDF2(SHA-256)', 'xyz', 32, 200) + (salt, iterations, pbkdf) = botan.pbkdf_timed("PBKDF2(SHA-256)", "xyz", 32, 200) - cmp_pbkdf = botan.pbkdf('PBKDF2(SHA-256)', 'xyz', 32, iterations, salt)[2] + cmp_pbkdf = botan.pbkdf("PBKDF2(SHA-256)", "xyz", 32, iterations, salt)[2] self.assertEqual(pbkdf, cmp_pbkdf) def test_argon2i(self): m = 8 p = 1 - passphrase = 'pass' - salt = hex_decode('6161616161616161') + passphrase = "pass" + salt = hex_decode("6161616161616161") out_len = 16 - output = botan.argon2('Argon2id', out_len, passphrase, salt, m, 1, p) - self.assertEqual(hex_encode(output), '39cc496a60824ddea31a83b0aecdb7a1') + output = botan.argon2("Argon2id", out_len, passphrase, salt, m, 1, p) + self.assertEqual(hex_encode(output), "39cc496a60824ddea31a83b0aecdb7a1") - output = botan.argon2('Argon2id', out_len, passphrase, salt, m, 2, p) - self.assertEqual(hex_encode(output), 'dfc6f1993cb9f0090c1c6e25629e607b') + output = botan.argon2("Argon2id", out_len, passphrase, salt, m, 2, p) + self.assertEqual(hex_encode(output), "dfc6f1993cb9f0090c1c6e25629e607b") def test_scrypt(self): - scrypt = botan.scrypt(10, '', '', 16, 1, 1) + scrypt = botan.scrypt(10, "", "", 16, 1, 1) self.assertEqual(hex_encode(scrypt), "77d6576238657b203b19") - scrypt = botan.scrypt(32, 'password', 'NaCl', 1024, 8, 16) - self.assertEqual(hex_encode(scrypt), "fdbabe1c9d3472007856e7190d01e9fe7c6ad7cbc8237830e77376634b373162") + scrypt = botan.scrypt(32, "password", "NaCl", 1024, 8, 16) + self.assertEqual( + hex_encode(scrypt), + "fdbabe1c9d3472007856e7190d01e9fe7c6ad7cbc8237830e77376634b373162", + ) def test_bcrypt(self): r = botan.RandomNumberGenerator() - phash = botan.bcrypt('testing', r) + phash = botan.bcrypt("testing", r) self.assertTrue(isinstance(phash, str)) self.assertTrue(phash.startswith("$2a$")) - self.assertTrue(botan.check_bcrypt('testing', phash)) - self.assertFalse(botan.check_bcrypt('live fire', phash)) + self.assertTrue(botan.check_bcrypt("testing", phash)) + self.assertFalse(botan.check_bcrypt("live fire", phash)) - self.assertTrue(botan.check_bcrypt('test', '$2a$04$wjen1fAA.UW6UxthpKK.huyOoxvCR7ATRCVC4CBIEGVDOCtr8Oj1C')) + self.assertTrue( + botan.check_bcrypt( + "test", "$2a$04$wjen1fAA.UW6UxthpKK.huyOoxvCR7ATRCVC4CBIEGVDOCtr8Oj1C" + ) + ) def test_mac(self): - - hmac = botan.MsgAuthCode('HMAC(SHA-256)') - self.assertEqual(hmac.algo_name(), 'HMAC(SHA-256)') + hmac = botan.MsgAuthCode("HMAC(SHA-256)") + self.assertEqual(hmac.algo_name(), "HMAC(SHA-256)") self.assertEqual(hmac.minimum_keylength(), 0) self.assertEqual(hmac.maximum_keylength(), 4096) - expected = hex_decode('A21B1F5D4CF4F73A4DD939750F7A066A7F98CC131CB16A6692759021CFAB8181') + expected = hex_decode( + "A21B1F5D4CF4F73A4DD939750F7A066A7F98CC131CB16A6692759021CFAB8181" + ) - hmac.set_key(hex_decode('0102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F20')) - hmac.update(hex_decode('616263')) + hmac.set_key( + hex_decode( + "0102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F20" + ) + ) + hmac.update(hex_decode("616263")) produced = hmac.final() self.assertEqual(hex_encode(expected), hex_encode(produced)) @@ -146,12 +166,12 @@ def test_mac(self): self.assertEqual(hex_encode(expected), hex_encode(produced)) def test_gmac(self): - gmac = botan.MsgAuthCode('GMAC(AES-128)') - self.assertEqual(gmac.algo_name(), 'GMAC(AES-128)') - gmac.set_key(hex_decode('00000000000000000000000000000000')) - gmac.set_nonce(hex_decode('000000000000000000000000')) + gmac = botan.MsgAuthCode("GMAC(AES-128)") + self.assertEqual(gmac.algo_name(), "GMAC(AES-128)") + gmac.set_key(hex_decode("00000000000000000000000000000000")) + gmac.set_nonce(hex_decode("000000000000000000000000")) - expected = hex_decode('58E2FCCEFA7E3061367F1D57A4E7455A') + expected = hex_decode("58E2FCCEFA7E3061367F1D57A4E7455A") produced = gmac.final() @@ -170,20 +190,22 @@ def test_rng(self): output3 = user_rng.get(1021) self.assertEqual(len(output3), 1021) - with self.assertRaisesRegex(botan.BotanException, r".*Unexpected arguments for RNG.*"): + with self.assertRaisesRegex( + botan.BotanException, r".*Unexpected arguments for RNG.*" + ): botan.RandomNumberGenerator("user", unexpected="unexpected") - system_rng = botan.RandomNumberGenerator('system') + system_rng = botan.RandomNumberGenerator("system") user_rng.reseed_from_rng(system_rng, 256) - user_rng.add_entropy('seed material...') + user_rng.add_entropy("seed material...") def test_esdm_rng(self): try: esdm_rng = botan.RandomNumberGenerator("esdm-full") except botan.BotanException as ex: - if ex.error_code() == -40: # Not Implemented + if ex.error_code() == -40: # Not Implemented self.skipTest("No ESDM support in this build") else: raise ex @@ -211,7 +233,7 @@ def test_tpm2_rng(self): try: tpm2_ctx = botan.TPM2Context(ARGS.tpm2_tcti_name, ARGS.tpm2_tcti_conf) except botan.BotanException as ex: - if ex.error_code() == -40: # Not Implemented + if ex.error_code() == -40: # Not Implemented self.skipTest("No TPM2 support in this build") else: raise ex @@ -223,26 +245,39 @@ def test_tpm2_rng(self): session = botan.TPM2UnauthenticatedSession(tpm2_ctx) # no TPM context provided - with self.assertRaisesRegex(botan.BotanException, r".*without a TPM2 context.*"): + with self.assertRaisesRegex( + botan.BotanException, r".*without a TPM2 context.*" + ): botan.RandomNumberGenerator("tpm2") # invalid session object provided - with self.assertRaisesRegex(botan.BotanException, r".*0 to 3 TPM2Session objects.*"): - botan.RandomNumberGenerator("tpm2", tpm2_context=tpm2_ctx, - tpm2_sessions=int(0)) + with self.assertRaisesRegex( + botan.BotanException, r".*0 to 3 TPM2Session objects.*" + ): + botan.RandomNumberGenerator( + "tpm2", tpm2_context=tpm2_ctx, tpm2_sessions=int(0) + ) # too many "sessions" provided - with self.assertRaisesRegex(botan.BotanException, r".*0 to 3 TPM2Session objects.*"): - botan.RandomNumberGenerator("tpm2", tpm2_context=tpm2_ctx, - tpm2_sessions=[session, None, None, None]) + with self.assertRaisesRegex( + botan.BotanException, r".*0 to 3 TPM2Session objects.*" + ): + botan.RandomNumberGenerator( + "tpm2", tpm2_context=tpm2_ctx, tpm2_sessions=[session, None, None, None] + ) # unexpected kwarg provided - with self.assertRaisesRegex(botan.BotanException, r".*Unexpected arguments for TPM2.*"): - botan.RandomNumberGenerator("tpm2", tpm2_context=tpm2_ctx, - unexpected_arg="unexpected") + with self.assertRaisesRegex( + botan.BotanException, r".*Unexpected arguments for TPM2.*" + ): + botan.RandomNumberGenerator( + "tpm2", tpm2_context=tpm2_ctx, unexpected_arg="unexpected" + ) # session provided as a single session (not wrapped into an Iterable) - tpm2_rng = botan.RandomNumberGenerator("tpm2", tpm2_context=tpm2_ctx, tpm2_sessions=session) + tpm2_rng = botan.RandomNumberGenerator( + "tpm2", tpm2_context=tpm2_ctx, tpm2_sessions=session + ) output1 = tpm2_rng.get(32) output2 = tpm2_rng.get(32) @@ -250,73 +285,91 @@ def test_tpm2_rng(self): self.assertEqual(len(output1), 32) self.assertEqual(len(output2), 32) self.assertNotEqual(output1, output2) - tpm2_rng.add_entropy('xkcd #221: 4 - chosen by fair dice roll') + tpm2_rng.add_entropy("xkcd #221: 4 - chosen by fair dice roll") # session provided wrapped into an Iterable - tpm2_rng2 = botan.RandomNumberGenerator("tpm2", tpm2_context=tpm2_ctx, tpm2_sessions=[session]) + tpm2_rng2 = botan.RandomNumberGenerator( + "tpm2", tpm2_context=tpm2_ctx, tpm2_sessions=[session] + ) output3 = tpm2_rng2.get(32) self.assertEqual(len(output3), 32) self.assertNotEqual(output2, output3) def test_hash(self): - try: - _h = botan.HashFunction('NoSuchHash') + _h = botan.HashFunction("NoSuchHash") except botan.BotanException as e: self.assertEqual(str(e), "botan_hash_init failed: -40 (Not implemented)") - sha256 = botan.HashFunction('SHA-256') - self.assertEqual(sha256.algo_name(), 'SHA-256') + sha256 = botan.HashFunction("SHA-256") + self.assertEqual(sha256.algo_name(), "SHA-256") self.assertEqual(sha256.output_length(), 32) self.assertEqual(sha256.block_size(), 64) - sha256.update('ignore this please') + sha256.update("ignore this please") sha256.clear() - sha256.update('a') + sha256.update("a") hash1 = sha256.final() - self.assertEqual(hex_encode(hash1), "ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb") + self.assertEqual( + hex_encode(hash1), + "ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb", + ) - sha256.update(hex_decode('61')) + sha256.update(hex_decode("61")) sha256_2 = sha256.copy_state() - sha256.update(hex_decode('6263')) + sha256.update(hex_decode("6263")) h2 = sha256.final() - self.assertEqual(hex_encode(h2), "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad") + self.assertEqual( + hex_encode(h2), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + ) self.assertEqual(hex_encode(sha256_2.final()), hex_encode(hash1)) - sha256.update(hex_decode('C2AF5C5F28E38384295F2FC2AF')) - self.assertEqual(hex_encode(sha256.final()), - "08bfce15fd2406114825ee6f770a06b1b00c129cb48fcddc54ef58b5de48bdf5") + sha256.update(hex_decode("C2AF5C5F28E38384295F2FC2AF")) + self.assertEqual( + hex_encode(sha256.final()), + "08bfce15fd2406114825ee6f770a06b1b00c129cb48fcddc54ef58b5de48bdf5", + ) - sha256.update(r'¯\_(ツ)_/¯') - self.assertEqual(hex_encode(sha256.final()), - "08bfce15fd2406114825ee6f770a06b1b00c129cb48fcddc54ef58b5de48bdf5") + sha256.update(r"¯\_(ツ)_/¯") + self.assertEqual( + hex_encode(sha256.final()), + "08bfce15fd2406114825ee6f770a06b1b00c129cb48fcddc54ef58b5de48bdf5", + ) def test_cipher(self): - for mode in ['AES-128/CTR-BE', 'Serpent/GCM', 'ChaCha20Poly1305', 'AES-128/CBC/PKCS7']: + for mode in [ + "AES-128/CTR-BE", + "Serpent/GCM", + "ChaCha20Poly1305", + "AES-128/CBC/PKCS7", + ]: try: enc = botan.SymmetricCipher(mode, encrypt=True) except botan.BotanException as e: - raise RuntimeError("Failed to create encrypting cipher for " + mode) from e + raise RuntimeError( + "Failed to create encrypting cipher for " + mode + ) from e - if mode == 'AES-128/CTR-BE': - self.assertEqual(enc.algo_name(), 'CTR-BE(AES-128)') + if mode == "AES-128/CTR-BE": + self.assertEqual(enc.algo_name(), "CTR-BE(AES-128)") self.assertFalse(enc.is_authenticated()) self.assertEqual(enc.update_granularity(), 1) self.assertGreater(enc.ideal_update_granularity(), 1) - elif mode == 'Serpent/GCM': - self.assertEqual(enc.algo_name(), 'Serpent/GCM(16)') + elif mode == "Serpent/GCM": + self.assertEqual(enc.algo_name(), "Serpent/GCM(16)") self.assertTrue(enc.is_authenticated()) self.assertEqual(enc.update_granularity(), 1) self.assertGreater(enc.ideal_update_granularity(), 16) - elif mode == 'ChaCha20Poly1305': - self.assertEqual(enc.algo_name(), 'ChaCha20Poly1305') + elif mode == "ChaCha20Poly1305": + self.assertEqual(enc.algo_name(), "ChaCha20Poly1305") self.assertTrue(enc.is_authenticated()) self.assertEqual(enc.update_granularity(), 1) self.assertGreater(enc.ideal_update_granularity(), 1) - elif mode == 'AES-128/CBC/PKCS7': - self.assertEqual(enc.algo_name(), 'AES-128/CBC/PKCS7') + elif mode == "AES-128/CBC/PKCS7": + self.assertEqual(enc.algo_name(), "AES-128/CBC/PKCS7") self.assertFalse(enc.is_authenticated()) self.assertEqual(enc.update_granularity(), 16) self.assertGreater(enc.ideal_update_granularity(), 16) @@ -333,7 +386,7 @@ def test_cipher(self): enc.set_key(key) enc.start(iv) - update_result = enc.update('') + update_result = enc.update("") assert not update_result ct = enc.finish(pt) @@ -341,22 +394,22 @@ def test_cipher(self): try: dec = botan.SymmetricCipher(mode, encrypt=False) except botan.BotanException as e: - raise RuntimeError("Failed to create decrypting cipher for " + mode) from e + raise RuntimeError( + "Failed to create decrypting cipher for " + mode + ) from e dec.set_key(key) dec.start(iv) decrypted = dec.finish(ct) self.assertEqual(decrypted, pt) - def test_mceliece(self): rng = botan.RandomNumberGenerator() - mce_priv = botan.PrivateKey.create('McEliece', '2960,57', rng) + mce_priv = botan.PrivateKey.create("McEliece", "2960,57", rng) mce_pub = mce_priv.get_public_key() self.assertEqual(mce_pub.estimated_strength(), 128) def test_rsa_load_store(self): - rsa_priv_pem = """-----BEGIN PRIVATE KEY----- MIICeAIBADANBgkqhkiG9w0BAQEFAASCAmIwggJeAgEAAoGBALWtiBjcofJW/4+r CIjQZn2V3yCYsNIBpMdVkNPr36FZ3ZHGSv2ggmCe+IWy0fTcBVyP+fo3HC8zmOC2 @@ -412,7 +465,7 @@ def test_rsa_load_store(self): def test_key_crypto(self): rng = botan.RandomNumberGenerator() - priv = botan.PrivateKey.create('RSA', '1024', rng) + priv = botan.PrivateKey.create("RSA", "1024", rng) passphrase = "super secret tell noone" for is_pem in [True, False]: @@ -422,25 +475,32 @@ def test_key_crypto(self): dec1 = botan.PrivateKey.load(enc1, passphrase) self.assertEqual(dec1.export(is_pem), ref_val) - pem2 = priv.export_encrypted(passphrase, rng, is_pem, msec=10, cipher="AES-128/SIV") + pem2 = priv.export_encrypted( + passphrase, rng, is_pem, msec=10, cipher="AES-128/SIV" + ) dec2 = botan.PrivateKey.load(pem2, passphrase) self.assertEqual(dec2.export(is_pem), ref_val) - pem3 = priv.export_encrypted(passphrase, rng, is_pem, msec=10, cipher="AES-128/GCM", pbkdf="Scrypt") + pem3 = priv.export_encrypted( + passphrase, rng, is_pem, msec=10, cipher="AES-128/GCM", pbkdf="Scrypt" + ) dec3 = botan.PrivateKey.load(pem3, passphrase) self.assertEqual(dec3.export(is_pem), ref_val) def test_stateful_operations(self): rng = botan.RandomNumberGenerator() - priv1 = botan.PrivateKey.create('RSA', '1024', rng) + priv1 = botan.PrivateKey.create("RSA", "1024", rng) self.assertEqual(priv1.stateful_operation(), False) try: remaining = priv1.remaining_operations() except botan.BotanException as e: - self.assertEqual(str(e), "botan_privkey_remaining_operations failed: -3 (No value available)") + self.assertEqual( + str(e), + "botan_privkey_remaining_operations failed: -3 (No value available)", + ) - priv2 = botan.PrivateKey.create('XMSS', 'XMSS-SHA2_10_256', rng) + priv2 = botan.PrivateKey.create("XMSS", "XMSS-SHA2_10_256", rng) self.assertEqual(priv2.stateful_operation(), True) remaining = priv2.remaining_operations() @@ -460,7 +520,10 @@ def test_check_key(self): try: rsapub = botan.PublicKey.load_rsa(n - 1, e) except botan.BotanException as e: - self.assertEqual(str(e), "botan_pubkey_load_rsa failed: -1 (Invalid input): Invalid RSA public key parameters") + self.assertEqual( + str(e), + "botan_pubkey_load_rsa failed: -1 (Invalid input): Invalid RSA public key parameters", + ) def _pksign_roundtrips(self, sk, pk, param_str): def verify_positive_and_negative(verifier, sig): @@ -470,7 +533,9 @@ def verify_positive_and_negative(verifier, sig): rng = botan.RandomNumberGenerator() msg = "test message" - raw_bytes = bytes.fromhex("8100112233445566778899AABBCCDDEEFF") # these bytes can't be decoded as UTF-8! + raw_bytes = bytes.fromhex( + "8100112233445566778899AABBCCDDEEFF" + ) # these bytes can't be decoded as UTF-8! # Check that update() takes UTF-8 data signer = botan.PKSign(sk, param_str) @@ -513,13 +578,13 @@ def verify_positive_and_negative(verifier, sig): def _ecc_sec1_convert_to_compressed(uncompressed_sec1): assert uncompressed_sec1[0] == 0x04 is_odd = uncompressed_sec1[-1] & 0x01 != 0 - x = uncompressed_sec1[1:1 + (len(uncompressed_sec1) - 1) // 2] + x = uncompressed_sec1[1 : 1 + (len(uncompressed_sec1) - 1) // 2] return (b"\x03" if is_odd else b"\x02") + x def test_rsa(self): rng = botan.RandomNumberGenerator() - rsapriv = botan.PrivateKey.create('RSA', '1024', rng) - self.assertEqual(rsapriv.algo_name(), 'RSA') + rsapriv = botan.PrivateKey.create("RSA", "1024", rng) + self.assertEqual(rsapriv.algo_name(), "RSA") priv_pem = rsapriv.to_pem() priv_der = rsapriv.to_der() @@ -528,7 +593,7 @@ def test_rsa(self): self.assertGreater(len(priv_pem), len(priv_der)) rsapub = rsapriv.get_public_key() - self.assertEqual(rsapub.algo_name(), 'RSA') + self.assertEqual(rsapub.algo_name(), "RSA") self.assertEqual(rsapub.estimated_strength(), 80) pub_pem = rsapub.to_pem() @@ -549,47 +614,47 @@ def test_rsa(self): self._pksign_roundtrips(rsapriv, rsapub, "PSS(SHA-384)") - salt = b'saltyseawater' - kem_e = botan.KemEncrypt(rsapub, 'KDF2(SHA-256)') + salt = b"saltyseawater" + kem_e = botan.KemEncrypt(rsapub, "KDF2(SHA-256)") (shared_key, encap_key) = kem_e.create_shared_key(rng, salt, 32) self.assertEqual(len(shared_key), 32) - self.assertEqual(len(encap_key), 1024//8) + self.assertEqual(len(encap_key), 1024 // 8) - kem_d = botan.KemDecrypt(rsapriv, 'KDF2(SHA-256)') + kem_d = botan.KemDecrypt(rsapriv, "KDF2(SHA-256)") shared_key_d = kem_d.decrypt_shared_key(salt, 32, encap_key) self.assertEqual(shared_key, shared_key_d) def test_kyber(self): rng = botan.RandomNumberGenerator() - kyber_priv = botan.PrivateKey.create('Kyber', 'Kyber-1024-r3', rng) + kyber_priv = botan.PrivateKey.create("Kyber", "Kyber-1024-r3", rng) kyber_pub = kyber_priv.get_public_key() salt = rng.get(16) - kem_e = botan.KemEncrypt(kyber_pub, 'KDF2(SHA-256)') + kem_e = botan.KemEncrypt(kyber_pub, "KDF2(SHA-256)") (shared_key, encap_key) = kem_e.create_shared_key(rng, salt, 32) self.assertEqual(len(shared_key), 32) self.assertEqual(len(encap_key), 1568) - kem_d = botan.KemDecrypt(kyber_priv, 'KDF2(SHA-256)') + kem_d = botan.KemDecrypt(kyber_priv, "KDF2(SHA-256)") shared_key_d = kem_d.decrypt_shared_key(salt, 32, encap_key) self.assertEqual(shared_key, shared_key_d) def test_ecdsa(self): rng = botan.RandomNumberGenerator() - hash_fn = 'SHA-256' - group = 'secp256r1' - msg = 'test message' + hash_fn = "SHA-256" + group = "secp256r1" + msg = "test message" if not botan.ECGroup.supports_named_group(group): self.skipTest("No secp256r1 group support in this build") return - priv = botan.PrivateKey.create('ECDSA', group, rng) + priv = botan.PrivateKey.create("ECDSA", group, rng) pub = priv.get_public_key() - self.assertEqual(pub.get_field('public_x'), priv.get_field('public_x')) - self.assertEqual(pub.get_field('public_y'), priv.get_field('public_y')) + self.assertEqual(pub.get_field("public_x"), priv.get_field("public_x")) + self.assertEqual(pub.get_field("public_y"), priv.get_field("public_y")) self._pksign_roundtrips(priv, pub, hash_fn) @@ -599,7 +664,7 @@ def test_ecdsa(self): verifier = botan.PKVerify(pub, hash_fn) verifier.update(msg) - #fails because DER/not-DER mismatch + # fails because DER/not-DER mismatch self.assertFalse(verifier.check_signature(signature)) verifier = botan.PKVerify(pub, hash_fn, True) @@ -607,13 +672,13 @@ def test_ecdsa(self): self.assertTrue(verifier.check_signature(signature)) # Load public key from components - pub_x = pub.get_field('public_x') - pub_y = priv.get_field('public_y') + pub_x = pub.get_field("public_x") + pub_y = priv.get_field("public_y") pub2 = botan.PublicKey.load_ecdsa(group, pub_x, pub_y) self._pksign_roundtrips(priv, pub2, hash_fn) # Load private key from component - priv2 = botan.PrivateKey.load_ecdsa(group, priv.get_field('x')) + priv2 = botan.PrivateKey.load_ecdsa(group, priv.get_field("x")) self._pksign_roundtrips(priv2, pub, hash_fn) # Load public key from SEC.1 encoding @@ -621,35 +686,37 @@ def test_ecdsa(self): pub3 = botan.PublicKey.load_ecdsa_sec1(group, uncompressed_sec1) self._pksign_roundtrips(priv, pub3, hash_fn) - compressed_sec1 = BotanPythonTests._ecc_sec1_convert_to_compressed(uncompressed_sec1) + compressed_sec1 = BotanPythonTests._ecc_sec1_convert_to_compressed( + uncompressed_sec1 + ) pub4 = botan.PublicKey.load_ecdsa_sec1(group, compressed_sec1) self._pksign_roundtrips(priv, pub4, hash_fn) def test_sm2(self): rng = botan.RandomNumberGenerator() - hash_fn = 'SM3' - group = 'sm2p256v1' + hash_fn = "SM3" + group = "sm2p256v1" if not botan.ECGroup.supports_named_group(group): self.skipTest("No sm2p256v1 group support in this build") return - priv = botan.PrivateKey.create('SM2', group, rng) + priv = botan.PrivateKey.create("SM2", group, rng) pub = priv.get_public_key() - self.assertEqual(pub.get_field('public_x'), priv.get_field('public_x')) - self.assertEqual(pub.get_field('public_y'), priv.get_field('public_y')) + self.assertEqual(pub.get_field("public_x"), priv.get_field("public_x")) + self.assertEqual(pub.get_field("public_y"), priv.get_field("public_y")) self._pksign_roundtrips(priv, pub, hash_fn) # Load public key from components - pub_x = pub.get_field('public_x') - pub_y = priv.get_field('public_y') + pub_x = pub.get_field("public_x") + pub_y = priv.get_field("public_y") pub2 = botan.PublicKey.load_sm2(group, pub_x, pub_y) self._pksign_roundtrips(priv, pub2, hash_fn) # Load private key from component - priv2 = botan.PrivateKey.load_sm2(group, priv.get_field('x')) + priv2 = botan.PrivateKey.load_sm2(group, priv.get_field("x")) self._pksign_roundtrips(priv2, pub, hash_fn) # Load public key from SEC.1 encoding @@ -657,22 +724,24 @@ def test_sm2(self): pub3 = botan.PublicKey.load_sm2_sec1(group, uncompressed_sec1) self._pksign_roundtrips(priv, pub3, hash_fn) - compressed_sec1 = BotanPythonTests._ecc_sec1_convert_to_compressed(uncompressed_sec1) + compressed_sec1 = BotanPythonTests._ecc_sec1_convert_to_compressed( + uncompressed_sec1 + ) pub4 = botan.PublicKey.load_sm2_sec1(group, compressed_sec1) self._pksign_roundtrips(priv, pub4, hash_fn) def test_ecdh(self): - a_rng = botan.RandomNumberGenerator('user') - b_rng = botan.RandomNumberGenerator('user') + a_rng = botan.RandomNumberGenerator("user") + b_rng = botan.RandomNumberGenerator("user") - kdf = 'KDF2(SHA-384)' + kdf = "KDF2(SHA-384)" - for grp in ['secp256r1', 'secp384r1', 'brainpool256r1']: + for grp in ["secp256r1", "secp384r1", "brainpool256r1"]: if not botan.ECGroup.supports_named_group(grp): continue - a_priv = botan.PrivateKey.create('ECDH', grp, a_rng) - b_priv = botan.PrivateKey.create('ECDH', grp, b_rng) + a_priv = botan.PrivateKey.create("ECDH", grp, a_rng) + b_priv = botan.PrivateKey.create("ECDH", grp, b_rng) a_op = botan.PKKeyAgreement(a_priv, kdf) b_op = botan.PKKeyAgreement(b_priv, kdf) @@ -697,7 +766,7 @@ def test_ecdh(self): a_pem = a_priv.to_pem() - a_priv_x = a_priv.get_field('x') + a_priv_x = a_priv.get_field("x") new_a = botan.PrivateKey.load_ecdh(grp, a_priv_x) @@ -707,7 +776,9 @@ def test_ecdh(self): self.assertEqual(int(a_raw, base=16), a_priv_x) uncompressed_sec1 = a_priv.get_public_key().to_raw() - compressed_sec1 = BotanPythonTests._ecc_sec1_convert_to_compressed(uncompressed_sec1) + compressed_sec1 = BotanPythonTests._ecc_sec1_convert_to_compressed( + uncompressed_sec1 + ) new_a_pub1 = botan.PublicKey.load_ecdh_sec1(grp, uncompressed_sec1) new_a_pub2 = botan.PublicKey.load_ecdh_sec1(grp, compressed_sec1) self.assertEqual(new_a_pub1.to_raw(), new_a_pub2.to_raw()) @@ -716,13 +787,12 @@ def test_ecdh(self): b_key2 = b_op2.agree(compressed_sec1, 32, salt) self.assertEqual(b_key2, b_key) - def test_rfc7748_kex(self): rng = botan.RandomNumberGenerator() - for alg in ['X25519', 'X448']: - a_priv = botan.PrivateKey.create(alg, '', rng) - b_priv = botan.PrivateKey.create(alg, '', rng) + for alg in ["X25519", "X448"]: + a_priv = botan.PrivateKey.create(alg, "", rng) + b_priv = botan.PrivateKey.create(alg, "", rng) a_op = botan.PKKeyAgreement(a_priv, "Raw") b_op = botan.PKKeyAgreement(b_priv, "Raw") @@ -741,36 +811,48 @@ def test_rfc7748_kex(self): def test_eddsa(self): rng = botan.RandomNumberGenerator() - for alg in ['Ed25519', 'Ed448']: - priv = botan.PrivateKey.create(alg, '', rng) + for alg in ["Ed25519", "Ed448"]: + priv = botan.PrivateKey.create(alg, "", rng) pub = priv.get_public_key() self._pksign_roundtrips(priv, pub, "") def test_certs(self): - cert = botan.X509Cert(filename=test_data("src/tests/data/x509/ecc/isrg-root-x2.pem")) + cert = botan.X509Cert( + filename=test_data("src/tests/data/x509/ecc/isrg-root-x2.pem") + ) pubkey = cert.subject_public_key() self.assertEqual(len(cert.subject_public_key_bits()), 118) - self.assertEqual(pubkey.algo_name(), 'ECDSA') + self.assertEqual(pubkey.algo_name(), "ECDSA") self.assertEqual(pubkey.estimated_strength(), 192) - self.assertEqual(cert.fingerprint("SHA-1"), - "BD:B1:B9:3C:D5:97:8D:45:C6:26:14:55:F8:DB:95:C7:5A:D1:53:AF") + self.assertEqual( + cert.fingerprint("SHA-1"), + "BD:B1:B9:3C:D5:97:8D:45:C6:26:14:55:F8:DB:95:C7:5A:D1:53:AF", + ) - self.assertEqual(hex_encode(cert.serial_number()), "41d29dd172eaeea780c12c6ce92f8752") - self.assertEqual(hex_encode(cert.subject_key_id()), - "7c4296aede4b483bfa92f89e8ccf6d8ba9723795") + self.assertEqual( + hex_encode(cert.serial_number()), "41d29dd172eaeea780c12c6ce92f8752" + ) + self.assertEqual( + hex_encode(cert.subject_key_id()), + "7c4296aede4b483bfa92f89e8ccf6d8ba9723795", + ) - self.assertEqual(cert.subject_dn('Name', 0), 'ISRG Root X2') - self.assertEqual(cert.subject_dn('Organization', 0), 'Internet Security Research Group') - self.assertEqual(cert.subject_dn('Country', 0), 'US') + self.assertEqual(cert.subject_dn("Name", 0), "ISRG Root X2") + self.assertEqual( + cert.subject_dn("Organization", 0), "Internet Security Research Group" + ) + self.assertEqual(cert.subject_dn("Country", 0), "US") self.assertTrue(cert.to_string().startswith("Version: 3")) - self.assertEqual(cert.issuer_dn('Name', 0), 'ISRG Root X2') - self.assertEqual(cert.issuer_dn('Organization', 0), 'Internet Security Research Group') - self.assertEqual(cert.issuer_dn('Country', 0), 'US') + self.assertEqual(cert.issuer_dn("Name", 0), "ISRG Root X2") + self.assertEqual( + cert.issuer_dn("Organization", 0), "Internet Security Research Group" + ) + self.assertEqual(cert.issuer_dn("Country", 0), "US") self.assertEqual(cert.not_before(), 1599177600) self.assertEqual(cert.not_after(), 2231510400) @@ -789,44 +871,90 @@ def test_certs(self): end04 = botan.X509Cert(test_data("src/tests/data/x509/nist/test04/end.crt")) int04_1 = botan.X509Cert(test_data("src/tests/data/x509/nist/test04/int1.crt")) int04_2 = botan.X509Cert(test_data("src/tests/data/x509/nist/test04/int2.crt")) - self.assertEqual(end04.verify([int04_1, int04_2], [], test_data("src/tests/data/x509/nist/"), required_strength=80), 0) - self.assertEqual(end04.verify([int04_1, int04_2], [], required_strength=80), 3000) - self.assertEqual(end04.verify([int04_1, int04_2], [root], required_strength=80, hostname="User1-CP.02.01"), 0) - self.assertEqual(end04.verify([int04_1, int04_2], [root], required_strength=80, hostname="invalid"), 4008) - self.assertEqual(end04.verify([int04_1, int04_2], [root], required_strength=80, reference_time=1), 2000) + self.assertEqual( + end04.verify( + [int04_1, int04_2], + [], + test_data("src/tests/data/x509/nist/"), + required_strength=80, + ), + 0, + ) + self.assertEqual( + end04.verify([int04_1, int04_2], [], required_strength=80), 3000 + ) + self.assertEqual( + end04.verify( + [int04_1, int04_2], + [root], + required_strength=80, + hostname="User1-CP.02.01", + ), + 0, + ) + self.assertEqual( + end04.verify( + [int04_1, int04_2], [root], required_strength=80, hostname="invalid" + ), + 4008, + ) + self.assertEqual( + end04.verify( + [int04_1, int04_2], [root], required_strength=80, reference_time=1 + ), + 2000, + ) - self.assertEqual(botan.X509Cert.validation_status(0), 'Verified') - self.assertEqual(botan.X509Cert.validation_status(3000), 'Certificate issuer not found') - self.assertEqual(botan.X509Cert.validation_status(4008), 'Certificate does not match provided name') + self.assertEqual(botan.X509Cert.validation_status(0), "Verified") + self.assertEqual( + botan.X509Cert.validation_status(3000), "Certificate issuer not found" + ) + self.assertEqual( + botan.X509Cert.validation_status(4008), + "Certificate does not match provided name", + ) rootcrl = botan.X509CRL(test_data("src/tests/data/x509/nist/root.crl")) end01 = botan.X509Cert(test_data("src/tests/data/x509/nist/test01/end.crt")) - self.assertEqual(end01.verify([], [root], required_strength=80, crls=[rootcrl]), 0) + self.assertEqual( + end01.verify([], [root], required_strength=80, crls=[rootcrl]), 0 + ) int20 = botan.X509Cert(test_data("src/tests/data/x509/nist/test20/int.crt")) end20 = botan.X509Cert(test_data("src/tests/data/x509/nist/test20/end.crt")) int20crl = botan.X509CRL(test_data("src/tests/data/x509/nist/test20/int.crl")) - self.assertEqual(end20.verify([int20], [root], required_strength=80, crls=[int20crl, rootcrl]), 5000) - self.assertEqual(botan.X509Cert.validation_status(5000), 'Certificate is revoked') + self.assertEqual( + end20.verify( + [int20], [root], required_strength=80, crls=[int20crl, rootcrl] + ), + 5000, + ) + self.assertEqual( + botan.X509Cert.validation_status(5000), "Certificate is revoked" + ) int21 = botan.X509Cert(test_data("src/tests/data/x509/nist/test21/int.crt")) end21 = botan.X509Cert(test_data("src/tests/data/x509/nist/test21/end.crt")) int21crl = botan.X509CRL(test_data("src/tests/data/x509/nist/test21/int.crl")) - self.assertEqual(end21.verify([int21], [root], required_strength=80, crls=[int21crl, rootcrl]), 5000) + self.assertEqual( + end21.verify( + [int21], [root], required_strength=80, crls=[int21crl, rootcrl] + ), + 5000, + ) self.assertTrue(int20.is_revoked(rootcrl)) self.assertFalse(int04_1.is_revoked(rootcrl)) self.assertTrue(end21.is_revoked(int21crl)) - def test_mpi(self): z = botan.MPI() self.assertEqual(z.bit_count(), 0) - five = botan.MPI('5') + five = botan.MPI("5") self.assertEqual(five.bit_count(), 3) - big = botan.MPI('0x85839682368923476892367235') + big = botan.MPI("0x85839682368923476892367235") self.assertEqual(big.bit_count(), 104) small = botan.MPI(0xDEADBEEF) radix = botan.MPI("DEADBEEF", 16) @@ -922,7 +1050,7 @@ def test_mpi_random(self): self.assertEqual(upper.bit_count(), 512) lower = upper >> 32 - self.assertEqual(lower.bit_count(), 512-32) + self.assertEqual(lower.bit_count(), 512 - 32) for _i in range(10): x = botan.MPI.random_range(rng, lower, upper) @@ -930,14 +1058,13 @@ def test_mpi_random(self): self.assertGreater(x, lower) def test_fpe(self): - - modulus = botan.MPI('1000000000') - key = b'001122334455' + modulus = botan.MPI("1000000000") + key = b"001122334455" fpe = botan.FormatPreservingEncryptionFE1(modulus, key) - value = botan.MPI('392910392') - tweak = 'tweak value' + value = botan.MPI("392910392") + tweak = "tweak value" ctext = fpe.encrypt(value, tweak) @@ -946,19 +1073,20 @@ def test_fpe(self): self.assertEqual(value, ptext) def test_keywrap(self): - key = hex_decode('00112233445566778899aabbccddeeff') - kek = hex_decode('000102030405060708090a0b0c0d0e0f') + key = hex_decode("00112233445566778899aabbccddeeff") + kek = hex_decode("000102030405060708090a0b0c0d0e0f") wrapped = botan.nist_key_wrap(kek, key) - self.assertEqual(hex_encode(wrapped), '1fa68b0a8112b447aef34bd8fb5a7b829d3e862371d2cfe5') + self.assertEqual( + hex_encode(wrapped), "1fa68b0a8112b447aef34bd8fb5a7b829d3e862371d2cfe5" + ) - self.assertEqual(len(wrapped), 16+8) + self.assertEqual(len(wrapped), 16 + 8) unwrapped = botan.nist_key_unwrap(kek, wrapped) - self.assertEqual(hex_encode(unwrapped), '00112233445566778899aabbccddeeff') + self.assertEqual(hex_encode(unwrapped), "00112233445566778899aabbccddeeff") def test_hotp(self): - - hotp = botan.HOTP(b'12345678901234567890') + hotp = botan.HOTP(b"12345678901234567890") self.assertEqual(hotp.generate(0), 755224) self.assertEqual(hotp.generate(1), 287082) @@ -970,8 +1098,7 @@ def test_hotp(self): self.assertEqual(hotp.check(520489, 0, 9), (True, 10)) def test_totp(self): - - totp = botan.TOTP(b'12345678901234567890', digest="SHA-1", digits=8) + totp = botan.TOTP(b"12345678901234567890", digest="SHA-1", digits=8) self.assertEqual(totp.generate(59), 94287082) self.assertEqual(totp.generate(1111111109), 7081804) @@ -986,27 +1113,37 @@ def test_totp(self): self.assertTrue(totp.check(7081804, 1111111109 + 30, 1)) def test_srp6(self): - identity = 'alice' - password = 'password123' + identity = "alice" + password = "password123" rng = botan.RandomNumberGenerator() - group = 'modp/srp/1024' - hash_fn = 'SHA-512' + group = "modp/srp/1024" + hash_fn = "SHA-512" # Test successful authentication server = botan.Srp6ServerSession(group) salt = rng.get(24) - verifier = botan.srp6_generate_verifier(identity, password, salt, group, hash_fn) + verifier = botan.srp6_generate_verifier( + identity, password, salt, group, hash_fn + ) b = server.step1(verifier, hash_fn, rng) - (a, key_c) = botan.srp6_client_agree(identity, password, group, hash_fn, salt, b, rng) + (a, key_c) = botan.srp6_client_agree( + identity, password, group, hash_fn, salt, b, rng + ) key_s = server.step2(a) self.assertEqual(key_c, key_s) def test_kyber_raw_keys(self): - a_pub_bits = hex_decode("38d4851e5c010da39a7470bc1c80916f78c7bd5891dcd3b1ea84b6f051b346b803c1b97c94604020b7279b27836c3049ea0b9a3758510b7559593237ade72587462206b709f365848c96559326a5fe6c6f4cf531fd1a18477267f66ba14af20163c41f1138a01995147f271ddfc2be5361b281e6029d210584b2859b7383667284f767bb32782baad10933da0138a3a0660a149531c03f9c8ffccb33e3c3a5a7984e21fab26aa72a8a6b942f265e52551a9c800e5a44805f0c0141a05554213387f105df56458496bd8f469051886da223cb9fe78e7b390bf94b0a937691af9550082b76d045cb4d29c23c67942608d078a1c80f24767a945d19f077d82c9b9b197073464abe69cf7c5626177308f384672d5263b0c4826db4470e1a70e4751e3918abe8fcbc3bc0531ae89e5512214b5cc94a16a014bcb3826c79fbf4add0825eeefbab88cb7cff37bb8d491f8de902578a1e961655565b7718782a23504fdc13c783f130e177925e305d1fbc63cc8c15c2c67f85500cca785de9f480490558ef71aaf0fb5b513914401269b309c4c59c64d2a757d8855f58465615925f1ea6812cb143fff383e1048e285118bf932944b86fbdf4b1b9e65685664a07775c46952aaada1168f54b47c7a231e7355c64637467b5a3c09cab67bb35f58640c2726283bb63530a15f66eca48a840c00ca8862e283c73bfbb413a2915b8d1159a043f12c59bfa828248249b76106faa61a127a0280c586350e7a42cb74ca49cabd606891ec7cb8e84affe4b2e14c71658332b755611bab7977fa76ce736b21ed34a17ac0ec3561ca9b282d4a2bc407697924b1cf918ba83d3a4fdc82564c95bd904bdeee91ed6ccb36baa88a05c80712901bf280aee6538ec2078c2a84ee5862fc137cd92e97968d69fc3453a1e1cb161c50c9f2473a0d09037b188a0fa01efc344c2ac8fe8592b0a58456662a95033659a158a2d90a6e50c253a87975785ce29c4570000a154d4b3b2c642205c8c7cf9ac6b1071fbb368ab950a744b88c95ba5243017831120a9048338d29847830d12a933a09abd21a46b828cb14e808cd35129c9dc6e5b931d4a126fefe07909618e2b4586e7b6b424963b7323ba505ba112bb9b834a7d1b78ad0df53d556a1c69369f09148b1dc9938df59223f087fd6833be5b2bc2651fe58911ac01467f9297dfdc22b41a0f1702718710b78cf35b1865813a896d45214d338155b6c043c532330c002d520739467a504a866637fb3451c849f8f83e6a94147f168da53acdf9d8affd968a84124a9abc09af960cd3b29f2344831bb41e67605eebf00df202857117399dd748b6514aed61bb2f6cb841d168d5f35e20054573a331cd4882a04b072c179158825bcf471266da0dcceab1a021c73254751d5a161c1a92062c220a217a69d9823314b4de996fe8d45f6db5af16c1561495a4c43090bc394c94e1b0ec738eb56267201c2ecd1c7b4993c0efc0284bdc9a091c294f95703a7178822c8a95b79b1e4591e0998d893875c1a879c08a073cc67df426bba792c18ae6c1feba879bec54812c2affa012973b700ad48e271078280864268600a7aa309eaa1098750a0f8a522eb929577b412f7855613688b72f9bc85c0a13b9d041586fd583feb12afd5a402dd33b43543f5fa4eb436c8d") - b_priv_bits = hex_decode("9bc4986eec853ac90896c7300f914216773baa687f57f81f94e6b5a26515c0a74c73b19dc6b9888755a1d1db6a1128b02c5144ed9ba75e874a68a14dee3a9bf770b767121fbbdb292e097444757fe2f324a875b0a07a4fbdcc8fd6137fce80c69f412be103b01f8b3bed392600f830d1b20d73ca0b386666d16c5d96db10e309041890bfd4ab7bdec27191979abe7637e76153918cc2af1545724abfd95ca496281a0cc6ca63a87819b75aa86452e5027d429cad64a9c029112a3a7b9fb993c6a6d0671f6c9e24986b83f67cf1236d94c051559616826a947eaa15074025d1ab810c80220e8a8c2c222d4e8594bf35811b444a57e9bf94fb2479d174d6b38c8c3b4730437c244b2513232ec798adec4a8e597bca83bca5948468f93c022ba65311a91e3d95c299c70c3f1a43cd0c83dd500bd2211b9b385b82f2c003b10d295765b3e367563a5f19033fc1231db9620f835c95f5a261190a5da1c24ed528316f0a1520d223e208a8c99b24d28598ade74fc88370e7f45102c5a6411891c661b356b9a32e1cc0fafaa085d7670e8bcb5e768eb255204f2445b5b73b04079881903a553c865744343a925c7899777b1c9dd5579a481512f8157061606a9a67c041d38bc179048be17dd9e19dc0a572bce595afa3b68ff21bf78a63a7560636b6bb01ec3362e2aaabc8965818b7f2417ca8f66a5a2a67f72a3931e125d638a872862a7b680a54aa1f25d90dbd567635ec6664919e29517325a5c5048cc8d1c31af5e4866e85025b9184a7b75ed7f2c9c8d88259fa2ec5b05ed3751584f88308c67ff1a72fa2453305160baf404db7d4be56f1547b20bb7dec23f02b10db584b28ca40d8b39c1c3ab9f3d7bbda0822604ca48f26694d69810aa888ae3c0983c5ba4cb74211f7a5361ccdee694f4202aebe13a75b1b2dda1b3232376be213582afc4fde3474766671fe865e2fd98384eb65b2f349f1e24269b91bd9d08c80849735a9951304afd130b5c2211314630aed4b6ac3b1252a0999ff5a3ec26a283342389d219fd243706128b389eb2334fb2a6184a4eab6735d7428df5633ce030b8f853ee47c4401fc5d06b43c9b66b7aeeb23b5f000a30a6f88f027ee3994fe8b63e51b83bc24bb733a3773a35cbe138f6d9c91a3a3898bca6776030d740ac355176547d624719656a9a44e91c63faf7699dc6c2c45575718d48828828b39043c2fda2af416837efc38d17c56d4b63c63a5ab43434647d029f7b236b288958f06910763610f8b2f027a8dcd780039ab34a6871427476ff6500240e83b87c95dcfa45ac5315ef34b343fb609eb296e915c849bb8c57f57c69b177eaa8456377403fe8c6627a3282d45308f675d67085a15f0b1b55aa2a8f21afd6c05c3c00e9eb8c32418cb41963ce427b43e7545c58325c7b9368db2333de424dbeb3430f007d18a68d73b7dc67960b28206a68a1be400a770b5cd9d45a72824ca00345ac56491c1414fe5287a2eb2ad61f3bdcd0c84c335b04a703425d79dbd02b0a0e90de5b331c3c29f6562969e04cdf7095b2a7646b3d006b0b83cb68580b5ccb71de1b4d9f131bac133d6088e10613a00599d81d4818403a4bea83905304cc45ca645a9b2c6484cf9490f1755c744d9988ed60475e6ae44355ed15c7b549366f29581ec2721fd6704e0ca3f878812805675141c0a15a7b7ac35a9e3f8b2a010bc184981c57852895b2695d56131e32326717f6b101df1bc82b3ba0222d52656d118538c4ca3416be1c76ba37a9901a36e4883be6c541f2bbb561818cd2f136b98f658250545c1fa5bcfdc04374016db1c5132447fd6d568866451c25412f72967de868eaeb9c546fb40ea88cd84a1a586ca51c74bc9c3e56e104323afb658d1ac003151bdc35879a4b6762648bff0caa682f1b3319805d2326d5a46af832aacefbaa1ea820568ea3925870e9b6577eb93898e1b0cfbd1c995cb4cd6cabb979813819749b40a9952f50e97c4365e777dcfe9084219294c205acb350e07db9c98f53444546460962e2bf5aa1cc12273d882fcc215a7397b3f9b307c56b9a0429b30f88453a2376669b28b4bc4b84b51714b6652b7a1a0b53e9ec61b55ca51cdb38243239ee5f18243e515f178768a888f475b3d9060136bb22ee355b3da02c16ad83bdbb4aaa13809cb4bcd5bb53710737d3883632f9254e3336af61621a376720572450e3937c0d930e349adbfa7642ff822b9135e18f943e0178617604d10e0c09ffb3e09783c09d12cbe93311757af9857b77d1488ed39321ca97c3745c5bc176ca81274c8321bcb2029938b32bbd01aec137032f9760849701649120050b50d8353c36b8bb7724a67e7660fc93324065d63912c35a86d8fe60683067bbd2685c552ad8c65c77c57c937676fd61595c453174bf996e9d3a9ccb837b25464115c1ba3343ac097b80735aed3225091167cd8f841ffe49c5e698ef542124253084623179394433a4b61547b9ee09c98c2736ea086bd69d1bc7ae68a6ae5ce682a215860006b4604dee45a3e212f97643acb77a79a880382e483537c5198d4483a176d25aac9c3670a3f30956f18ad441904776bcd48131c7d6465bce0c133010f3176c92ec962f5c6b84d4c3ae949619cf48172997ca3b1ccf5c8cc7a67923e1295801048a3d40ac4f2c6467c750fc71314a0c1fc22637dd52e7ea50907ea973d765a6a9bb2b11aa405b9187f72026696710e61af3e41c33da1a05eb65e6523704f078e74e32f10e00247967aee3a8c6546889ef67cd613ad7236583f2104122ac6c6a40a84dc96d81c569e76a952c0a25f396e48337a4fe029a4c91cc7406872706a55573b75f160a4facad7c85fab141c63454bf48990729096ce9965604c7cc1e60ae6868dcc41bc3df71c3e5593f0488b0c6a3063e817f9f4bacb17599c8666ff3591126b4891fb7f5d29660bab60cf5007043a4311d41ab3b29787184f3d3c9ab7cc247f635145b67e970505ba44ad0e06b11ec5cda4175295199d19d660204cbdc17947cc66442d3a2cd408a20fe98174f31ee4e5bb3fa8bcd102bedc26527e9bae836442978a6ccbe510f93ab77569ab1f09d0e6312dd0cc0bcaf095fa8a52a7212d14714a7bf852416c9b026301bd965c30a43d24d97298346a46b2c4bc814ba4059653358b03c9456c60bf0193932eaa2f24ea8e4b010a5a4425ce4540fbab90d8e55c97ac2687f15ff5299278824a08d4743e1a62e1c6619cd3278cd75a97a5b4e3a38668b26c99bf8461495793bbb1b12ca369c825cb31d68731326bf4764b416bb3339ae9c9ce46d9da0e714c0bae8712a670d0e5dcfdd1dd0d045932c79c559b2ab3c7300e2d894b0eaa40a6ab254506d8c1176a33c4a1b2879604b1b80df48d31dd") - b_pub_bits = hex_decode("f9490f1755c744d9988ed60475e6ae44355ed15c7b549366f29581ec2721fd6704e0ca3f878812805675141c0a15a7b7ac35a9e3f8b2a010bc184981c57852895b2695d56131e32326717f6b101df1bc82b3ba0222d52656d118538c4ca3416be1c76ba37a9901a36e4883be6c541f2bbb561818cd2f136b98f658250545c1fa5bcfdc04374016db1c5132447fd6d568866451c25412f72967de868eaeb9c546fb40ea88cd84a1a586ca51c74bc9c3e56e104323afb658d1ac003151bdc35879a4b6762648bff0caa682f1b3319805d2326d5a46af832aacefbaa1ea820568ea3925870e9b6577eb93898e1b0cfbd1c995cb4cd6cabb979813819749b40a9952f50e97c4365e777dcfe9084219294c205acb350e07db9c98f53444546460962e2bf5aa1cc12273d882fcc215a7397b3f9b307c56b9a0429b30f88453a2376669b28b4bc4b84b51714b6652b7a1a0b53e9ec61b55ca51cdb38243239ee5f18243e515f178768a888f475b3d9060136bb22ee355b3da02c16ad83bdbb4aaa13809cb4bcd5bb53710737d3883632f9254e3336af61621a376720572450e3937c0d930e349adbfa7642ff822b9135e18f943e0178617604d10e0c09ffb3e09783c09d12cbe93311757af9857b77d1488ed39321ca97c3745c5bc176ca81274c8321bcb2029938b32bbd01aec137032f9760849701649120050b50d8353c36b8bb7724a67e7660fc93324065d63912c35a86d8fe60683067bbd2685c552ad8c65c77c57c937676fd61595c453174bf996e9d3a9ccb837b25464115c1ba3343ac097b80735aed3225091167cd8f841ffe49c5e698ef542124253084623179394433a4b61547b9ee09c98c2736ea086bd69d1bc7ae68a6ae5ce682a215860006b4604dee45a3e212f97643acb77a79a880382e483537c5198d4483a176d25aac9c3670a3f30956f18ad441904776bcd48131c7d6465bce0c133010f3176c92ec962f5c6b84d4c3ae949619cf48172997ca3b1ccf5c8cc7a67923e1295801048a3d40ac4f2c6467c750fc71314a0c1fc22637dd52e7ea50907ea973d765a6a9bb2b11aa405b9187f72026696710e61af3e41c33da1a05eb65e6523704f078e74e32f10e00247967aee3a8c6546889ef67cd613ad7236583f2104122ac6c6a40a84dc96d81c569e76a952c0a25f396e48337a4fe029a4c91cc7406872706a55573b75f160a4facad7c85fab141c63454bf48990729096ce9965604c7cc1e60ae6868dcc41bc3df71c3e5593f0488b0c6a3063e817f9f4bacb17599c8666ff3591126b4891fb7f5d29660bab60cf5007043a4311d41ab3b29787184f3d3c9ab7cc247f635145b67e970505ba44ad0e06b11ec5cda4175295199d19d660204cbdc17947cc66442d3a2cd408a20fe98174f31ee4e5bb3fa8bcd102bedc26527e9bae836442978a6ccbe510f93ab77569ab1f09d0e6312dd0cc0bcaf095fa8a52a7212d14714a7bf852416c9b026301bd965c30a43d24d97298346a46b2c4bc814ba4059653358b03c9456c60bf0193932eaa2f24ea8e4b010a5a4425ce4540fbab90d8e55c97ac2687f15ff5299278824a08d4743e1a62e1c6619cd3278cd75a97a5b4e3a38668b26c99bf8461495793bbb1b12ca369c825cb31d68731326bf4764b416bb333") + a_pub_bits = hex_decode( + "38d4851e5c010da39a7470bc1c80916f78c7bd5891dcd3b1ea84b6f051b346b803c1b97c94604020b7279b27836c3049ea0b9a3758510b7559593237ade72587462206b709f365848c96559326a5fe6c6f4cf531fd1a18477267f66ba14af20163c41f1138a01995147f271ddfc2be5361b281e6029d210584b2859b7383667284f767bb32782baad10933da0138a3a0660a149531c03f9c8ffccb33e3c3a5a7984e21fab26aa72a8a6b942f265e52551a9c800e5a44805f0c0141a05554213387f105df56458496bd8f469051886da223cb9fe78e7b390bf94b0a937691af9550082b76d045cb4d29c23c67942608d078a1c80f24767a945d19f077d82c9b9b197073464abe69cf7c5626177308f384672d5263b0c4826db4470e1a70e4751e3918abe8fcbc3bc0531ae89e5512214b5cc94a16a014bcb3826c79fbf4add0825eeefbab88cb7cff37bb8d491f8de902578a1e961655565b7718782a23504fdc13c783f130e177925e305d1fbc63cc8c15c2c67f85500cca785de9f480490558ef71aaf0fb5b513914401269b309c4c59c64d2a757d8855f58465615925f1ea6812cb143fff383e1048e285118bf932944b86fbdf4b1b9e65685664a07775c46952aaada1168f54b47c7a231e7355c64637467b5a3c09cab67bb35f58640c2726283bb63530a15f66eca48a840c00ca8862e283c73bfbb413a2915b8d1159a043f12c59bfa828248249b76106faa61a127a0280c586350e7a42cb74ca49cabd606891ec7cb8e84affe4b2e14c71658332b755611bab7977fa76ce736b21ed34a17ac0ec3561ca9b282d4a2bc407697924b1cf918ba83d3a4fdc82564c95bd904bdeee91ed6ccb36baa88a05c80712901bf280aee6538ec2078c2a84ee5862fc137cd92e97968d69fc3453a1e1cb161c50c9f2473a0d09037b188a0fa01efc344c2ac8fe8592b0a58456662a95033659a158a2d90a6e50c253a87975785ce29c4570000a154d4b3b2c642205c8c7cf9ac6b1071fbb368ab950a744b88c95ba5243017831120a9048338d29847830d12a933a09abd21a46b828cb14e808cd35129c9dc6e5b931d4a126fefe07909618e2b4586e7b6b424963b7323ba505ba112bb9b834a7d1b78ad0df53d556a1c69369f09148b1dc9938df59223f087fd6833be5b2bc2651fe58911ac01467f9297dfdc22b41a0f1702718710b78cf35b1865813a896d45214d338155b6c043c532330c002d520739467a504a866637fb3451c849f8f83e6a94147f168da53acdf9d8affd968a84124a9abc09af960cd3b29f2344831bb41e67605eebf00df202857117399dd748b6514aed61bb2f6cb841d168d5f35e20054573a331cd4882a04b072c179158825bcf471266da0dcceab1a021c73254751d5a161c1a92062c220a217a69d9823314b4de996fe8d45f6db5af16c1561495a4c43090bc394c94e1b0ec738eb56267201c2ecd1c7b4993c0efc0284bdc9a091c294f95703a7178822c8a95b79b1e4591e0998d893875c1a879c08a073cc67df426bba792c18ae6c1feba879bec54812c2affa012973b700ad48e271078280864268600a7aa309eaa1098750a0f8a522eb929577b412f7855613688b72f9bc85c0a13b9d041586fd583feb12afd5a402dd33b43543f5fa4eb436c8d" + ) + b_priv_bits = hex_decode( + "9bc4986eec853ac90896c7300f914216773baa687f57f81f94e6b5a26515c0a74c73b19dc6b9888755a1d1db6a1128b02c5144ed9ba75e874a68a14dee3a9bf770b767121fbbdb292e097444757fe2f324a875b0a07a4fbdcc8fd6137fce80c69f412be103b01f8b3bed392600f830d1b20d73ca0b386666d16c5d96db10e309041890bfd4ab7bdec27191979abe7637e76153918cc2af1545724abfd95ca496281a0cc6ca63a87819b75aa86452e5027d429cad64a9c029112a3a7b9fb993c6a6d0671f6c9e24986b83f67cf1236d94c051559616826a947eaa15074025d1ab810c80220e8a8c2c222d4e8594bf35811b444a57e9bf94fb2479d174d6b38c8c3b4730437c244b2513232ec798adec4a8e597bca83bca5948468f93c022ba65311a91e3d95c299c70c3f1a43cd0c83dd500bd2211b9b385b82f2c003b10d295765b3e367563a5f19033fc1231db9620f835c95f5a261190a5da1c24ed528316f0a1520d223e208a8c99b24d28598ade74fc88370e7f45102c5a6411891c661b356b9a32e1cc0fafaa085d7670e8bcb5e768eb255204f2445b5b73b04079881903a553c865744343a925c7899777b1c9dd5579a481512f8157061606a9a67c041d38bc179048be17dd9e19dc0a572bce595afa3b68ff21bf78a63a7560636b6bb01ec3362e2aaabc8965818b7f2417ca8f66a5a2a67f72a3931e125d638a872862a7b680a54aa1f25d90dbd567635ec6664919e29517325a5c5048cc8d1c31af5e4866e85025b9184a7b75ed7f2c9c8d88259fa2ec5b05ed3751584f88308c67ff1a72fa2453305160baf404db7d4be56f1547b20bb7dec23f02b10db584b28ca40d8b39c1c3ab9f3d7bbda0822604ca48f26694d69810aa888ae3c0983c5ba4cb74211f7a5361ccdee694f4202aebe13a75b1b2dda1b3232376be213582afc4fde3474766671fe865e2fd98384eb65b2f349f1e24269b91bd9d08c80849735a9951304afd130b5c2211314630aed4b6ac3b1252a0999ff5a3ec26a283342389d219fd243706128b389eb2334fb2a6184a4eab6735d7428df5633ce030b8f853ee47c4401fc5d06b43c9b66b7aeeb23b5f000a30a6f88f027ee3994fe8b63e51b83bc24bb733a3773a35cbe138f6d9c91a3a3898bca6776030d740ac355176547d624719656a9a44e91c63faf7699dc6c2c45575718d48828828b39043c2fda2af416837efc38d17c56d4b63c63a5ab43434647d029f7b236b288958f06910763610f8b2f027a8dcd780039ab34a6871427476ff6500240e83b87c95dcfa45ac5315ef34b343fb609eb296e915c849bb8c57f57c69b177eaa8456377403fe8c6627a3282d45308f675d67085a15f0b1b55aa2a8f21afd6c05c3c00e9eb8c32418cb41963ce427b43e7545c58325c7b9368db2333de424dbeb3430f007d18a68d73b7dc67960b28206a68a1be400a770b5cd9d45a72824ca00345ac56491c1414fe5287a2eb2ad61f3bdcd0c84c335b04a703425d79dbd02b0a0e90de5b331c3c29f6562969e04cdf7095b2a7646b3d006b0b83cb68580b5ccb71de1b4d9f131bac133d6088e10613a00599d81d4818403a4bea83905304cc45ca645a9b2c6484cf9490f1755c744d9988ed60475e6ae44355ed15c7b549366f29581ec2721fd6704e0ca3f878812805675141c0a15a7b7ac35a9e3f8b2a010bc184981c57852895b2695d56131e32326717f6b101df1bc82b3ba0222d52656d118538c4ca3416be1c76ba37a9901a36e4883be6c541f2bbb561818cd2f136b98f658250545c1fa5bcfdc04374016db1c5132447fd6d568866451c25412f72967de868eaeb9c546fb40ea88cd84a1a586ca51c74bc9c3e56e104323afb658d1ac003151bdc35879a4b6762648bff0caa682f1b3319805d2326d5a46af832aacefbaa1ea820568ea3925870e9b6577eb93898e1b0cfbd1c995cb4cd6cabb979813819749b40a9952f50e97c4365e777dcfe9084219294c205acb350e07db9c98f53444546460962e2bf5aa1cc12273d882fcc215a7397b3f9b307c56b9a0429b30f88453a2376669b28b4bc4b84b51714b6652b7a1a0b53e9ec61b55ca51cdb38243239ee5f18243e515f178768a888f475b3d9060136bb22ee355b3da02c16ad83bdbb4aaa13809cb4bcd5bb53710737d3883632f9254e3336af61621a376720572450e3937c0d930e349adbfa7642ff822b9135e18f943e0178617604d10e0c09ffb3e09783c09d12cbe93311757af9857b77d1488ed39321ca97c3745c5bc176ca81274c8321bcb2029938b32bbd01aec137032f9760849701649120050b50d8353c36b8bb7724a67e7660fc93324065d63912c35a86d8fe60683067bbd2685c552ad8c65c77c57c937676fd61595c453174bf996e9d3a9ccb837b25464115c1ba3343ac097b80735aed3225091167cd8f841ffe49c5e698ef542124253084623179394433a4b61547b9ee09c98c2736ea086bd69d1bc7ae68a6ae5ce682a215860006b4604dee45a3e212f97643acb77a79a880382e483537c5198d4483a176d25aac9c3670a3f30956f18ad441904776bcd48131c7d6465bce0c133010f3176c92ec962f5c6b84d4c3ae949619cf48172997ca3b1ccf5c8cc7a67923e1295801048a3d40ac4f2c6467c750fc71314a0c1fc22637dd52e7ea50907ea973d765a6a9bb2b11aa405b9187f72026696710e61af3e41c33da1a05eb65e6523704f078e74e32f10e00247967aee3a8c6546889ef67cd613ad7236583f2104122ac6c6a40a84dc96d81c569e76a952c0a25f396e48337a4fe029a4c91cc7406872706a55573b75f160a4facad7c85fab141c63454bf48990729096ce9965604c7cc1e60ae6868dcc41bc3df71c3e5593f0488b0c6a3063e817f9f4bacb17599c8666ff3591126b4891fb7f5d29660bab60cf5007043a4311d41ab3b29787184f3d3c9ab7cc247f635145b67e970505ba44ad0e06b11ec5cda4175295199d19d660204cbdc17947cc66442d3a2cd408a20fe98174f31ee4e5bb3fa8bcd102bedc26527e9bae836442978a6ccbe510f93ab77569ab1f09d0e6312dd0cc0bcaf095fa8a52a7212d14714a7bf852416c9b026301bd965c30a43d24d97298346a46b2c4bc814ba4059653358b03c9456c60bf0193932eaa2f24ea8e4b010a5a4425ce4540fbab90d8e55c97ac2687f15ff5299278824a08d4743e1a62e1c6619cd3278cd75a97a5b4e3a38668b26c99bf8461495793bbb1b12ca369c825cb31d68731326bf4764b416bb3339ae9c9ce46d9da0e714c0bae8712a670d0e5dcfdd1dd0d045932c79c559b2ab3c7300e2d894b0eaa40a6ab254506d8c1176a33c4a1b2879604b1b80df48d31dd" + ) + b_pub_bits = hex_decode( + "f9490f1755c744d9988ed60475e6ae44355ed15c7b549366f29581ec2721fd6704e0ca3f878812805675141c0a15a7b7ac35a9e3f8b2a010bc184981c57852895b2695d56131e32326717f6b101df1bc82b3ba0222d52656d118538c4ca3416be1c76ba37a9901a36e4883be6c541f2bbb561818cd2f136b98f658250545c1fa5bcfdc04374016db1c5132447fd6d568866451c25412f72967de868eaeb9c546fb40ea88cd84a1a586ca51c74bc9c3e56e104323afb658d1ac003151bdc35879a4b6762648bff0caa682f1b3319805d2326d5a46af832aacefbaa1ea820568ea3925870e9b6577eb93898e1b0cfbd1c995cb4cd6cabb979813819749b40a9952f50e97c4365e777dcfe9084219294c205acb350e07db9c98f53444546460962e2bf5aa1cc12273d882fcc215a7397b3f9b307c56b9a0429b30f88453a2376669b28b4bc4b84b51714b6652b7a1a0b53e9ec61b55ca51cdb38243239ee5f18243e515f178768a888f475b3d9060136bb22ee355b3da02c16ad83bdbb4aaa13809cb4bcd5bb53710737d3883632f9254e3336af61621a376720572450e3937c0d930e349adbfa7642ff822b9135e18f943e0178617604d10e0c09ffb3e09783c09d12cbe93311757af9857b77d1488ed39321ca97c3745c5bc176ca81274c8321bcb2029938b32bbd01aec137032f9760849701649120050b50d8353c36b8bb7724a67e7660fc93324065d63912c35a86d8fe60683067bbd2685c552ad8c65c77c57c937676fd61595c453174bf996e9d3a9ccb837b25464115c1ba3343ac097b80735aed3225091167cd8f841ffe49c5e698ef542124253084623179394433a4b61547b9ee09c98c2736ea086bd69d1bc7ae68a6ae5ce682a215860006b4604dee45a3e212f97643acb77a79a880382e483537c5198d4483a176d25aac9c3670a3f30956f18ad441904776bcd48131c7d6465bce0c133010f3176c92ec962f5c6b84d4c3ae949619cf48172997ca3b1ccf5c8cc7a67923e1295801048a3d40ac4f2c6467c750fc71314a0c1fc22637dd52e7ea50907ea973d765a6a9bb2b11aa405b9187f72026696710e61af3e41c33da1a05eb65e6523704f078e74e32f10e00247967aee3a8c6546889ef67cd613ad7236583f2104122ac6c6a40a84dc96d81c569e76a952c0a25f396e48337a4fe029a4c91cc7406872706a55573b75f160a4facad7c85fab141c63454bf48990729096ce9965604c7cc1e60ae6868dcc41bc3df71c3e5593f0488b0c6a3063e817f9f4bacb17599c8666ff3591126b4891fb7f5d29660bab60cf5007043a4311d41ab3b29787184f3d3c9ab7cc247f635145b67e970505ba44ad0e06b11ec5cda4175295199d19d660204cbdc17947cc66442d3a2cd408a20fe98174f31ee4e5bb3fa8bcd102bedc26527e9bae836442978a6ccbe510f93ab77569ab1f09d0e6312dd0cc0bcaf095fa8a52a7212d14714a7bf852416c9b026301bd965c30a43d24d97298346a46b2c4bc814ba4059653358b03c9456c60bf0193932eaa2f24ea8e4b010a5a4425ce4540fbab90d8e55c97ac2687f15ff5299278824a08d4743e1a62e1c6619cd3278cd75a97a5b4e3a38668b26c99bf8461495793bbb1b12ca369c825cb31d68731326bf4764b416bb333" + ) - b_priv = botan.PrivateKey.load_kyber(b_priv_bits) #2400 + b_priv = botan.PrivateKey.load_kyber(b_priv_bits) # 2400 privkey_read = b_priv.view_kyber_raw_key() privkey_generic_raw = b_priv.to_raw() @@ -1019,13 +1156,15 @@ def test_kyber_raw_keys(self): self.assertEqual(pubkey_read, b_pub_bits) self.assertEqual(pubkey_generic_raw, b_pub_bits) - a_pub = botan.PublicKey.load_kyber(a_pub_bits) #1184 + a_pub = botan.PublicKey.load_kyber(a_pub_bits) # 1184 pubkey_read = a_pub.view_kyber_raw_key() self.assertEqual(pubkey_read, a_pub_bits) def test_ml_kem_raw_keys(self): mlkem_mode = "ML-KEM-512" - sk = botan.PrivateKey.create("ML-KEM", mlkem_mode, botan.RandomNumberGenerator("user")) + sk = botan.PrivateKey.create( + "ML-KEM", mlkem_mode, botan.RandomNumberGenerator("user") + ) pk = sk.get_public_key() sk_bits = sk.to_raw() @@ -1039,7 +1178,9 @@ def test_ml_kem_raw_keys(self): def test_ml_dsa_raw_keys(self): mldsa_mode = "ML-DSA-4x4" - sk = botan.PrivateKey.create("ML-DSA", mldsa_mode, botan.RandomNumberGenerator("user")) + sk = botan.PrivateKey.create( + "ML-DSA", mldsa_mode, botan.RandomNumberGenerator("user") + ) pk = sk.get_public_key() sk_bits = sk.to_raw() @@ -1053,7 +1194,9 @@ def test_ml_dsa_raw_keys(self): def test_slh_dsa_raw_keys(self): slhdsa_mode = "SLH-DSA-SHAKE-128f" - sk = botan.PrivateKey.create("SLH-DSA", slhdsa_mode, botan.RandomNumberGenerator("user")) + sk = botan.PrivateKey.create( + "SLH-DSA", slhdsa_mode, botan.RandomNumberGenerator("user") + ) pk = sk.get_public_key() sk_bits = sk.to_raw() @@ -1067,7 +1210,9 @@ def test_slh_dsa_raw_keys(self): def test_frodokem_raw_keys(self): frodo_mode = "FrodoKEM-640-SHAKE" - sk = botan.PrivateKey.create("FrodoKEM", frodo_mode, botan.RandomNumberGenerator("user")) + sk = botan.PrivateKey.create( + "FrodoKEM", frodo_mode, botan.RandomNumberGenerator("user") + ) pk = sk.get_public_key() sk_bits = sk.to_raw() @@ -1081,7 +1226,9 @@ def test_frodokem_raw_keys(self): def test_classic_mceliece_raw_keys(self): cmce_mode = "348864f" - sk = botan.PrivateKey.create("ClassicMcEliece", cmce_mode, botan.RandomNumberGenerator("user")) + sk = botan.PrivateKey.create( + "ClassicMcEliece", cmce_mode, botan.RandomNumberGenerator("user") + ) pk = sk.get_public_key() sk_bits = sk.to_raw() @@ -1099,7 +1246,9 @@ def test_oids(self): new_oid = botan.OID.from_string("1.2.3.4.5.6.7.8") new_oid.register("random-name-that-definitely-has-no-oid") - new_oid_from_string = botan.OID.from_string("random-name-that-definitely-has-no-oid") + new_oid_from_string = botan.OID.from_string( + "random-name-that-definitely-has-no-oid" + ) self.assertEqual(new_oid, new_oid_from_string) self.assertEqual(new_oid.to_string(), new_oid_from_string.to_string()) self.assertEqual(new_oid.to_name(), new_oid_from_string.to_name()) @@ -1148,21 +1297,39 @@ def test_ec_group(self): priv = botan.PrivateKey.create_ec("ECDSA", group_from_name, rng) self.assertEqual(priv.algo_name(), "ECDSA") - self.assertEqual(group_from_name, botan.ECGroup.from_pem(group_from_name.to_pem())) - self.assertEqual(group_from_name, botan.ECGroup.from_ber(group_from_name.to_der())) + self.assertEqual( + group_from_name, botan.ECGroup.from_pem(group_from_name.to_pem()) + ) + self.assertEqual( + group_from_name, botan.ECGroup.from_ber(group_from_name.to_der()) + ) if botan.ECGroup.supports_application_specific_group(): secp256r1_new_oid = botan.OID.from_string("1.3.6.1.4.1.25258.100.0") secp256r1_new_oid.register("secp256r1-but-manually-registered") - p = botan.MPI("FFFFFFFF00000001000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFF", 16) - a = botan.MPI("FFFFFFFF00000001000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFC", 16) - b = botan.MPI("5AC635D8AA3A93E7B3EBBD55769886BC651D06B0CC53B0F63BCE3C3E27D2604B", 16) - g_x = botan.MPI("6B17D1F2E12C4247F8BCE6E563A440F277037D812DEB33A0F4A13945D898C296", 16) - g_y = botan.MPI("4FE342E2FE1A7F9B8EE7EB4A7C0F9E162BCE33576B315ECECBB6406837BF51F5", 16) - order = botan.MPI("FFFFFFFF00000000FFFFFFFFFFFFFFFFBCE6FAADA7179E84F3B9CAC2FC632551", 16) + p = botan.MPI( + "FFFFFFFF00000001000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFF", 16 + ) + a = botan.MPI( + "FFFFFFFF00000001000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFC", 16 + ) + b = botan.MPI( + "5AC635D8AA3A93E7B3EBBD55769886BC651D06B0CC53B0F63BCE3C3E27D2604B", 16 + ) + g_x = botan.MPI( + "6B17D1F2E12C4247F8BCE6E563A440F277037D812DEB33A0F4A13945D898C296", 16 + ) + g_y = botan.MPI( + "4FE342E2FE1A7F9B8EE7EB4A7C0F9E162BCE33576B315ECECBB6406837BF51F5", 16 + ) + order = botan.MPI( + "FFFFFFFF00000000FFFFFFFFFFFFFFFFBCE6FAADA7179E84F3B9CAC2FC632551", 16 + ) - group_from_parameters = botan.ECGroup.from_params(secp256r1_new_oid, p, a, b, g_x, g_y, order) + group_from_parameters = botan.ECGroup.from_params( + secp256r1_new_oid, p, a, b, g_x, g_y, order + ) self.assertEqual(group_from_name, group_from_parameters) @@ -1189,15 +1356,13 @@ def test_encode(self): k = 2 input_bytes = b"abcdefgh" + b"ijklmnop" output_shares = botan.zfec_encode(k, n, input_bytes) - self.assertEqual( - output_shares, - [b'abcdefgh', b'ijklmnop', b'qrstuvwX'] - ) + self.assertEqual(output_shares, [b"abcdefgh", b"ijklmnop", b"qrstuvwX"]) def test_encode_decode(self): """ Simple round-trip tests. """ + def byte_iter(): b = 0 while True: @@ -1209,10 +1374,7 @@ def byte_iter(): for k in range(1, 5): for n in range(k, k + 5): for x in range(128, 256, 5): - input_bytes = b"".join([ - next(random_bytes) - for _ in range(x * k) - ]) + input_bytes = b"".join([next(random_bytes) for _ in range(x * k)]) with self.subTest("encode_decode variant", n=n, k=k, size=x): self._encode_decode_test(n, k, input_bytes) @@ -1228,19 +1390,16 @@ def _encode_decode_test(self, n, k, input_bytes): # "unzip" the enumerated permutation indexes, shares = zip(*inputs) decoded = botan.zfec_decode(k, n, indexes, shares) - self.assertEqual( - b"".join(decoded), - input_bytes - ) + self.assertEqual(b"".join(decoded), input_bytes) def main(): parser = argparse.ArgumentParser() - parser.add_argument('--test-data-dir', default='.') + parser.add_argument("--test-data-dir", default=".") parser.add_argument("--tpm2-tcti-name") parser.add_argument("--tpm2-tcti-conf") - parser.add_argument('unittest_args', nargs='*') + parser.add_argument("unittest_args", nargs="*") global ARGS ARGS = parser.parse_args() @@ -1248,5 +1407,6 @@ def main(): sys.argv[1:] = ARGS.unittest_args unittest.main() -if __name__ == '__main__': + +if __name__ == "__main__": main() diff --git a/src/scripts/test_strubbed_symbols.py b/src/scripts/test_strubbed_symbols.py index 294c7ac986b..c1ac7695a40 100755 --- a/src/scripts/test_strubbed_symbols.py +++ b/src/scripts/test_strubbed_symbols.py @@ -19,6 +19,7 @@ SCRIPT_LOCATION = os.path.dirname(os.path.abspath(__file__)) GDB_EXTENSION = os.path.join(SCRIPT_LOCATION, "gdb", "strubtest.py") + class CPU(Enum): ANY = "any" X86_64 = "x86_64" @@ -27,15 +28,23 @@ class CPU(Enum): @staticmethod def current(): cpu = platform.machine().lower() - if cpu in ['x86_64', 'amd64']: + if cpu in ["x86_64", "amd64"]: return CPU.X86_64 - elif cpu in ['aarch64', 'arm64']: + elif cpu in ["aarch64", "arm64"]: return CPU.AARCH64 else: raise ValueError(f"Unsupported or unknown CPU architecture: {cpu}") + class StrubTest: - def __init__(self, symbol, inferior_cmdline, masked_cpuid_bits = None, cpu = CPU.ANY, expect_fail = False): + def __init__( + self, + symbol, + inferior_cmdline, + masked_cpuid_bits=None, + cpu=CPU.ANY, + expect_fail=False, + ): self.symbol = symbol self.inferior_cmdline = inferior_cmdline self.masked_cpuid_bits = masked_cpuid_bits @@ -44,19 +53,32 @@ def __init__(self, symbol, inferior_cmdline, masked_cpuid_bits = None, cpu = CPU @property def gdb_command(self): - return ["gdb", "-x", GDB_EXTENSION, - "-ex", f"strubtest {self.symbol}", - "-ex", "run", - "--batch", - "--args", *self.inferior_cmdline] + return [ + "gdb", + "-x", + GDB_EXTENSION, + "-ex", + f"strubtest {self.symbol}", + "-ex", + "run", + "--batch", + "--args", + *self.inferior_cmdline, + ] @property def rendered_gdb_command(self): - return " ".join([token if " " not in token else f"'{token}'" for token in self.gdb_command]) + return " ".join( + [token if " " not in token else f"'{token}'" for token in self.gdb_command] + ) @property def environment(self): - return {"BOTAN_CLEAR_CPUID": ",".join(self.masked_cpuid_bits)} if self.masked_cpuid_bits else {} + return ( + {"BOTAN_CLEAR_CPUID": ",".join(self.masked_cpuid_bits)} + if self.masked_cpuid_bits + else {} + ) @property def rendered_environment(self): @@ -74,7 +96,12 @@ def run(self): return self._skip("incompatible platform") try: - proc = subprocess.run(self.gdb_command, capture_output=True, check=False, env=self.rendered_environment) + proc = subprocess.run( + self.gdb_command, + capture_output=True, + check=False, + env=self.rendered_environment, + ) except subprocess.SubprocessError as ex: return self._fail("subprocess failure", exception=ex) @@ -89,12 +116,17 @@ def run(self): else: return self._fail("never invoked", proc_result=proc) - def _print_debug_report(self, proc_result = None, exception = None): + def _print_debug_report(self, proc_result=None, exception=None): print(f" ran: {self.rendered_gdb_command}") if self.environment: print(" Environment:") - print(indent("\n".join([f"{k}={v}" for (k,v) in self.environment.items()]), " " * 6)) + print( + indent( + "\n".join([f"{k}={v}" for (k, v) in self.environment.items()]), + " " * 6, + ) + ) if exception: print(" Exception:") print(indent(str(exception), " " * 6)) @@ -106,7 +138,7 @@ def _print_debug_report(self, proc_result = None, exception = None): print(" stderr:") print(indent(proc_result.stderr.decode("utf-8"), " " * 6)) - def _fail(self, errmsg, proc_result = None, exception = None, may_be_expected = False): + def _fail(self, errmsg, proc_result=None, exception=None, may_be_expected=False): if may_be_expected and self.expect_fail: print(f"{errmsg} (expected)") return True @@ -115,7 +147,7 @@ def _fail(self, errmsg, proc_result = None, exception = None, may_be_expected = self._print_debug_report(proc_result=proc_result, exception=exception) return False - def _succeed(self, proc_result = None): + def _succeed(self, proc_result=None): if not self.expect_fail: print("ok") else: @@ -127,14 +159,16 @@ def _skip(self, reason): print(f"skipped ({reason})") return True -def dummy_file(size = 1024): + +def dummy_file(size=1024): with tempfile.NamedTemporaryFile(delete=False) as temp_file: temp_file.write(os.urandom(size)) return temp_file.name + def main(): parser = argparse.ArgumentParser() - parser.add_argument('--botan-cli', required=True) + parser.add_argument("--botan-cli", required=True) args = parser.parse_args() cli = args.botan_cli @@ -143,17 +177,37 @@ def main(): tests = [ # This is a self-test, it is expected to fail because the version # information is naturally not annotated for stack scrubbing. - StrubTest("Botan::version_string", [cli, "version", "--full"], expect_fail=True), - + StrubTest( + "Botan::version_string", [cli, "version", "--full"], expect_fail=True + ), # Below is a list of all strub-annotated symbols. Each of these # symbols is tested by running the Botan CLI with GDB and checking # that the stack scrubbing was performed correctly. - - StrubTest("Botan::SHA_256::compress_digest", [cli, "hash", "--algo=SHA-256", myfile]), - StrubTest("Botan::SHA_256::compress_digest_x86", [cli, "hash", "--algo=SHA-256", myfile], cpu=CPU.X86_64), - StrubTest("Botan::SHA_256::compress_digest_armv8", [cli, "hash", "--algo=SHA-256", myfile], cpu=CPU.AARCH64), - StrubTest("Botan::SHA_256::compress_digest_x86_avx2", [cli, "hash", "--algo=SHA-256", myfile], cpu=CPU.X86_64, masked_cpuid_bits=["intel_sha"]), - StrubTest("Botan::SHA_256::compress_digest_x86_simd", [cli, "hash", "--algo=SHA-256", myfile], cpu=CPU.X86_64, masked_cpuid_bits=["intel_sha", "avx2"]), + StrubTest( + "Botan::SHA_256::compress_digest", [cli, "hash", "--algo=SHA-256", myfile] + ), + StrubTest( + "Botan::SHA_256::compress_digest_x86", + [cli, "hash", "--algo=SHA-256", myfile], + cpu=CPU.X86_64, + ), + StrubTest( + "Botan::SHA_256::compress_digest_armv8", + [cli, "hash", "--algo=SHA-256", myfile], + cpu=CPU.AARCH64, + ), + StrubTest( + "Botan::SHA_256::compress_digest_x86_avx2", + [cli, "hash", "--algo=SHA-256", myfile], + cpu=CPU.X86_64, + masked_cpuid_bits=["intel_sha"], + ), + StrubTest( + "Botan::SHA_256::compress_digest_x86_simd", + [cli, "hash", "--algo=SHA-256", myfile], + cpu=CPU.X86_64, + masked_cpuid_bits=["intel_sha", "avx2"], + ), ] results = [test.run() for test in tests] @@ -161,5 +215,6 @@ def main(): return 1 if any(not ok for ok in results) else 0 -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/scripts/tls_scanner/tls_scanner.py b/src/scripts/tls_scanner/tls_scanner.py index 1004ffeec28..1085cc5169b 100755 --- a/src/scripts/tls_scanner/tls_scanner.py +++ b/src/scripts/tls_scanner/tls_scanner.py @@ -8,19 +8,21 @@ import subprocess import re + def format_report(client_output): - version_re = re.compile('TLS (v1\.[0-3]) using ([A-Z0-9_]+)') + version_re = re.compile("TLS (v1\.[0-3]) using ([A-Z0-9_]+)") version_match = version_re.search(client_output) - #print(client_output) + # print(client_output) if version_match: return "Established %s %s" % (version_match.group(1), version_match.group(2)) else: return client_output -def scanner(args = None): + +def scanner(args=None): if args is None: args = sys.argv @@ -31,8 +33,12 @@ def scanner(args = None): scanners = {} for url in [s.strip() for s in open(args[1]).readlines()]: - scanners[url] = subprocess.Popen(['../../../botan', 'tls_client', '--policy=policy.txt', url], - stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE) + scanners[url] = subprocess.Popen( + ["../../../botan", "tls_client", "--policy=policy.txt", url], + stdout=subprocess.PIPE, + stdin=subprocess.PIPE, + stderr=subprocess.PIPE, + ) for url in scanners.keys(): scanners[url].stdin.close() @@ -47,7 +53,7 @@ def scanner(args = None): scanners[url].poll() if scanners[url].returncode is not None: break - #print("Waiting %d more seconds for %s" % (timeout-i, url)) + # print("Waiting %d more seconds for %s" % (timeout-i, url)) time.sleep(1) if scanners[url].returncode is not None: @@ -59,5 +65,6 @@ def scanner(args = None): return 0 -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(scanner()) diff --git a/src/scripts/website.py b/src/scripts/website.py index ec30b8e3f78..84a693ca1f7 100755 --- a/src/scripts/website.py +++ b/src/scripts/website.py @@ -8,7 +8,7 @@ Botan is released under the Simplified BSD License (see license.txt) """ -import optparse # pylint: disable=deprecated-module +import optparse # pylint: disable=deprecated-module import subprocess import sys import errno @@ -16,24 +16,28 @@ import tempfile import os + def run_and_check(cmd_line, cwd=None): - print("Executing %s ..." % (' '.join(cmd_line))) + print("Executing %s ..." % (" ".join(cmd_line))) - proc = subprocess.Popen(cmd_line, - cwd=cwd, - close_fds=True, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + proc = subprocess.Popen( + cmd_line, + cwd=cwd, + close_fds=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) (stdout, stderr) = proc.communicate() if proc.returncode != 0: - print("Error running %s" % (' '.join(cmd_line))) + print("Error running %s" % (" ".join(cmd_line))) print(stdout) print(stderr) sys.exit(1) + def rmtree_ignore_missing(path): try: shutil.rmtree(path) @@ -41,26 +45,33 @@ def rmtree_ignore_missing(path): # check errno? pass + def configure_build(botan_dir, build_dir): + run_and_check( + [ + os.path.join(botan_dir, "configure.py"), + "--with-doxygen", + "--with-sphinx", + "--with-build-dir=%s" % (build_dir), + ] + ) - run_and_check([os.path.join(botan_dir, 'configure.py'), - '--with-doxygen', '--with-sphinx', - '--with-build-dir=%s' % (build_dir)]) def run_doxygen(tmp_dir, output_dir): - run_and_check(['doxygen', os.path.join(tmp_dir, 'build/botan.doxy')]) - shutil.move(os.path.join(tmp_dir, 'build/docs/doxygen'), output_dir) + run_and_check(["doxygen", os.path.join(tmp_dir, "build/botan.doxy")]) + shutil.move(os.path.join(tmp_dir, "build/docs/doxygen"), output_dir) -def run_sphinx(botan_dir, tmp_dir, output_dir): - sphinx_config = os.path.join(botan_dir, 'src/configs/sphinx') - sphinx_dir = os.path.join(tmp_dir, 'sphinx') +def run_sphinx(botan_dir, tmp_dir, output_dir): + sphinx_config = os.path.join(botan_dir, "src/configs/sphinx") + sphinx_dir = os.path.join(tmp_dir, "sphinx") os.mkdir(sphinx_dir) - shutil.copyfile(os.path.join(botan_dir, 'readme.rst'), - os.path.join(sphinx_dir, 'index.rst')) + shutil.copyfile( + os.path.join(botan_dir, "readme.rst"), os.path.join(sphinx_dir, "index.rst") + ) - for f in ['news.rst', os.path.join('doc', 'security.rst')]: + for f in ["news.rst", os.path.join("doc", "security.rst")]: shutil.copy(os.path.join(botan_dir, f), sphinx_dir) toc = """.. toctree:: @@ -72,64 +83,76 @@ def run_sphinx(botan_dir, tmp_dir, output_dir): API Reference """ - contents_rst = open(os.path.join(sphinx_dir, 'contents.rst'), 'w', encoding='utf8') + contents_rst = open(os.path.join(sphinx_dir, "contents.rst"), "w", encoding="utf8") contents_rst.write(toc) contents_rst.close() - sphinx_invoke = ['sphinx-build', '-t', 'website', '-c', sphinx_config, '-b', 'html', '-j', 'auto', '-W'] - - handbook_dir = os.path.join(botan_dir, 'doc') + sphinx_invoke = [ + "sphinx-build", + "-t", + "website", + "-c", + sphinx_config, + "-b", + "html", + "-j", + "auto", + "-W", + ] + + handbook_dir = os.path.join(botan_dir, "doc") run_and_check(sphinx_invoke + [sphinx_dir, output_dir]) - run_and_check(sphinx_invoke + [handbook_dir, os.path.join(output_dir, 'handbook')]) + run_and_check(sphinx_invoke + [handbook_dir, os.path.join(output_dir, "handbook")]) - rmtree_ignore_missing(os.path.join(output_dir, '.doctrees')) - rmtree_ignore_missing(os.path.join(output_dir, 'handbook', '.doctrees')) - os.remove(os.path.join(output_dir, '.buildinfo')) - os.remove(os.path.join(output_dir, 'handbook', '.buildinfo')) + rmtree_ignore_missing(os.path.join(output_dir, ".doctrees")) + rmtree_ignore_missing(os.path.join(output_dir, "handbook", ".doctrees")) + os.remove(os.path.join(output_dir, ".buildinfo")) + os.remove(os.path.join(output_dir, "handbook", ".buildinfo")) # share _static subdirs - shutil.rmtree(os.path.join(output_dir, 'handbook', '_static')) - os.symlink('../_static', os.path.join(output_dir, 'handbook', '_static')) + shutil.rmtree(os.path.join(output_dir, "handbook", "_static")) + os.symlink("../_static", os.path.join(output_dir, "handbook", "_static")) # Build PDF - latex_output = os.path.join(tmp_dir, 'latex') - run_and_check(['sphinx-build', '-c', sphinx_config, '-b', 'latex', handbook_dir, latex_output]) + latex_output = os.path.join(tmp_dir, "latex") + run_and_check( + ["sphinx-build", "-c", sphinx_config, "-b", "latex", handbook_dir, latex_output] + ) # Have to run twice because TeX - run_and_check(['pdflatex', 'botan.tex'], cwd=latex_output) - run_and_check(['pdflatex', 'botan.tex'], cwd=latex_output) + run_and_check(["pdflatex", "botan.tex"], cwd=latex_output) + run_and_check(["pdflatex", "botan.tex"], cwd=latex_output) - shutil.copy(os.path.join(latex_output, 'botan.pdf'), - os.path.join(output_dir, 'handbook')) + shutil.copy( + os.path.join(latex_output, "botan.pdf"), os.path.join(output_dir, "handbook") + ) def main(args): parser = optparse.OptionParser() - parser.add_option('-o', '--output-dir', default=None, - help="Where to write output") + parser.add_option("-o", "--output-dir", default=None, help="Where to write output") (options, args) = parser.parse_args(args) output_dir = options.output_dir - tmp_dir = tempfile.mkdtemp(prefix='botan_website_') + tmp_dir = tempfile.mkdtemp(prefix="botan_website_") # assumes we live in src/scripts - botan_dir = os.path.normpath(os.path.join(os.path.dirname(__file__), - "..", "..")) + botan_dir = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", "..")) - if os.access(os.path.join(botan_dir, 'configure.py'), os.X_OK) is False: + if os.access(os.path.join(botan_dir, "configure.py"), os.X_OK) is False: print("Can't find configure.py in %s", botan_dir) return 1 if output_dir is None: cwd = os.getcwd() - if os.path.basename(cwd) == 'botan-website': - output_dir = '.' + if os.path.basename(cwd) == "botan-website": + output_dir = "." else: - output_dir = os.path.join(cwd, 'botan-website') + output_dir = os.path.join(cwd, "botan-website") try: os.mkdir(output_dir) @@ -139,7 +162,7 @@ def main(args): else: raise e - for subdir in ['_static', '_sources', 'doxygen', 'handbook']: + for subdir in ["_static", "_sources", "doxygen", "handbook"]: try: shutil.rmtree(os.path.join(output_dir, subdir)) except OSError as e: @@ -153,10 +176,10 @@ def main(args): run_doxygen(tmp_dir, output_dir) run_sphinx(botan_dir, tmp_dir, output_dir) - for f in ['doc/pgpkey.txt', 'license.txt']: + for f in ["doc/pgpkey.txt", "license.txt"]: shutil.copy(os.path.join(botan_dir, f), output_dir) - favicon = open(os.path.join(output_dir, 'favicon.ico'), 'w', encoding='utf8') + favicon = open(os.path.join(output_dir, "favicon.ico"), "w", encoding="utf8") # Create an empty favicon.ico file so it gets cached by browsers favicon.close() @@ -164,5 +187,6 @@ def main(args): return 0 -if __name__ == '__main__': + +if __name__ == "__main__": sys.exit(main(sys.argv)) From 4eebbcbcee6a8474967f1d0845472b3b6de94303 Mon Sep 17 00:00:00 2001 From: Jack Lloyd Date: Sat, 6 Sep 2025 10:59:57 -0400 Subject: [PATCH 2/2] Add commit applying ruff to git-blame-ignore-revs --- src/configs/git-blame-ignore-revs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/configs/git-blame-ignore-revs b/src/configs/git-blame-ignore-revs index dd24d65e4f4..7cb6a879185 100644 --- a/src/configs/git-blame-ignore-revs +++ b/src/configs/git-blame-ignore-revs @@ -19,3 +19,6 @@ fe7a2a1d850ae02934ea04d25fa214c825442535 # [clang-format] AllowShortBlocksOnASingleLine de48a2eb6d2d77e104044da84c2fb7df5c08d65a + +# [ruff] initial application +a05a80f3a993a726c53efe513c11480c594572be