From 8f67a1f8c350ba3a7efe4c1468df032ea213c2e1 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Mon, 28 Jun 2021 11:38:49 +0200 Subject: [PATCH 01/40] move pack example and extend it --- examples/pack.py | 86 ---------- examples/upp-creator.py | 351 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 351 insertions(+), 86 deletions(-) delete mode 100644 examples/pack.py create mode 100644 examples/upp-creator.py diff --git a/examples/pack.py b/examples/pack.py deleted file mode 100644 index 110bb46..0000000 --- a/examples/pack.py +++ /dev/null @@ -1,86 +0,0 @@ -import binascii -import hashlib -import json -import logging -import pickle -import secrets -import sys -import time -from uuid import UUID - -import ubirch - -logging.basicConfig(format='%(asctime)s %(name)20.20s %(levelname)-8.8s %(message)s', level=logging.INFO) -logger = logging.getLogger() - - -######################################################################## -# Implement the ubirch-protocol with signing and saving the signatures -class Proto(ubirch.Protocol): - - def __init__(self, key_store: ubirch.KeyStore) -> None: - super().__init__() - self._ks = key_store - - def load(self, uuid: UUID): - try: - with open(uuid.hex + ".sig", "rb") as f: - signatures = pickle.load(f) - logger.debug("loaded {} known signatures".format(len(signatures))) - self.set_saved_signatures(signatures) - except: - logger.warning("no existing saved signatures") - pass - - def persist(self, uuid: UUID): - signatures = self.get_saved_signatures() - with open(uuid.hex + ".sig", "wb") as f: - pickle.dump(signatures, f) - - def _sign(self, uuid: UUID, message: bytes) -> bytes: - return self._ks.find_signing_key(uuid).sign(message) - - -######################################################################## - -if len(sys.argv) < 2: - print("usage:") - print(" python3 ./pack.py ") - print(" e.g.: python3 ./pack.py 56bd9b85-6c6e-4a24-bf71-f2ac2de10183") - sys.exit(0) - -uuid = UUID(hex=sys.argv[1]) - -# create a keystore for the device -keystore = ubirch.KeyStore("demo-device.jks", "keystore") - -# check if the device already has keys or generate a new pair -if not keystore.exists_signing_key(uuid): - keystore.create_ed25519_keypair(uuid) - -logger.info("public key [base64]: {}".format( - binascii.b2a_base64(keystore.find_verifying_key(uuid).to_bytes()).decode().rstrip('\n'))) - -# create an instance of the protocol with signature saving -protocol = Proto(keystore) -protocol.load(uuid) - -# include an ID and timestamp in the data message to ensure a unique hash -message = { - "uuid": str(uuid), - "timestamp": int(time.time()), - "data": "{:d}".format(secrets.randbits(16)) -} - -# create a compact rendering of the message to ensure determinism when creating the hash -serialized = json.dumps(message, separators=(',', ':'), sort_keys=True, ensure_ascii=False).encode() - -# calculate the hash of the message -message_hash = hashlib.sha512(serialized).digest() - -# create a new chained protocol message with the hash of the message -upp = protocol.message_chained(uuid, 0x00, message_hash) -logger.info("UPP: {}".format(binascii.hexlify(upp).decode())) - -# store signature persistently for chaining -protocol.persist(uuid) diff --git a/examples/upp-creator.py b/examples/upp-creator.py new file mode 100644 index 0000000..1382698 --- /dev/null +++ b/examples/upp-creator.py @@ -0,0 +1,351 @@ +import binascii +import hashlib +import json +import logging +import pickle +import sys +import time +import argparse +from uuid import UUID + +import ubirch + +DEFAULT_TYPE = "0x00" # binary/unknown type +DEFAULT_VERSION = "0x23" # chained upp +DEFAULT_KS = "devices.jks" +DEFAULT_KS_PWD = "keystore" +DEFAULT_KEYREG = "False" +DEFAULT_HASH = "sha512" +DEFAULT_OUTPUT = "upp.bin" +DEFAULT_NOSTDOUT = "False" + +logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) +logger = logging.getLogger() + + +######################################################################## +# Implement the ubirch-protocol with signing and saving the signatures +class Proto(ubirch.Protocol): + def __init__(self, key_store: ubirch.KeyStore) -> None: + super().__init__() + self._ks = key_store + + def load(self, uuid: UUID): + try: + with open(uuid.hex + ".sig", "rb") as f: + signatures = pickle.load(f) + logger.debug("loaded {} known signatures".format(len(signatures))) + self.set_saved_signatures(signatures) + except: + logger.warning("no existing saved signatures") + pass + + def persist(self, uuid: UUID): + signatures = self.get_saved_signatures() + with open(uuid.hex + ".sig", "wb") as f: + pickle.dump(signatures, f) + + def _sign(self, uuid: UUID, message: bytes) -> bytes: + return self._ks.find_signing_key(uuid).sign(message) +######################################################################## + + +class Main: + def __init__(self): + self.argparser : argparse.ArgumentParser = None + self.args : argparse.Namespace = None + + self.data : str = None + self.version : int = None + self.version_str : str = None + self.type : int = None + self.type_str : str = None + self.hash : str = None + self.uuid : UUID = None + self.uuid_str : str = None + self.keystore_path : str = None + self.keystore_pass : str + self.output : str = None + self.keyreg_str : str = None + self.keyreg : bool = None + self.nostdout_str : str = None + self.nostdout : bool = None + self.payload : bytes = None + + self.hasher : object = None + self.keystore : ubirch.KeyStore = None + self.proto : Proto = None + self.payload : bytes = None + self.upp : bytes = None + + # initialize the argument parser + self.setup_argparse() + + return + + def setup_argparse(self): + self.argparser = argparse.ArgumentParser( + description="Create a uBirch Protocol Package (UPP)", + epilog="Note that when using chained UPPs (--version 0x23), this tool will try to load/save signatures to UUID.sig, where UUID will be replaced with the actual UUID. " + "Make sure that the UUID.sig file is in your current working directory if you try to continue a UPP chain using this tool." + "Also beware that you will only be able to access the contents of a keystore when you use the same password you used when creating it. Otherwise all contents are lost. " + "When --hash off is set, contents of the DATA argument will be copied into the payload field of the UPP. Normally used for special messages (e.g. key registration). " + "For more information on possible values for --type and --version see https://github.com/ubirch/ubirch-protocol." + ) + + self.argparser.add_argument("uuid", metavar="UUID", type=str, + help="UUID to work with; e.g.: 56bd9b85-6c6e-4a24-bf71-f2ac2de10183" + ) + self.argparser.add_argument("data", metavar="DATA", type=str, + help="data to be packed into the UPP or hashed; e.g.: {\"t\": 23.4, \"ts\": 1624624140}" + ) + self.argparser.add_argument("--version", "-v", metavar="VERISON", type=str, default=DEFAULT_VERSION, + help="version of the UPP; 0x21 (unsigned; NOT IMPLEMENTED), 0x22 (signed) or 0x23 (chained) (default: %s)" % DEFAULT_VERSION + ) + self.argparser.add_argument("--type", "-t", metavar="TYPE", type=str, default=DEFAULT_TYPE, + help="type of the UPP (0 < type < 256); e.g.: 0x00 (unknown), 0x32 (msgpack), 0x53 (generic), ... (default and recommended: %s)" % DEFAULT_TYPE + ) + self.argparser.add_argument("--ks", "-k", metavar="KS", type=str, default=DEFAULT_KS, + help="keystore file path; e.g.: test.jks (default: %s)" % DEFAULT_KS + ) + self.argparser.add_argument("--kspwd", "-p", metavar="KSPWD", type=str, default=DEFAULT_KS_PWD, + help="keystore password; e.g.: secret (default: %s)" % DEFAULT_KS_PWD + ) + self.argparser.add_argument("--keyreg", "-r", metavar="KEYREG", type=str, default=DEFAULT_KEYREG, + help="generate a key registration UPP (data and --hash will be ignored); e.g.: true, false (default: %s)" % DEFAULT_KEYREG + ) + self.argparser.add_argument("--hash", metavar="HASH", type=str, default=DEFAULT_HASH, + help="hash algorithm for hashing the data; sha256, sha512 or off (disable hashing), ... (default and recommended: %s)" % DEFAULT_HASH + ) + self.argparser.add_argument("--output", "-o", metavar="OUTPUT", type=str, default=DEFAULT_OUTPUT, + help="file to write the generated UPP to (aside from standard output); e.g. upp.bin (default: %s)" % DEFAULT_OUTPUT + ) + self.argparser.add_argument("--nostdout", "-n", metavar="nostdout", type=str, default=DEFAULT_NOSTDOUT, + help="do not output anything to stdout; can be combined with --output /dev/stdout; e.g.: true, false (default: %s)" % DEFAULT_NOSTDOUT + ) + + def process_args(self) -> bool: + # parse cli arguments (exists on err) + self.args = self.argparser.parse_args() + + # get all needed args + self.data = self.args.data + self.uuid_str = self.args.uuid + self.version_str = self.args.version + self.type_str = self.args.type + self.keyreg_str = self.args.keyreg + self.hash = self.args.hash + self.keystore_path = self.args.ks + self.keystore_pass = self.args.kspwd + self.output = self.args.output + self.nostdout_str = self.args.nostdout + + # get the keyreg value + if self.keyreg_str.lower() in ["1", "yes", "y", "true"]: + self.keyreg = True + else: + self.keyreg = False + + # check the --hash argument + if self.hash.lower() == "sha512": + self.hasher = hashlib.sha512 + elif self.hash.lower() == "sha256": + self.hasher = hashlib.sha256 + elif self.hash.lower() == "off": + self.hasher = None + else: + logger.error("Invalid value for --hash: \"%s\" is not supported!" % self.hash) + + return False + + # check the uuid argument + try: + self.uuid = UUID(self.uuid_str) + except ValueError as e: + logger.error("Invalud UUID input string: \"%s\"!" % self.uuid_str) + logger.exception(e) + + return False + + # check the version argument + self.version = int(self.version_str, base=16 if "x" in self.version_str else 10) + + if self.version not in [0x22, 0x23]: + logger.error("Unsupported value for the --version argument: \"0x%x\" (%d)" % (self.version, self.version)) + + return False + + # check if the value for type is in range + self.type = int(self.type_str, base=16 if "x" in self.type_str else 10) + + if not (0 <= self.type < 256): + logger.error("Value for --type is out of range: \"0x%x\" (%d)" % (self.type, self.type)) + + return False + + # get the nostdout value + if self.nostdout_str.lower() in ["1", "yes", "y", "true"]: + self.nostdout = True + else: + self.nostdout = False + + # success + return True + + def init_keystore(self) -> bool: + try: + self.keystore = ubirch.KeyStore(self.keystore_path, self.keystore_pass) + + # check if the device already has keys or generate a new pair + if not self.keystore.exists_signing_key(self.uuid): + if self.nostdout == False: + logger.info("No keys found for \"%s\" in \"%s\" - generating a keypair" % (self.uuid_str, self.keystore_path)) + + self.keystore.create_ed25519_keypair(self.uuid) + + if self.nostdout == False: + logger.info("Public/Verifying key for \"%s\" [base64]: \"%s\"" % + (self.uuid_str, binascii.b2a_base64(self.keystore.find_verifying_key(self.uuid).to_bytes(), newline=False).decode())) + except Exception as e: + logger.exception(e) + + return False + + return True + + def init_proto(self) -> bool: + try: + self.proto = Proto(self.keystore) + self.proto.load(self.uuid) + except Exception as e: + logger.exception(e) + + False + + return True + + def prepare_payload(self) -> bool: + try: + if self.hasher == None: + self.payload = self.data.encode() + + if self.nostdout == False: + logger.info("UPP payload (raw data): \"%s\"" % self.payload) + else: + self.payload = self.hasher(self.data.encode()).digest() + + if self.nostdout == False: + logger.info("UPP payload (%s hash of the data) [base64]: \"%s\"" % (self.hash, binascii.b2a_base64(self.payload).decode().rstrip("\n"))) + except Exception as e: + logger.exception(e) + + return False + + return True + + def create_upp(self) -> bool: + try: + if self.keyreg == True: + # generate a key registration upp + if self.nostdout == False: + logger.info("Generating a key registration UPP for UUID \"%s\"" % self.uuid_str) + + self.upp = self.proto.message_signed(self.uuid, ubirch.ubirch_protocol.UBIRCH_PROTOCOL_TYPE_REG, self.keystore.get_certificate(self.uuid)) + pass + else: + if self.version == 0x22: + if self.nostdout == False: + logger.info("Generating a unchained signed UPP for UUID \"%s\"" % self.uuid_str) + + self.upp = self.proto.message_signed(self.uuid, self.type, self.payload) + elif self.version == 0x23: + if self.nostdout == False: + logger.info("Generating a chained signed UPP for UUID \"%s\"" % self.uuid_str) + + self.upp = self.proto.message_chained(self.uuid, self.type, self.payload) + + # save the new signature + self.proto.persist(self.uuid) + else: + # shouldnt get here/unsupported versions are caught in process_args() + raise(ValueError("Unsupported UPP version")) + except Exception as e: + logger.exception(e) + + return False + + return True + + def show_store_upp(self) -> bool: + try: + if self.nostdout == False: + logger.info("UPP [hex]: \"%s\"" % binascii.hexlify(self.upp).decode()) + + # try to write the upp + with open(self.output, "wb") as fd: + fd.write(self.upp) + + if self.nostdout == False: + logger.info("UPP written to \"%s\"" % self.output) + except Exception as e: + logger.exception(e) + + return False + + return True + + def run(self) -> int: + # process all raw argument values + if self.process_args() != True: + logger.error("Errors occured during argument processing - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # initialize the keystore + if self.init_keystore() != True: + logger.error("Errors occured while initializing the uBirch Keystore - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # initialize the uBirch protocol + if self.init_proto() != True: + logger.error("Errors occured while initializing the uBirch protocol - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # prepare the UPP payload + if self.prepare_payload() != True: + logger.error("Errors occured while preparing the UPP payload - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # create the UPP + if self.create_upp() != True: + logger.error("Errors occured while creating the UPP - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # show and store the UPP + if self.show_store_upp() != True: + logger.error("Erros occured while showing/storing the UPP - exiting!\n") + + self.argparser.print_usage() + + return 1 + + return 0 + + +# initialize/start the main class +if __name__ == "__main__": + sys.exit(Main().run()) From 653c5cdf674d22d2270858434e21d2253305ec14 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Mon, 28 Jun 2021 11:39:03 +0200 Subject: [PATCH 02/40] add script to dump keystore contents --- examples/keystore-dumper.py | 128 ++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 examples/keystore-dumper.py diff --git a/examples/keystore-dumper.py b/examples/keystore-dumper.py new file mode 100644 index 0000000..0fa8060 --- /dev/null +++ b/examples/keystore-dumper.py @@ -0,0 +1,128 @@ +import sys +import time +import argparse +import logging +import binascii +import uuid + +import ubirch + + +DEFAULT_SHOW_SIGN = "False" + + +logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) +logger = logging.getLogger() + + +class Main: + def __init__(self): + self.argparser : argparse.ArgumentParser = None + self.args : argparse.Namespace = None + + self.keystore_path : str = None + self.keystore_pass : str = None + self.show_sign_str : str = None + self.show_sign : bool = None + + self.keystore : ubirch.KeyStore = None + + # initialize the argument parser + self.setup_argparse() + + return + + def setup_argparse(self): + self.argparser = argparse.ArgumentParser( + description="Dump the contents of a keystore (.jks)", + epilog="" + ) + + self.argparser.add_argument("keystore", metavar="KEYSTORE", type=str, + help="keystore file path; e.g.: test.jks" + ) + self.argparser.add_argument("keystore_pass", metavar="KEYSTORE_PASS", type=str, + help="keystore password; e.g.: secret" + ) + self.argparser.add_argument("--show-sk", "-s", metavar="SHOW_SIGNING_KET", type=str, default=DEFAULT_SHOW_SIGN, + help="enables/disables showing of signing keys; e.g.: true, false (default: %s)" % DEFAULT_SHOW_SIGN + ) + + def process_args(self) -> bool: + # parse cli arguments (exists on err) + self.args = self.argparser.parse_args() + + # get all needed args + self.keystore_path = self.args.keystore + self.keystore_pass = self.args.keystore_pass + self.show_sign_str = self.args.show_sk + + # get the bool for show sk + if self.show_sign_str.lower() in ["1", "yes", "y", "true"]: + self.show_sign = True + else: + self.show_sign = False + + return True + + def init_keystore(self) -> bool: + try: + self.keystore = ubirch.KeyStore(self.keystore_path, self.keystore_pass) + except Exception as e: + logger.exception(e) + + return False + + return True + + def dump_keystore(self) -> bool: + verifying_keys = self.keystore._ks.certs + signing_keys = self.keystore._ks.private_keys + + # go trough the list of verifiying keys and print information for each entry + for vk_uuid in verifying_keys.keys(): + if self.show_sign == True: + t = signing_keys.get("pke_" + vk_uuid) + + sk = binascii.hexlify(t.pkey).decode() if t != None else "N / A" + else: + sk = "█" * 128 + + print("=" * 134) + print("UUID: %s" % str(uuid.UUID(hex=vk_uuid))) + print(" VK : %s" % binascii.hexlify(verifying_keys[vk_uuid].cert).decode()) + print(" SK : %s" % sk) + print("=" * 134) + + return True + + def run(self) -> int: + # process all raw argument values + if self.process_args() != True: + logger.error("Errors occured during argument processing - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # initialize the keystore + if self.init_keystore() != True: + logger.error("Errors occured while initializing the uBirch Keystore - exiting!\n") + + self.argparser.print_usage() + + return 1 + + if self.dump_keystore() != True: + logger.error("Errors occured while dumping the uBirch Keystore - exiting!\n") + + self.argparser.print_usage() + + return 1 + + return 0 + + +# initialize/start the main class +if __name__ == "__main__": + sys.exit(Main().run()) From a7706915e12f66e37cc8122133af45d7f1d842e5 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Mon, 28 Jun 2021 11:39:36 +0200 Subject: [PATCH 03/40] move verification example and extend it --- examples/test-verification.py | 26 ---- examples/upp-verifier.py | 266 ++++++++++++++++++++++++++++++++++ 2 files changed, 266 insertions(+), 26 deletions(-) delete mode 100644 examples/test-verification.py create mode 100644 examples/upp-verifier.py diff --git a/examples/test-verification.py b/examples/test-verification.py deleted file mode 100644 index d22ee63..0000000 --- a/examples/test-verification.py +++ /dev/null @@ -1,26 +0,0 @@ -from uuid import UUID - -from ed25519 import VerifyingKey - -import ubirch -from ubirch.ubirch_protocol import SIGNED - -remote_uuid = UUID(hex="6eac4d0b-16e6-4508-8c46-22e7451ea5a1") -remote_vk = VerifyingKey("b12a906051f102881bbb487ee8264aa05d8d0fcc51218f2a47f562ceb9b0d068", encoding='hex') -# a random signed ubirch-protocol message -keystore = ubirch.KeyStore("demo-device.jks", "keystore") -keystore.insert_ed25519_verifying_key(remote_uuid, remote_vk) - - -class ProtocolImpl(ubirch.Protocol): - def _verify(self, uuid: UUID, message: bytes, signature: bytes) -> dict: - return keystore.find_verifying_key(uuid).verify(signature, message) - - -proto = ProtocolImpl(SIGNED) - -message = bytes.fromhex( - "9512b06eac4d0b16e645088c4622e7451ea5a1ccef01da0040578a5b22ceb3e1" - "d0d0f8947c098010133b44d3b1d2ab398758ffed11507b607ed37dbbe006f645" - "f0ed0fdbeb1b48bb50fd71d832340ce024d5a0e21c0ebc8e0e") -print(proto.message_verify(message)) diff --git a/examples/upp-verifier.py b/examples/upp-verifier.py new file mode 100644 index 0000000..3bda61b --- /dev/null +++ b/examples/upp-verifier.py @@ -0,0 +1,266 @@ +import sys +import logging +import argparse +import msgpack +import binascii +import uuid +import ed25519 + +import ubirch + + +DEFAULT_INPUT = "upp.bin" + + +logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) +logger = logging.getLogger() + + +class Proto(ubirch.Protocol): + # UUIDs paired with public keys of uBirch Niomon on all stages + UUID_DEV = uuid.UUID(hex="9d3c78ff-22f3-4441-a5d1-85c636d486ff") + PUB_DEV = ed25519.VerifyingKey("a2403b92bc9add365b3cd12ff120d020647f84ea6983f98bc4c87e0f4be8cd66", encoding='hex') + UUID_DEMO = uuid.UUID(hex="07104235-1892-4020-9042-00003c94b60b") + PUB_DEMO = ed25519.VerifyingKey("39ff77632b034d0eba6d219c2ff192e9f24916c9a02672acb49fd05118aad251", encoding='hex') + UUID_PROD = uuid.UUID(hex="10b2e1a4-56b3-4fff-9ada-cc8c20f93016") + PUB_PROD = ed25519.VerifyingKey("ef8048ad06c0285af0177009381830c46cec025d01d86085e75a4f0041c2e690", encoding='hex') + + def __init__(self, ks : ubirch.KeyStore): + super().__init__() + + self.ks : ubirch.KeyStore = ks + + # insert all keys defined above into the keystore + if not self.ks.exists_verifying_key(self.UUID_DEV): + self.ks.insert_ed25519_verifying_key(self.UUID_DEV, self.PUB_DEV) + if not self.ks.exists_verifying_key(self.UUID_DEMO): + self.ks.insert_ed25519_verifying_key(self.UUID_DEMO, self.PUB_DEMO) + if not self.ks.exists_verifying_key(self.UUID_PROD): + self.ks.insert_ed25519_verifying_key(self.UUID_PROD, self.PUB_PROD) + + def _verify(self, uuid: uuid.UUID, message: bytes, signature: bytes): + return self.ks.find_verifying_key(uuid).verify(signature, message) + + +class Main: + def __init__(self): + self.argparser : argparse.ArgumentParser = None + self.args : argparse.Namespace = None + + self.vk_str : str = None + self.vk : ed25519.VerifyingKey = None + self.vk_uuid : uuid.UUID = None + self.vk_uuid_str : str = None + + self.input : str = None + + self.keystore : ubirch.KeyStore = None + self.proto : ubirch.Protocol = None + + self.upp : bytes = None + self.upp_uuid : uuid.UUID = None + self.upp_uuid_str : str = None + + # initialize the argument parser + self.setup_argparse() + + return + + def setup_argparse(self): + self.argparser = argparse.ArgumentParser( + description="Check if a UPP is valid/properly signed", + epilog="Note that when trying to verify a UPP sent by the uBirch backend (Niomon) a verifying key doesn't have to be provided via the -k option." + "Instead, this script will try to pick the correct stage key based on the UUID which is contained in the UPP, identifying the creator." + "If the UUID doesn't match any Niomon stage and no key was specified using -k, an error message will be printed." + ) + + self.argparser.add_argument("--verifying-key", "-k", metavar="VK", type=str, default="AUTO", + help="key to be used for verification; any verifying key in hex like \"b12a906051f102881bbb487ee8264aa05d8d0fcc51218f2a47f562ceb9b0d068\"" + ) + self.argparser.add_argument("--verifying-key-uuid", "-u", metavar="UUID", type=str, default="EMPTY", + help="the UUID for the key supplied via -k (only needed when -k is specified); e.g.: 6eac4d0b-16e6-4508-8c46-22e7451ea5a1" + ) + self.argparser.add_argument("--input", "-i", metavar="INPUT", type=str, default=DEFAULT_INPUT, + help="UPP input file path; e.g. upp.bin or /dev/stdin (default: %s)" % DEFAULT_INPUT + ) + + return + + def process_args(self) -> bool: + # parse cli arguments (exists on err) + self.args = self.argparser.parse_args() + + # get all needed args + self.vk_str = self.args.verifying_key + self.vk_uuid_str = self.args.verifying_key_uuid + self.input = self.args.input + + # check if a verifying key was supplied + if self.vk_str != "AUTO": + try: + # convert the string + self.vk = ed25519.VerifyingKey(self.vk_str, encoding="hex") + except Exception as e: + logger.error("Invalid verifying key supplied via --verifying-key/-k: \"%s\"" % self.vk_str) + logger.exception(e) + + return False + + if self.vk_uuid_str == "EMPTY": + logger.error("--verifying-key-uuid/-u must be specified when --verifying-key/-k is specified!") + + return False + + try: + self.vk_uuid = uuid.UUID(hex=self.vk_uuid_str) + except Exception as e: + logger.error("Invalid UUID supplied via --verifying-key-uuid/-u: \"%s\"" % self.vk_uuid_str) + logger.exception(e) + + return False + + return True + + def read_upp(self) -> bool: + # read the UPP from the input path + try: + logger.info("Reading the input UPP from \"%s\"" % self.input) + + with open(self.input, "rb") as fd: + self.upp = fd.read() + except Exception as e: + logger.exception(e) + + return False + + return True + + def init_keystore(self) -> bool: + try: + self.keystore = ubirch.KeyStore("-- temporary --", None) + except Exception as e: + logger.exception(e) + + return False + + return True + + def check_cli_vk(self) -> bool: + try: + if self.vk != None: + self.keystore.insert_ed25519_verifying_key(self.vk_uuid, self.vk) + + logger.info("Inserted \"%s\": \"%s\" (UUID/VK) into the keystore" % (self.vk_uuid_str, self.vk_str)) + except Exception as e: + logger.exception(e) + + return False + + return True + + def get_upp_uuid(self) -> bool: + try: + # unpack the upp + unpacked = msgpack.unpackb(self.upp) + + # get the uuid + self.upp_uuid = uuid.UUID(bytes=unpacked[1]) + self.upp_uuid_str =str(self.upp_uuid) + + logger.info("UUID of the UPP creator: \"%s\"" % self.upp_uuid_str) + except Exception as e: + logger.exception(e) + + return False + + return True + + def init_proto(self) -> bool: + try: + self.proto = Proto(self.keystore) + except Exception as e: + logger.exception(e) + + return False + + return True + + def verify_upp(self) -> bool: + try: + if self.proto.verfiy_signature(self.upp_uuid, self.upp) == True: + logger.info("Signature verified - the UPP is valid!") + else: + logger.info("The signature does not match - the UPP is invalid!") + except KeyError: + logger.error("No verifying key found for UUID \"%s\" - can't verify the UPP!" % self.upp_uuid_str) + + return False + except Exception as e: + logger.exception(e) + + return False + + return True + + def run(self): + # process all args + if self.process_args() != True: + logger.error("Errors occured during argument processing - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # read the upp + if self.read_upp() != True: + logger.error("Errors occured while reading the UPP from \"%s\" - exiting!\n" % self.input) + + self.argparser.print_usage() + + return 1 + + # initialize the keystore + if self.init_keystore() != True: + logger.error("Errors occured while initializing the keystore - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # check/insert the cli-provided verifying key + if self.check_cli_vk() != True: + logger.error("Errorc occured while inserting the verifying key into the keystore - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # get the uuid + if self.get_upp_uuid() != True: + logger.error("Errors occured while extracting the UUID from the UPP - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # initialize the Protocol + if self.init_proto() != True: + logger.error("Erros occured while initializing the Protocol - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # try to verify the message + if self.verify_upp() != True: + logger.error("Errors occured while verifying the UPP - exiting!\n") + + self.argparser.print_usage() + + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(Main().run()) \ No newline at end of file From 95dd8f79bbc98c4085fc95402b82f40d35a5fb87 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Mon, 28 Jun 2021 11:39:50 +0200 Subject: [PATCH 04/40] add script to send data and key registration upps --- examples/upp-sender.py | 248 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 248 insertions(+) create mode 100644 examples/upp-sender.py diff --git a/examples/upp-sender.py b/examples/upp-sender.py new file mode 100644 index 0000000..8d34134 --- /dev/null +++ b/examples/upp-sender.py @@ -0,0 +1,248 @@ +import sys +import logging +import argparse +import msgpack +import requests +import binascii +import uuid + +import ubirch + + +DEFAULT_ENV = "dev" +DEFAULT_INPUT = "upp.bin" +DEFAULT_OUTPUT = "response_upp.bin" + + +logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) +logger = logging.getLogger() + + +class Main: + def __init__(self): + self.argparser : argparse.ArgumentParser = None + self.args : argparse.Namespace = None + + self.uuid_str : str = None + self.uuid : uuid.UUID = None + self.auth : str = None + self.env : str = None + self.input : str = None + + self.iskeyreg : bool = False + + self.upp : bytes = None + self.api : ubirch.API = None + + # initialize the argument parser + self.setup_argparse() + + return + + def setup_argparse(self): + self.argparser = argparse.ArgumentParser( + description="Send a uBirch Protocol Package (UPP) to uBirch Niomon", + epilog="" + ) + + self.argparser.add_argument("uuid", metavar="UUID", type=str, + help="UUID to work with; e.g.: 56bd9b85-6c6e-4a24-bf71-f2ac2de10183" + ) + self.argparser.add_argument("auth", metavar="AUTH", type=str, + help="uBirch device authentication token" + ) + self.argparser.add_argument("--env", "-e", metavar="ENV", type=str, default=DEFAULT_ENV, + help="environment to operate in; dev, demo or prod (default: %s)" % DEFAULT_ENV + ) + self.argparser.add_argument("--input", "-i", metavar="INPUT", type=str, default=DEFAULT_INPUT, + help="UPP input file path; e.g. upp.bin or /dev/stdin (default: %s)" % DEFAULT_INPUT + ) + self.argparser.add_argument("--output", "-o", metavar="OUTPUT", type=str, default=DEFAULT_OUTPUT, + help="response UPP output file path (ignored for key registration UPPs); e.g. response_upp.bin (default: %s)" % DEFAULT_OUTPUT + ) + + return + + def process_args(self) -> bool: + # parse cli arguments (exists on err) + self.args = self.argparser.parse_args() + + # get all needed args + self.uuid_str = self.args.uuid + self.auth = self.args.auth + self.env = self.args.env + self.input = self.args.input + self.output = self.args.output + + # process the uuid + try: + self.uuid = uuid.UUID(hex=self.uuid_str) + except Exception as e: + logger.error("Invalid UUID: \"%s\"" % self.uuid_str) + logger.exception(e) + + return False + + # validate env + if self.env.lower() not in ["dev", "demo", "prod"]: + logger.error("Invalid value for --env: \"%s\"!" % self.env) + + return False + + return True + + def read_upp(self) -> bool: + # read the UPP from the input path + try: + logger.info("Reading the input UPP for \"%s\" from \"%s\"" % (self.uuid_str, self.input)) + + with open(self.input, "rb") as fd: + self.upp = fd.read() + except Exception as e: + logger.exception(e) + + return False + + return True + + def check_is_keyreg(self) -> bool: + # check if the UPP is a key registration UPP + try: + if msgpack.unpackb(self.upp)[2] == 1: + logger.info("The UPP is a key registration UPP - disabling identity registration check") + + self.iskeyreg = True + except Exception as e: + logger.exception(e) + + return False + + return True + + def init_api(self) -> bool: + try: + # initialize the uBirch api + self.api = ubirch.API(env=self.env, debug=True) + self.api.set_authentication(self.uuid, self.auth) + + if self.iskeyreg == False: + # check if the UUID is registered + if not self.api.is_identity_registered(self.uuid): + + logger.error("The identity for \"%s\" is not yet registered; please send a key registration UPP first" % self.uuid_str) + + return False + except Exception as e: + logger.exception(e) + + return False + + return True + + def send_upp(self) -> bool: + # niomon not accepting a UPP is not considered an error; so the return value will still be True + # this choice was made because this tool is meant for playing around/debugging etc. + + try: + # check which API function should be used + if self.iskeyreg: + r = self.api.register_identity(self.upp) + + if r.status_code == requests.codes.ok: + logger.info("The key resgistration message for \"%s\" was accepted" % self.uuid_str) + logger.info(r.content) + else: + logger.error("The key resgistration message for \"%s\" was not accepted; code: %d" % (self.uuid_str, r.status_code)) + logger.error(binascii.hexlify(r.content).decode()) + else: + r = self.api.send(self.uuid, self.upp) + + # set the response + self.response_upp = r.content + + if r.status_code == requests.codes.ok: + logger.info("The UPP for \"%s\" was accepted" % self.uuid_str) + logger.info(binascii.hexlify(r.content).decode()) + else: + logger.error("The UPP for \"%s\" was not accepted; code: %d" % (self.uuid_str, r.status_code)) + logger.error(binascii.hexlify(r.content).decode()) + + if r.status_code == 401: + logger.error("The UPP was rejected because of an authentication error! (Missing header/Invalid auth token)") + elif r.status_code == 403: + logger.error("The UPP wa rejected because of an verification error!") + except Exception as e: + logger.exception(e) + + return False + + return True + + def store_response_upp(self) -> bool: + try: + with open(self.output, "wb") as fd: + fd.write(self.response_upp) + + logger.info("The response UPP has been written to \"%s\"" % self.output) + except Exception as e: + logger.exception(e) + + return False + + return True + + def run(self): + # process all args + if self.process_args() != True: + logger.error("Errors occured during argument processing - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # read the upp + if self.read_upp() != True: + logger.error("Errors occured while reading the UPP from \"%s\" - exiting!\n" % self.input) + + self.argparser.print_usage() + + return 1 + + # check if it is a key registration upp (return value is not the result but err code) + if self.check_is_keyreg() != True: + logger.error("Errors occured while checking if the UPP is a key registration UPP - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # initialize the api + if self.init_api() != True: + logger.error("Errors occured while initializing the uBirch API - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # send the upp + if self.send_upp() != True: + logger.error("Errors occured while sending the UPP - exiting!\n") + + self.argparser.print_usage() + + return 1 + + if self.iskeyreg == False: + # store the response upp + if self.store_response_upp() != True: + logger.error("Erros occured while storing the response UPP to \"%s\" - exiting!" % self.output) + + self.argparser.print_usage() + + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(Main().run()) \ No newline at end of file From 3f924560a0d47745b2442d91e78ff9559003ee7b Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Mon, 28 Jun 2021 11:40:06 +0200 Subject: [PATCH 05/40] move unpacker --- examples/{unpack.py => upp-unpacker.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/{unpack.py => upp-unpacker.py} (100%) diff --git a/examples/unpack.py b/examples/upp-unpacker.py similarity index 100% rename from examples/unpack.py rename to examples/upp-unpacker.py From 670d7facc8287d201221da822c65d55b0e05dfa9 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Mon, 28 Jun 2021 11:40:19 +0200 Subject: [PATCH 06/40] add script to request anchoring status of an UPP --- examples/upp-anchoring-status.py | 203 +++++++++++++++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100644 examples/upp-anchoring-status.py diff --git a/examples/upp-anchoring-status.py b/examples/upp-anchoring-status.py new file mode 100644 index 0000000..4a52688 --- /dev/null +++ b/examples/upp-anchoring-status.py @@ -0,0 +1,203 @@ +import sys +import time +import argparse +import logging +import msgpack +import uuid +import base64 +import json +import requests + + +VERIFICATION_SERVICE = "https://verify.%s.ubirch.com/api/upp/verify/anchor" + +DEFAULT_ISHASH = "False" +DEFAULT_ENV = "dev" + + +logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) +logger = logging.getLogger() + + +class Main: + def __init__(self): + self.argparser : argparse.ArgumentParser = None + self.args : argparse.Namespace = None + + self.input : str = None + self.env : str = None + self.ishash_str : str = None + self.ishash : bool = None + + self.upp : bytes = None + self.hash : str = None + + # initialize the argument parser + self.setup_argparse() + + return + + def setup_argparse(self): + self.argparser = argparse.ArgumentParser( + description="Requests the verification/anchoring of a UPP from the uBirch backend", + epilog="When --ishash/-i is set to true, the input argument is treated as a base64 payload hash." + "Otherwise, it is expected to be some kind of path to read a UPP from." + "This can be a file path or also /dev/stdin if the UPP is piped to this program via standard input." + ) + + self.argparser.add_argument("input", metavar="INPUT", type=str, + help="input hash or upp path (depends on --ishash)" + ) + self.argparser.add_argument("--ishash", "-i", metavar="ISHASH", type=str, default=DEFAULT_ISHASH, + help="sets if INPUT is being treated as a hash or upp path; true or false (default: %s)" % DEFAULT_ISHASH + ) + self.argparser.add_argument("--env", "-e", metavar="ENV", type=str, default=DEFAULT_ENV, + help="the environment to operate in; dev, demo or prod (default: %s)" % DEFAULT_ENV + ) + + def process_args(self) -> bool: + # parse cli arguments (exists on err) + self.args = self.argparser.parse_args() + + # get all needed args + self.input = self.args.input + self.ishash_str = self.args.ishash + self.env = self.args.env + + # get the bool for show sk + if self.ishash_str.lower() in ["1", "yes", "y", "true"]: + self.ishash = True + else: + self.ishash = False + + return True + + def read_upp(self) -> bool: + # read the UPP from the input path + try: + logger.info("Reading the input UPP from \"%s\"" % self.input) + + with open(self.input, "rb") as fd: + self.upp = fd.read() + except Exception as e: + logger.exception(e) + + return False + + return True + + def get_hash_from_upp(self) -> bool: + try: + unpacked = msgpack.unpackb(self.upp) + + # check if this upp is signed (0x21 == unsigned) + if unpacked[0] == 0x21: + # unsigned - no signature at the end + self.hash = unpacked[-1] + else: + # signed - signature at the end + self.hash = unpacked[-2] + + logger.info("Extracted UPP hash: \"%s\"" % base64.b64encode(self.hash).decode()) + except Exception as e: + logger.exception(e) + + return False + + return True + + def get_hash_from_input(self) -> bool: + try: + self.hash = base64.b64decode(self.input) + + logger.info("Extracted hash from input: \"%s\"" % base64.b64encode(self.hash).decode()) + except Exception as e: + logger.exception(e) + + return False + + return True + + def get_status(self) -> bool: + try: + url = VERIFICATION_SERVICE % self.env + + logger.info("Requesting anchoring information from: \"%s\"" % url) + + r = requests.post( + url=url, + headers={'Accept': 'application/json', 'Content-Type': 'text/plain'}, + data=base64.b64encode(self.hash).decode().rstrip("\n") + ) + + if r.status_code == 200: + logger.info("The UPP is known to the uBirch backend! (code: %d)" % r.status_code) + + jobj = json.loads(r.content) + + print("Curr. UPP: \"%s\"" % jobj.get("upp", "-- no curr. upp information --")) + print("Prev. UPP: \"%s\"" % jobj.get("prev", "-- no prev. upp information --")) + + if jobj.get("anchors") == None: + logger.info("The UPP has NOT been anchored into any blockchains yet! Please retry later") + else: + logger.info("The UPP has been fully anchored!") + + print(jobj.get("anchors")) + elif r.status_code == 404: + logger.info("The UPP is NOT known to the uBirch backend! (code: %d)" % r.status_code) + except Exception as e: + logger.exception(e) + + return False + + return True + + def run(self) -> int: + # process all raw argument values + if self.process_args() != True: + logger.error("Errors occured during argument processing - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # check if the input data is the hash + if self.ishash == True: + # the hash can be extracted from the input parameter directly + if self.get_hash_from_input() != True: + logger.error("Errors occured while getting the hash from the input parameter - exiting!\n") + + self.argparser.print_usage() + + return 1 + else: + # the hash can be extracted from an upp which has to be read from a file + if self.read_upp() != True: + logger.error("Errors occured while reading the UPP from \"%s\" - exiting!\n" % self.input) + + self.argparser.print_usage() + + return 1 + + if self.get_hash_from_upp() != True: + logger.error("Errors occured while extracting the hash from the UPP - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # get the anchoring status + if self.get_status() != True: + logger.error("Errors occured while requesting the anchring status - exiting!\n") + + self.argparser.print_usage() + + return 1 + + return 0 + + +# initialize/start the main class +if __name__ == "__main__": + sys.exit(Main().run()) From c783b001a62fa4e746771be8439c98001fdfacb1 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Mon, 28 Jun 2021 13:19:24 +0200 Subject: [PATCH 07/40] distinguish between v1/v2 upps when splitting off the signature --- ubirch/ubirch_protocol.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ubirch/ubirch_protocol.py b/ubirch/ubirch_protocol.py index f98eb60..6250f8f 100644 --- a/ubirch/ubirch_protocol.py +++ b/ubirch/ubirch_protocol.py @@ -274,11 +274,17 @@ def upp_msgpack_split_signature(self, msgpackUPP) -> (bytes, bytes): :return: a tuple consiting of the message without the signature and the signature """ try: - return (msgpackUPP[:-66], msgpackUPP[-64:]) + if msgpackUPP[1] >> 4 == 2: + # version 2 upp - 2 byte signature header + return (msgpackUPP[:-66], msgpackUPP[-64:]) + elif msgpackUPP[1] >> 4 == 1: + # version 1 upp - 3 byte signature header + return (msgpackUPP[:-67], msgpackUPP[-64:]) + else: + raise ValueError("Invalid UPP version byte: %02x" % msgpack[1]) except IndexError: raise ValueError("The UPP-msgpack is too short: %d bytes" % len(msgpackUPP)) - # -> def verfiy_signature(self, uuid: UUID, msgpackUPP: bytes) -> True: def verfiy_signature(self, uuid: UUID, msgpackUPP: bytes) -> bool: """ Verify the integrity of the message and decode the contents From 0e6d2d2e59f949daa9eafe7c622cd6cd694ae8b0 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Mon, 28 Jun 2021 16:22:57 +0200 Subject: [PATCH 08/40] fix tests --- tests/test_ubirch_protocol.py | 9 ++++----- tests/test_ubirch_protocol_ecdsa.py | 28 +++++++++++++--------------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/tests/test_ubirch_protocol.py b/tests/test_ubirch_protocol.py index 87181c7..31341ea 100644 --- a/tests/test_ubirch_protocol.py +++ b/tests/test_ubirch_protocol.py @@ -110,7 +110,7 @@ def test_sign_not_implemented(self): def test_verify_not_implemented(self): p = ubirch.Protocol() try: - p.verfiy_signature(EXPECTED_SIGNED) + p.verfiy_signature(None, EXPECTED_SIGNED) except NotImplementedError as e: self.assertEqual(e.args[0], 'verification not implemented') @@ -169,7 +169,7 @@ def test_create_chained_message_with_hash(self): def test_verify_signed_message(self): p = Protocol() unpacked = p.unpack_upp(EXPECTED_SIGNED) - self.assertEqual(p.verfiy_signature(EXPECTED_SIGNED, unpackedUPP=unpacked), True) + self.assertEqual(p.verfiy_signature(UUID(bytes=unpacked[1]), bytes(EXPECTED_SIGNED)), True) self.assertEqual(SIGNED, unpacked[0]) self.assertEqual(TEST_UUID.bytes, unpacked[1]) self.assertEqual(0xEF, unpacked[2]) @@ -180,7 +180,7 @@ def test_verify_chained_messages(self): last_signature = b'\0' * 64 for i in range(0, 3): unpacked = p.unpack_upp(EXPECTED_CHAINED[i]) - self.assertEqual(p.verfiy_signature(EXPECTED_CHAINED[i], unpackedUPP=unpacked), True) + self.assertEqual(p.verfiy_signature(UUID(bytes=unpacked[1]), bytes(EXPECTED_CHAINED[i])), True) self.assertEqual(CHAINED, unpacked[0]) self.assertEqual(TEST_UUID.bytes, unpacked[1]) self.assertEqual(last_signature, unpacked[2]) @@ -239,7 +239,7 @@ def _verify(self, uuid: UUID, message: bytes, signature: bytes) -> bytes: unpacked = p.unpack_upp(message) - self.assertEqual(p.verfiy_signature(message, unpackedUPP=unpacked), True) + self.assertEqual(p.verfiy_signature(UUID(bytes=unpacked[1]), bytes(EXPECTED_CHAINED[i])), True) self.assertEqual(CHAINED & 0x0f, unpacked[0] & 0x0f) self.assertEqual(UUID(bytes=bytes.fromhex("af931b05acca758bc2aaeb98d6f93329")), UUID(bytes=unpacked[1])) self.assertEqual(0x54, unpacked[3]) @@ -266,7 +266,6 @@ def _verify(self, uuid: UUID, message: bytes, signature: bytes) -> bytes: unpacked = p.unpack_upp(message) - self.assertEqual(p.verfiy_signature(message, unpackedUPP=unpacked), True) self.assertEqual(SIGNED & 0x0f, unpacked[0] & 0x0f) self.assertEqual(1, unpacked[0] >> 4) self.assertEqual(UUID(bytes=bytes.fromhex("00000000000000000000000000000000")), UUID(bytes=unpacked[1])) diff --git a/tests/test_ubirch_protocol_ecdsa.py b/tests/test_ubirch_protocol_ecdsa.py index 133f3f9..aa30e25 100644 --- a/tests/test_ubirch_protocol_ecdsa.py +++ b/tests/test_ubirch_protocol_ecdsa.py @@ -87,7 +87,7 @@ def test_sign_not_implemented(self): def test_verify_not_implemented(self): p = ubirch.Protocol() try: - p.verfiy_signature(EXPECTED_SIGNED) + p.verfiy_signature(None, EXPECTED_SIGNED) except NotImplementedError as e: self.assertEqual(e.args[0], 'verification not implemented') @@ -96,14 +96,14 @@ def test_create_signed_message(self): message = p.message_signed(TEST_UUID, 0xEF, 1) logger.debug("MESSAGE: %s", binascii.hexlify(message)) self.assertEqual(EXPECTED_SIGNED[0:-64], message[0:-64]) - self.assertEqual(p.verfiy_signature(message), True) + self.assertEqual(p.verfiy_signature(TEST_UUID, message), True) def test_create_signed_message_with_hash(self): p = Protocol() message = p.message_signed(TEST_UUID, 0xEF, hashlib.sha512(b'1').digest()) logger.debug("MESSAGE: %s", binascii.hexlify(message)) self.assertEqual(EXPECTED_SIGNED_HASH, message[0:-64]) - self.assertEqual(p.verfiy_signature(message), True) + self.assertEqual(p.verfiy_signature(TEST_UUID, message), True) def test_create_chained_messages(self): p = Protocol() @@ -117,7 +117,7 @@ def test_create_chained_messages(self): logger.debug("EXPECT : %s", binascii.hexlify(EXPECTED)) self.assertEqual(EXPECTED[0:-64], message[0:-64], "message #{} failed".format(i + 1)) self.assertEqual(last_signature, message[22:22+64]) - self.assertEqual(p.verfiy_signature(message), True) + self.assertEqual(p.verfiy_signature(TEST_UUID, message), True) last_signature = message[-64:] def test_create_chained_message_with_hash(self): @@ -126,12 +126,12 @@ def test_create_chained_message_with_hash(self): logger.debug("MESSAGE: %s", binascii.hexlify(message)) self.assertEqual(EXPECTED_CHAINED_HASH[0:-64], message[0:-64]) - self.assertEqual(p.verfiy_signature(message), True) + self.assertEqual(p.verfiy_signature(TEST_UUID, message), True) def test_verify_signed_message(self): p = Protocol() unpacked = p.unpack_upp(EXPECTED_SIGNED) - self.assertEqual(p.verfiy_signature(EXPECTED_SIGNED, unpackedUPP=unpacked), True) + self.assertEqual(p.verfiy_signature(UUID(bytes=unpacked[1]), bytes(EXPECTED_SIGNED)), True) self.assertEqual(SIGNED, unpacked[0]) self.assertEqual(TEST_UUID.bytes, unpacked[1]) self.assertEqual(0xEF, unpacked[2]) @@ -142,7 +142,7 @@ def test_verify_chained_messages(self): last_signature = b'\0' * 64 for i in range(0, 3): unpacked = p.unpack_upp(EXPECTED_CHAINED[i]) - self.assertEqual(p.verfiy_signature(EXPECTED_CHAINED[i], unpackedUPP=unpacked), True) + self.assertEqual(p.verfiy_signature(UUID(bytes=unpacked[1]), bytes(EXPECTED_CHAINED[i])), True) self.assertEqual(CHAINED, unpacked[0]) self.assertEqual(TEST_UUID.bytes, unpacked[1]) self.assertEqual(last_signature, unpacked[2]) @@ -164,10 +164,8 @@ def test_verify_registration_message_sim_v1(self): unpacked = p.unpack_upp(message) - logger.debug(repr(unpacked)) - - self.assertEqual(p.verfiy_signature(message, unpackedUPP=unpacked), True) - self.assertEqual(vk, binascii.hexlify(unpacked[3]['pubKey']).decode()) + self.assertEqual(p.verfiy_signature(None, bytes(message)), True) + self.assertEqual(vk, binascii.hexlify(unpacked[3][b'pubKey']).decode()) def test_verify_signed_message_sim_v1(self): loc = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) @@ -181,7 +179,7 @@ def test_verify_signed_message_sim_v1(self): unpacked = p.unpack_upp(message) logger.debug(repr(unpacked)) - self.assertEqual(p.verfiy_signature(message, unpackedUPP=unpacked), True) + self.assertEqual(p.verfiy_signature(UUID(bytes=unpacked[1]), bytes(message)), True) self.assertEqual(hashlib.sha256(b"UBIRCH").digest(), unpacked[3]) def test_verify_registration_message_sim_v2(self): @@ -196,7 +194,7 @@ def test_verify_registration_message_sim_v2(self): unpacked = p.unpack_upp(message) logger.debug(repr(unpacked)) - self.assertEqual(p.verfiy_signature(message, unpackedUPP=unpacked), True) + self.assertEqual(p.verfiy_signature(UUID(bytes=unpacked[1]), message), True) self.assertEqual(vk, binascii.hexlify(unpacked[3]['pubKey']).decode()) def test_verify_signed_message_sim_v2(self): @@ -211,5 +209,5 @@ def test_verify_signed_message_sim_v2(self): unpacked = p.unpack_upp(message) logger.debug(repr(unpacked)) - self.assertEqual(p.verfiy_signature(message, unpackedUPP=unpacked), True) - self.assertEqual(hashlib.sha256(b"UBIRCH").digest(), unpacked[3]) \ No newline at end of file + self.assertEqual(p.verfiy_signature(UUID(bytes=unpacked[1]), message), True) + self.assertEqual(hashlib.sha256(b"UBIRCH").digest(), unpacked[3]) From b8a614b97c291655281bda06377a5e7c49b3c08b Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Thu, 1 Jul 2021 18:00:27 +0200 Subject: [PATCH 09/40] add a script to send data to the uBirch Simple Data Service --- examples/data-sender.py | 145 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 examples/data-sender.py diff --git a/examples/data-sender.py b/examples/data-sender.py new file mode 100644 index 0000000..136e5df --- /dev/null +++ b/examples/data-sender.py @@ -0,0 +1,145 @@ +import sys +import logging +import argparse +import msgpack +import requests +import binascii +import uuid + +import ubirch + + +DEFAULT_ENV = "dev" + + +logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) +logger = logging.getLogger() + + +class Main: + def __init__(self): + self.argparser : argparse.ArgumentParser = None + self.args : argparse.Namespace = None + + self.uuid_str : str = None + self.uuid : uuid.UUID = None + self.auth : str = None + self.env : str = None + self.input : str = None + + self.api : ubirch.API = None + + # initialize the argument parser + self.setup_argparse() + + return + + def setup_argparse(self): + self.argparser = argparse.ArgumentParser( + description="Send some data to the uBirch Simple Data Service", + epilog="Note that the input data should follow this pattern: " + "{\"timestamp\": TIMESTAMP, \"uuid\": \"UUID\", \"msg_type\": 0, \"data\": DATA, \"hash\": \"UPP_HASH\"}. " + "For more information take a look at the EXAMPLES.md file." + ) + + self.argparser.add_argument("uuid", metavar="UUID", type=str, + help="UUID to work with; e.g.: 56bd9b85-6c6e-4a24-bf71-f2ac2de10183" + ) + self.argparser.add_argument("auth", metavar="AUTH", type=str, + help="uBirch device authentication token" + ) + self.argparser.add_argument("--env", "-e", metavar="ENV", type=str, default=DEFAULT_ENV, + help="environment to operate in; dev, demo or prod (default: %s)" % DEFAULT_ENV + ) + self.argparser.add_argument("input", metavar="INPUT", type=str, + help="data to be sent to the simple data service" + ) + + return + + def process_args(self) -> bool: + # parse cli arguments (exists on err) + self.args = self.argparser.parse_args() + + # get all needed args + self.uuid_str = self.args.uuid + self.auth = self.args.auth + self.env = self.args.env + self.input = self.args.input + + # process the uuid + try: + self.uuid = uuid.UUID(hex=self.uuid_str) + except Exception as e: + logger.error("Invalid UUID: \"%s\"" % self.uuid_str) + logger.exception(e) + + return False + + # validate env + if self.env.lower() not in ["dev", "demo", "prod"]: + logger.error("Invalid value for --env: \"%s\"!" % self.env) + + return False + + return True + + def init_api(self) -> bool: + try: + # initialize the uBirch api + self.api = ubirch.API(env=self.env, debug=True) + self.api.set_authentication(self.uuid, self.auth) + except Exception as e: + logger.exception(e) + + return False + + return True + + def send_data(self) -> bool: + try: + r = self.api.send_data(self.uuid, self.input.encode()) + + # check the response + if r.status_code == 200: + logger.info("Successfully sent all data to the Simple Data Service! (%d)" % r.status_code) + else: + logger.error("Failed to send data to the Simple Data Service! (%d)" % r.status_code) + logger.error(r.content) + except Exception as e: + logger.exception(e) + + return False + + return True + + def run(self): + # process all args + if self.process_args() != True: + logger.error("Errors occured during argument processing - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # initialize the api + if self.init_api() != True: + logger.error("Errors occured while initializing the uBirch API - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # send data + if self.send_data() != True: + logger.error("Errors occured while sending data to the simple data service - exiting!\n") + + self.argparser.print_usage() + + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(Main().run()) \ No newline at end of file From 5b90291f33f1e0a294fca039efc391ce0bb75331 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Thu, 1 Jul 2021 18:13:08 +0200 Subject: [PATCH 10/40] fix old comment --- examples/upp-anchoring-status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/upp-anchoring-status.py b/examples/upp-anchoring-status.py index 4a52688..ae7fce9 100644 --- a/examples/upp-anchoring-status.py +++ b/examples/upp-anchoring-status.py @@ -64,7 +64,7 @@ def process_args(self) -> bool: self.ishash_str = self.args.ishash self.env = self.args.env - # get the bool for show sk + # get the bool for ishash if self.ishash_str.lower() in ["1", "yes", "y", "true"]: self.ishash = True else: From caed4639c7a875175a654577a2ce1a369a4f7ace Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Thu, 1 Jul 2021 18:40:09 +0200 Subject: [PATCH 11/40] only print anchoring information if the UPP is anchored --- examples/upp-anchoring-status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/upp-anchoring-status.py b/examples/upp-anchoring-status.py index ae7fce9..9195ef4 100644 --- a/examples/upp-anchoring-status.py +++ b/examples/upp-anchoring-status.py @@ -143,7 +143,7 @@ def get_status(self) -> bool: else: logger.info("The UPP has been fully anchored!") - print(jobj.get("anchors")) + print(jobj.get("anchors")) elif r.status_code == 404: logger.info("The UPP is NOT known to the uBirch backend! (code: %d)" % r.status_code) except Exception as e: From 53bab4f1b09234b3eea6ec7e766cca5da7139489 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Thu, 1 Jul 2021 19:53:30 +0200 Subject: [PATCH 12/40] move and extend data verifier script --- examples/data-verifier.py | 289 ++++++++++++++++++++++++++++++++++++++ examples/data_verifier.py | 28 ---- 2 files changed, 289 insertions(+), 28 deletions(-) create mode 100644 examples/data-verifier.py delete mode 100644 examples/data_verifier.py diff --git a/examples/data-verifier.py b/examples/data-verifier.py new file mode 100644 index 0000000..830a28a --- /dev/null +++ b/examples/data-verifier.py @@ -0,0 +1,289 @@ + +# import hashlib +# import json + +# import requests + +# VERIFICATION_SERVICE = "https://verify.prod.ubirch.com/api/upp/verify/anchor" + +# with open("data_to_verify.json") as f: +# message = json.load(f) + +# # create a compact rendering of the message to ensure determinism when creating the hash +# serialized = json.dumps(message, separators=(',', ':'), sort_keys=True, ensure_ascii=False).encode() +# print("rendered data:\n\t{}\n".format(serialized.decode())) + +# # calculate hash of message +# data_hash = hashlib.sha256(serialized).digest() +# print("hash [base64]:\n\t{}\n".format(base64.b64encode(data_hash).decode())) + +# # verify existence of the hash in the UBIRCH backend +# r = requests.post(url=VERIFICATION_SERVICE, +# headers={'Accept': 'application/json', 'Content-Type': 'text/plain'}, +# data=base64.b64encode(data_hash).decode().rstrip('\n')) + +# if 200 <= r.status_code < 300: +# print("verification successful:\n\t{}\n".format(r.content.decode())) +# else: +# print("verification FAIL: ({})\n\tdata hash could not be verified\n".format(r.status_code)) + +import sys +import argparse +import base64 +import logging +import json +import hashlib +import requests + + +VERIFICATION_SERVICE = "https://verify.%s.ubirch.com/api/upp/verify/anchor" + +DEFAULT_ISPATH = "False" +DEFAULT_ENV = "dev" +DEFAULT_ISJSON = "True" +DEFAULT_HASH = "sha256" + + +logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) +logger = logging.getLogger() + + +class Main: + def __init__(self): + self.argparser : argparse.ArgumentParser = None + self.args : argparse.Namespace = None + + self.input : str = None + self.env : str = None + self.ispath_str : str = None + self.ispath : bool = None + self.isjson_str : str = None + self.isjson : bool = None + self.hashalg : str = None + + self.ishash : bool = False + self.hasher : object = None + + self.data : bytes = None + self.hash : str = None + + # initialize the argument parser + self.setup_argparse() + + return + + def setup_argparse(self): + self.argparser = argparse.ArgumentParser( + description="Check if the hash of given input data is known to the uBirch backend (verify it)", + epilog="When --ispath/-i is set to true, the input data is treated as a file path to read the " + "actual input data from. When setting --hash/-a to off, the input argument is expected " + "to be a valid base64 encoded hash." + ) + + self.argparser.add_argument("input", metavar="INPUT", type=str, + help="input data or data file path (depends on --ispath)" + ) + self.argparser.add_argument("--ispath", "-i", metavar="ISHASH", type=str, default=DEFAULT_ISPATH, + help="sets if INPUT is being treated as data or data file path; true or false (default: %s)" % DEFAULT_ISPATH + ) + self.argparser.add_argument("--env", "-e", metavar="ENV", type=str, default=DEFAULT_ENV, + help="the environment to operate in; dev, demo or prod (default: %s)" % DEFAULT_ENV + ) + self.argparser.add_argument("--isjson", "-j", metavar="ISJSON", type=str, default=DEFAULT_ISJSON, + help="tells the script to treat the input data as json and serealize it (see EXAMPLES.md for more information); true or false (default: %s)" % DEFAULT_ISJSON + ) + self.argparser.add_argument("--hash", "-a", metavar="HASH", type=str, default=DEFAULT_HASH, + help="sets the hash algorithm to use; sha256, sha512 or OFF to treat the input data as hash (default: %s)" % DEFAULT_HASH + ) + + def process_args(self) -> bool: + # parse cli arguments (exists on err) + self.args = self.argparser.parse_args() + + # get all needed args + self.input = self.args.input + self.ispath_str = self.args.ispath + self.isjson_str = self.args.isjson + self.env = self.args.env + self.hashalg = self.args.hash + + # check the value for --hash + if self.hashalg.lower() == "off": + self.ishash = True + elif self.hashalg.lower() == "sha256": + self.hasher = hashlib.sha256() + self.ishash = False + elif self.hashalg.lower() == "sha512": + self.hasher = hashlib.sha512() + self.ishash = False + else: + logger.error("the value for --hash/-a must be \"sha256\", \"sha512\" or \"off\"; \"%s\" is invalid!" % self.hashalg) + + return False + + # check the value for --env + if self.env not in ["prod", "demo", "dev"]: + logger.error("the value for --env/-e must be \"prod\", \"demo\" or \"dev\"; \"%s\" is invalid!" % self.env) + + return False + + # get the bool for ispath + if self.ispath_str.lower() in ["1", "yes", "y", "true"]: + self.ispath = True + else: + self.ispath = False + + # get the bool for isjson + if self.isjson_str.lower() in ["1", "yes", "y", "true"]: + self.isjson = True + else: + self.isjson = False + + return True + + def read_data(self) -> bool: + # read data from the input path + try: + logger.info("Reading the input data from \"%s\"" % self.input) + + with open(self.input, "rb") as fd: + self.data = fd.read() + except Exception as e: + logger.exception(e) + + return False + + return True + + def serialize_json(self) -> bool: + try: + # load the string as json and put it back into a string, serealizing it + self.data = json.loads(self.data) + self.data = json.dumps(self.data, separators=(',', ':'), sort_keys=True, ensure_ascii=False).encode() + + logger.info("Serialized JSON: \"%s\"" % self.data.decode()) + except Exception as e: + logger.exception(e) + + return False + + return True + + def get_hash_from_data(self) -> bool: + try: + # calculate the hash + self.hasher.update(self.data) + + self.hash = self.hasher.digest() + self.hash = base64.b64encode(self.hash).decode().rstrip("\n") + + logger.info("Calculated hash: \"%s\"" % self.hash) + except Exception as e: + logger.exception(e) + + return False + + return True + + def get_hash_from_input(self) -> bool: + self.hash = self.input + + return True + + def get_status(self) -> bool: + try: + url = VERIFICATION_SERVICE % self.env + + logger.info("Requesting anchoring information from: \"%s\"" % url) + + r = requests.post( + url=url, + headers={'Accept': 'application/json', 'Content-Type': 'text/plain'}, + data=self.hash + ) + + if r.status_code == 200: + logger.info("The hash is known to the uBirch backend! (code: %d)" % r.status_code) + + jobj = json.loads(r.content) + + print("Curr. UPP: \"%s\"" % jobj.get("upp", "-- no curr. upp information --")) + print("Prev. UPP: \"%s\"" % jobj.get("prev", "-- no prev. upp information --")) + + if jobj.get("anchors") in [None, []]: + logger.info("The corresponding UPP has NOT been anchored into any blockchains yet! Please retry later") + else: + logger.info("The corresponding UPP has been fully anchored!") + + print(jobj.get("anchors")) + elif r.status_code == 404: + logger.info("The hash is NOT known to the uBirch backend! (code: %d)" % r.status_code) + except Exception as e: + logger.exception(e) + + return False + + return True + + def run(self) -> int: + # process all raw argument values + if self.process_args() != True: + logger.error("Errors occured during argument processing - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # check if the input data is the hash + if self.ishash == True: + # the hash can be extracted from the input parameter directly + if self.get_hash_from_input() != True: + logger.error("Errors occured while getting the hash from the input parameter - exiting!\n") + + self.argparser.print_usage() + + return 1 + else: + # check if the input is a path or the actual data + if self.ispath == True: + # read the data from the given path/file + if self.read_data() != True: + logger.error("Errors occured while reading data from \"%s\" - exiting!\n" % self.input) + + self.argparser.print_usage() + + return 1 + else: + self.data = self.input + + # check if the input data is json/should be serialized + if self.isjson == True: + if self.serialize_json() != True: + logger.error("Error occured while serealizing the JSON data - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # calculate the hash + if self.get_hash_from_data() != True: + logger.error("Error calculating the hash - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # get the anchoring status + if self.get_status() != True: + logger.error("Errors occured while requesting the anchring status - exiting!\n") + + self.argparser.print_usage() + + return 1 + + return 0 + + +# initialize/start the main class +if __name__ == "__main__": + sys.exit(Main().run()) diff --git a/examples/data_verifier.py b/examples/data_verifier.py deleted file mode 100644 index f14bbb5..0000000 --- a/examples/data_verifier.py +++ /dev/null @@ -1,28 +0,0 @@ -import base64 -import hashlib -import json - -import requests - -VERIFICATION_SERVICE = "https://verify.prod.ubirch.com/api/upp/verify/anchor" - -with open("data_to_verify.json") as f: - message = json.load(f) - -# create a compact rendering of the message to ensure determinism when creating the hash -serialized = json.dumps(message, separators=(',', ':'), sort_keys=True, ensure_ascii=False).encode() -print("rendered data:\n\t{}\n".format(serialized.decode())) - -# calculate hash of message -data_hash = hashlib.sha256(serialized).digest() -print("hash [base64]:\n\t{}\n".format(base64.b64encode(data_hash).decode())) - -# verify existence of the hash in the UBIRCH backend -r = requests.post(url=VERIFICATION_SERVICE, - headers={'Accept': 'application/json', 'Content-Type': 'text/plain'}, - data=base64.b64encode(data_hash).decode().rstrip('\n')) - -if 200 <= r.status_code < 300: - print("verification successful:\n\t{}\n".format(r.content.decode())) -else: - print("verification FAIL: ({})\n\tdata hash could not be verified\n".format(r.status_code)) From b960adc69c245579ea68abab75f2c49fe9ab328a Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Thu, 1 Jul 2021 19:53:54 +0200 Subject: [PATCH 13/40] fix result displaying --- examples/upp-anchoring-status.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/upp-anchoring-status.py b/examples/upp-anchoring-status.py index 9195ef4..1c39f81 100644 --- a/examples/upp-anchoring-status.py +++ b/examples/upp-anchoring-status.py @@ -40,8 +40,8 @@ def __init__(self): def setup_argparse(self): self.argparser = argparse.ArgumentParser( description="Requests the verification/anchoring of a UPP from the uBirch backend", - epilog="When --ishash/-i is set to true, the input argument is treated as a base64 payload hash." - "Otherwise, it is expected to be some kind of path to read a UPP from." + epilog="When --ishash/-i is set to true, the input argument is treated as a base64 payload hash. " + "Otherwise, it is expected to be some kind of path to read a UPP from. " "This can be a file path or also /dev/stdin if the UPP is piped to this program via standard input." ) @@ -138,7 +138,7 @@ def get_status(self) -> bool: print("Curr. UPP: \"%s\"" % jobj.get("upp", "-- no curr. upp information --")) print("Prev. UPP: \"%s\"" % jobj.get("prev", "-- no prev. upp information --")) - if jobj.get("anchors") == None: + if jobj.get("anchors") in [None, []]: logger.info("The UPP has NOT been anchored into any blockchains yet! Please retry later") else: logger.info("The UPP has been fully anchored!") From f0ec0624ddf4c2f5ab03f87f0af9e5aee995c75a Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Thu, 1 Jul 2021 19:54:26 +0200 Subject: [PATCH 14/40] add ability to serialize json before creating the hash for the UPP --- examples/upp-creator.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/examples/upp-creator.py b/examples/upp-creator.py index 1382698..785ada9 100644 --- a/examples/upp-creator.py +++ b/examples/upp-creator.py @@ -16,6 +16,7 @@ DEFAULT_KS_PWD = "keystore" DEFAULT_KEYREG = "False" DEFAULT_HASH = "sha512" +DEFAULT_ISJSON = "False" DEFAULT_OUTPUT = "upp.bin" DEFAULT_NOSTDOUT = "False" @@ -66,6 +67,8 @@ def __init__(self): self.keystore_path : str = None self.keystore_pass : str self.output : str = None + self.isjson_str : str = None + self.isjson : bool = None self.keyreg_str : str = None self.keyreg : bool = None self.nostdout_str : str = None @@ -117,6 +120,9 @@ def setup_argparse(self): self.argparser.add_argument("--hash", metavar="HASH", type=str, default=DEFAULT_HASH, help="hash algorithm for hashing the data; sha256, sha512 or off (disable hashing), ... (default and recommended: %s)" % DEFAULT_HASH ) + self.argparser.add_argument("--isjson", "-j", metavar="ISJSON", type=str, default=DEFAULT_ISJSON, + help="tells the script to treat the input data as json and serealize it (see EXAMPLES.md for more information); true or false (default: %s)" % DEFAULT_ISJSON + ) self.argparser.add_argument("--output", "-o", metavar="OUTPUT", type=str, default=DEFAULT_OUTPUT, help="file to write the generated UPP to (aside from standard output); e.g. upp.bin (default: %s)" % DEFAULT_OUTPUT ) @@ -135,6 +141,7 @@ def process_args(self) -> bool: self.type_str = self.args.type self.keyreg_str = self.args.keyreg self.hash = self.args.hash + self.isjson_str = self.args.isjson self.keystore_path = self.args.ks self.keystore_pass = self.args.kspwd self.output = self.args.output @@ -189,6 +196,12 @@ def process_args(self) -> bool: else: self.nostdout = False + # get the isjson value + if self.isjson_str.lower() in ["1", "yes", "y", "true"]: + self.isjson = True + else: + self.isjson = False + # success return True @@ -232,6 +245,13 @@ def prepare_payload(self) -> bool: if self.nostdout == False: logger.info("UPP payload (raw data): \"%s\"" % self.payload) else: + if self.isjson == True: + # load the string as json and put it back into a string, serealizing it + self.data = json.loads(self.data) + self.data = json.dumps(self.data, separators=(',', ':'), sort_keys=True, ensure_ascii=False) + + logger.info("Serialized data JSON: \"%s\"" % self.data) + self.payload = self.hasher(self.data.encode()).digest() if self.nostdout == False: From 5fc0190b8eb1d15c745021c53a97793153e88a2c Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Fri, 2 Jul 2021 14:31:19 +0200 Subject: [PATCH 15/40] fix invalid indentation --- examples/upp-creator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/upp-creator.py b/examples/upp-creator.py index 785ada9..d089b6c 100644 --- a/examples/upp-creator.py +++ b/examples/upp-creator.py @@ -270,7 +270,7 @@ def create_upp(self) -> bool: if self.nostdout == False: logger.info("Generating a key registration UPP for UUID \"%s\"" % self.uuid_str) - self.upp = self.proto.message_signed(self.uuid, ubirch.ubirch_protocol.UBIRCH_PROTOCOL_TYPE_REG, self.keystore.get_certificate(self.uuid)) + self.upp = self.proto.message_signed(self.uuid, ubirch.ubirch_protocol.UBIRCH_PROTOCOL_TYPE_REG, self.keystore.get_certificate(self.uuid)) pass else: if self.version == 0x22: From 3775c12d284c121fd34976a74e799349af36d681 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Fri, 2 Jul 2021 16:57:56 +0200 Subject: [PATCH 16/40] fix parameter description --- examples/data-verifier.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/data-verifier.py b/examples/data-verifier.py index 830a28a..87f8c61 100644 --- a/examples/data-verifier.py +++ b/examples/data-verifier.py @@ -83,7 +83,7 @@ def setup_argparse(self): self.argparser.add_argument("input", metavar="INPUT", type=str, help="input data or data file path (depends on --ispath)" ) - self.argparser.add_argument("--ispath", "-i", metavar="ISHASH", type=str, default=DEFAULT_ISPATH, + self.argparser.add_argument("--ispath", "-i", metavar="ISPATH", type=str, default=DEFAULT_ISPATH, help="sets if INPUT is being treated as data or data file path; true or false (default: %s)" % DEFAULT_ISPATH ) self.argparser.add_argument("--env", "-e", metavar="ENV", type=str, default=DEFAULT_ENV, From 25a15fb16af986f05f19fc04d28071fa05be86e9 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Fri, 2 Jul 2021 17:01:30 +0200 Subject: [PATCH 17/40] add documentation for some examples --- examples/EXAMPLES.md | 365 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 365 insertions(+) create mode 100644 examples/EXAMPLES.md diff --git a/examples/EXAMPLES.md b/examples/EXAMPLES.md new file mode 100644 index 0000000..38d2f71 --- /dev/null +++ b/examples/EXAMPLES.md @@ -0,0 +1,365 @@ +# uBirch-Protocol-Python Examples +This file documents how to use the examples provided alongside the [uBirch-Protocol-Python](https://github.com/ubirch/ubirch-protocol-python). Those examples aim to provide an insight of how to use the [ubirch-protocol](https://pypi.org/project/ubirch-protocol/) python library, which is implemented in the `/ubirch/` directory in this repository. + +## Table of contents +- [uBirch-Protocol-Python Examples](#ubirch-protocol-python-examples) + - [Table of contents](#table-of-contents) + - [From measurement to blockchain-anchored UPP](#from-measurement-to-blockchain-anchored-upp) + - [Setup](#setup) + - [Generating and managing a keypair](#generating-and-managing-a-keypair) + - [Registering a public key](#registering-a-public-key) + - [Gathering Data](#gathering-data) + - [Creating a UPP](#creating-a-upp) + - [Sending a UPP](#sending-a-upp) + - [Verifying a UPP](#verifying-a-upp) + - [Examining a UPP](#examining-a-upp) + - [Checking the anchoring status of an UPP](#checking-the-anchoring-status-of-an-upp) + - [Verifying a measurement](#verifying-a-measurement) + - [Sending data to the Simple Data Service](#sending-data-to-the-simple-data-service) + - [Example uBirch client implementation](#example-ubirch-client-implementation) + +## From measurement to blockchain-anchored UPP +The process needed to get a UPP to be anchored in the blockchain can be cut down into multiple steps. For each of those steps there is an example in this directory, demonstrating how one could handle them. There also are examples showing a full example-client implementation. + +0. [Setup](#setup) +1. [Gathering Data](#gathering-data) + +### Setup +Before anything, you will need to do/get a couple of things: +- Choose a stage to work on +- Get a account for the uBirch-Console + - https://console.prod.ubirch.com for the `prod` stage + - https://console.demo.ubirch.com for the `demo` stage + - https://console.dev.ubirch.com for the `dev` stage +- Get a UUID (can be generated randomly or on the basis of certain device properties like MAC-Addresses) +- Create a "Thing" at the uBirch-Console; remember/note down the used UUID and the generated Auth-Token + +You should now have the following information at hand: +- The stage you want to work on (later referred to as `env`) +- The UUID of your device or "fake" device in this instance +- The authentication token (`auth token`) for the named UUID + +The values used below are `f5ded8a3-d462-41c4-a8dc-af3fd072a217` for the UUID, `demo` for the env and +`xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` for the auth token. + +### Generating and managing a keypair +To create, or more precisely, to _sign_ a UPP, a device will need a keypair. This keypair consist of a private key (_signing key_) and public key (_verifying key_). The signing key is used to sign UPPs and the verifying key can be used by the uBirch backend to check if the signature is valid and belongs to the correct sender/signer. So logically, it doesn't matter who knows to verifying key, but the signing key must be kept secret all the time. In a real usecase, a device might store it in a TPM ([Trusted platform module](https://en.wikipedia.org/wiki/Trusted_Platform_Module)) or use other counter measures against an attacker reading the key from the device. For this demo, keypairs will be stored in a [JKS Keystore](https://en.wikipedia.org/wiki/Java_KeyStore) using the [`pyjks`](https://pypi.org/project/pyjks/) library. For that you will have to chose and remember a file path for that keystore and a password used to encrypt it. The process of actually generating the keypair is exlpained [bellow](#registering-a-public-key). + +**NOTE** that losing access to the signing key, especially if it is already registered at the uBirch backend, will take away the ability to create and send any new UPPs from that device/UUID, since there is no way of creating a valid signature that would be accepted by the backend. + +### Registering a public key +To enable the uBirch backend to verify a UPP, it needs to know the corresponding verifying key. Therefore, the device needs to send this key to the backend before starting to send UPPs supposed to be verified and anchored. Registering a verifying key is also done by sending a special kind of UPP containing this key. This can be done by using two scripts: +``` +upp-creator.py +upp-sender.py +``` +Both of these scripts will be explained in more detail in [Creating a UPP](#creating-a-upp) and [Sending a UPP](#sending-a-upp). To generate a _Public Key Registration UPP_ this command can be used: +``` +$ python upp-creator.py -t 1 --ks devices.jks --kspwd keystore --keyreg true --output keyreg_upp.bin f5ded8a3-d462-41c4-a8dc-af3fd072a217 none + +2021-07-02 11:51:50,483 root init_keystore() INFO No keys found for "f5ded8a3-d462-41c4-a8dc-af3fd072a217" in "devices.jks" - generating a keypair +2021-07-02 11:51:50,485 ubirch.ubirch_ks insert_ed25519_keypa() INFO inserted new key pair for f5ded8a3d46241c4a8dcaf3fd072a217: e0264e7d9428149cef59ccecb8813b214d8f94c62e3e836d7546d3f8bd884a4c +2021-07-02 11:51:50,485 root init_keystore() INFO Public/Verifying key for "f5ded8a3-d462-41c4-a8dc-af3fd072a217" [base64]: "4CZOfZQoFJzvWczsuIE7IU2PlMYuPoNtdUbT+L2ISkw=" +2021-07-02 11:51:50,485 root load() WARNING no existing saved signatures +2021-07-02 11:51:50,485 root prepare_payload() INFO UPP payload (sha512 hash of the data) [base64]: "q5Op6V1w7bBgJVEc6k4rgEf7fh3q9yRPwNPt9efLV9j7e5Ub3rPGtVJxSHh0nrGbkQPmSoNjXoiFx9Ph0PxWSQ==" +2021-07-02 11:51:50,485 root create_upp() INFO Generating a key registration UPP for UUID "f5ded8a3-d462-41c4-a8dc-af3fd072a217" +2021-07-02 11:51:50,485 root show_store_upp() INFO UPP [hex]: "9522c410f5ded8a3d46241c4a8dcaf3fd072a2170187a9616c676f726974686dab4543435f45443235353139a763726561746564ce60dec596aa68774465766963654964c410f5ded8a3d46241c4a8dcaf3fd072a216a67075624b6579c420e0264e7d9428149cef59ccecb8813b214d8f94c62e3e836d7546d3f8bd884a4ca87075624b65794964c420e0264e7d9428149cef59ccecb8813b214d8f94c62e3e836d7546d3f8bd884a4cad76616c69644e6f744166746572ce62bff916ae76616c69644e6f744265666f7265ce60dec596c440cadb70d30250a5a2dd2eb44b645e54b56387f228607fbf6f59a11493befa118f0e9c79da1f7d85ba5a4076c134f8b4aff04173adfc4b858ec491be2366988900" +2021-07-02 11:51:50,485 root show_store_upp() INFO UPP written to "keyreg_upp.bin" +``` +The generated key registration UPP has been saved to `keyreg_upp.bin`. Sending the UPP can be done like following (remember to put the correct value for the env of your choice): +``` +$ python upp-sender.py --env demo --input keyreg_upp.bin --output response_upp.bin f5ded8a3-d462-41c4-a8dc-af3fd072a217 xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + +2021-07-02 12:05:49,556 root read_upp() INFO Reading the input UPP for "f5ded8a3-d462-41c4-a8dc-af3fd072a217" from "keyreg_upp.bin" +2021-07-02 12:05:49,556 root check_is_keyreg() INFO The UPP is a key registration UPP - disabling identity registration check +2021-07-02 12:05:50,712 root send_upp() INFO The key resgistration message for "f5ded8a3-d462-41c4-a8dc-af3fd072a217" was accepted +2021-07-02 12:05:50,712 root send_upp() INFO b'{"pubKeyInfo":{"algorithm":"ECC_ED25519","created":"2021-06-27T13:12:46.000Z","hwDeviceId":"f5ded8a3-d462-41c4-a8dc-af3fd072a217","pubKey":"dsbKkw9HpsTvlLGgmiaYAM4M/ytFcySoF5UbfScffxg=","pubKeyId":"dsbKkw9HpsTvlLGgmiaYAM4M/ytFcySoF5UbfScffxg=","validNotAfter":"2022-06-27T13:12:46.000Z","validNotBefore":"2021-06-27T13:12:46.000Z"},"signature":"1e47255c7494fb54d5ab10897d53c1a38872e6a67af3d53e5ae49d6c190d0aaab70741a1af9b08793aaae82ea5a207402d5a15c563b859525e193a05f0a6510b"}' +``` +After the command successfully completed there should be an entry in the `PublicKeys` tab of the device. + +### Gathering Data +UPPs are usually used to anchor the hash of some kind of data. This data, in theory, can be everything. All examples below will use a simple string representing a JSON object like this for simplicity: +```json +{ + "ts": 1625163338, + "T": 11.2, + "H": 35.8, + "S": "OK" +} +``` +Translated to a hypothetical usecase this could be a measurement taken at `1625163338` (Unix-Timestamp), stating that the sensor measured `11.2 C` in temperature (`T`) and `35.8 %H` in humidity (`H`). The status - `S` - is `'OK'`. There is no script for this step, since it can easily done by hand. + +### Creating a UPP +After gathering some measurement data a UPP can be created. The won't contain the actual measurement data, but a hash of it. The script used to create UPPs in this example is `upp-creator.py`. +``` +$ python3 upp-creator.py --help + +usage: upp-creator.py [-h] [--version VERISON] [--type TYPE] [--ks KS] [--kspwd KSPWD] [--keyreg KEYREG] [--hash HASH] [--isjson ISJSON] [--output OUTPUT] [--nostdout nostdout] UUID DATA + +Create a uBirch Protocol Package (UPP) + +positional arguments: + UUID UUID to work with; e.g.: 56bd9b85-6c6e-4a24-bf71-f2ac2de10183 + DATA data to be packed into the UPP or hashed; e.g.: {"t": 23.4, "ts": 1624624140} + +optional arguments: + -h, --help show this help message and exit + --version VERISON, -v VERISON + version of the UPP; 0x21 (unsigned; NOT IMPLEMENTED), 0x22 (signed) or 0x23 (chained) (default: 0x23) + --type TYPE, -t TYPE type of the UPP (0 < type < 256); e.g.: 0x00 (unknown), 0x32 (msgpack), 0x53 (generic), ... (default and recommended: 0x00) + --ks KS, -k KS keystore file path; e.g.: test.jks (default: devices.jks) + --kspwd KSPWD, -p KSPWD + keystore password; e.g.: secret (default: keystore) + --keyreg KEYREG, -r KEYREG + generate a key registration UPP (data and --hash will be ignored); e.g.: true, false (default: False) + --hash HASH hash algorithm for hashing the data; sha256, sha512 or off (disable hashing), ... (default and recommended: sha512) + --isjson ISJSON, -j ISJSON + tells the script to treat the input data as json and serealize it (see EXAMPLES.md for more information); true or false (default: False) + --output OUTPUT, -o OUTPUT + file to write the generated UPP to (aside from standard output); e.g. upp.bin (default: upp.bin) + --nostdout nostdout, -n nostdout + do not output anything to stdout; can be combined with --output /dev/stdout; e.g.: true, false (default: False) + +Note that when using chained UPPs (--version 0x23), this tool will try to load/save signatures to UUID.sig, where UUID will be replaced with the actual UUID. Make sure that the UUID.sig file is in your current working directory if you try to continue a UPP chain +using this tool.Also beware that you will only be able to access the contents of a keystore when you use the same password you used when creating it. Otherwise all contents are lost. When --hash off is set, contents of the DATA argument will be copied into the +payload field of the UPP. Normally used for special messages (e.g. key registration). For more information on possible values for --type and --version see https://github.com/ubirch/ubirch-protocol. +``` +The script allows multiple modes of operation which can be set through different command line arguments. Some of those directly set fields in the resulting UPP. Please consult the [uBirch Protocol Readme](https://github.com/ubirch/ubirch-protocol#basic-message-format) for further information on those fields and their possible values. +- `--version/-v` This flag sets the version field of the UPP. The version field actually consists of two sub-fields. The higher four bits set the actual version (`1` or `2`) and the "mode". The higher four bits will be set to two (`0010`) set in almost all usecases. The mode can eather be a simple UPP without a signature, an UPP with a signature and an UPP with a signature + the signature of the previous UPP embedded into it. The one would be called a _Chained UPP_. Unsigned UPPs (`-v 0x21`) are not implemented. Signed UPPs have `-v 0x22` and chained ones `-v 0x23`. +- `--type/-t` This flag sets the type field of the UPP. It it used to indicate what the UPP contains/should be used for. It can be set to `0x00` in most cases. One of the cases in which a specific value is required are Key Registration Messages, as described in [Registering a Public Key](#registering-a-public-key). +- `--k/-k` The path to the keystore that contains the keypair for the device or should be used to store a newly generated keypair. If the keystore pointed to by this parameter doesn't exist, the script will simple create it. +- `--kspwd/-p` The password to decrypt/encrypt the keystore. You must remember this or you will lose access to the keystore and all its contents. +- `--keyreg/-k` Tells the script that the UPP that should be generated is a key registration UPP. The effect of that is that the script will ignore any custom input data and the `--hash` parameter (below). Instead, the UPP will contain the public key certificate. This parameter is a binary flag which can have two values: `true` or `false`. +- `--hash` Sets the hash algorithm to used to hash the input data. The produced hash will then be inserted into the payload field of the UPP. This parameter can have three values: `sha512`, `sha256` and `off`. When set to off, the input data will be directly put into the UPP without hashing it. This is only useful in some special cases like when manually assembling key registration messages (normally the `--keyreg` option should be used for that). +- `--isjson/-j` A binary flag that indicates that the input data is in JSON format. The script will serialize the JSON object before calculating the hash. This has the advantage one doesn't have to remember the order in which fields are listed in a JSON object to still be able to reconstruct the hash later on. Serializing the JSON is done like this: `json.dumps(self.data, separators=(',', ':'), sort_keys=True, ensure_ascii=False)` where `self.data` contains the JSON object which was loaded like this: `self.data = json.loads(self.dataStr)` where `dataStr` contains the input string which should represent a JSON object. This flag can have two values: `true` or `false`. +- `--output/-o` Tells the script where to write the generated UPP to. +- `--nostdout/-n` Binary flag to disable printing of any log messages to standard output. This can be used for piping a created UPP to another program. For this `--output /dev/stdout` would have to be set. +- `UUID` The UUID of the device as a hex-string, like `f5ded8a3-d462-41c4-a8dc-af3fd072a217`. +- `DATA` The data that is going to be hashed. If `--isjson true` is provided, it has to be a string representing a valid JSON object. **Note** that even though this argument will be ignored when `--keyreg true` is set, it must still exist. + +One common examples of using this script might look like this: +``` +$ python3 upp-creator.py --version 0x23 --isjson true --output upp.bin --hash sha256 f5ded8a3-d462-41c4-a8dc-af3fd072a217 '{ + "ts": 1625163338, + "T": 11.2, + "H": 35.8, + "S": "OK" +}' +2021-07-02 15:07:53,040 root init_keystore() INFO Public/Verifying key for "f5ded8a3-d462-41c4-a8dc-af3fd072a217" [base64]: "dsbKkw9HpsTvlLGgmiaYAM4M/ytFcySoF5UbfScffxg=" +2021-07-02 15:07:53,041 root prepare_payload() INFO Serialized data JSON: "{"H":35.8,"S":"OK","T":11.2,"ts":1625163338}" +2021-07-02 15:07:53,041 root prepare_payload() INFO UPP payload (sha256 hash of the data) [base64]: "dfQu7wBCL2aCuAqWLkyHEXCzTlKHdfMr7PMrxEcwY6A=" +2021-07-02 15:07:53,041 root create_upp() INFO Generating a chained signed UPP for UUID "f5ded8a3-d462-41c4-a8dc-af3fd072a217" +2021-07-02 15:07:53,041 root show_store_upp() INFO UPP [hex]: "9623c410f5ded8a3d46241c4a8dcaf3fd072a217c440cbe84f33c1d80a9a2a68f10c61c843567035d19179a703bb5e0aff4e920d9b8535acb171f1fd55271371d199fc985f33cf0b31f3c6ecfa7be684b561ac6d900f00c42075f42eef00422f6682b80a962e4c871170b34e528775f32becf32bc4473063a0c440ccc7e39d9a1acbf39d307d08d5b5f74218016e0b9e74d1efc7640c540c4cda1bf182b389a7ed9fd3fefb047ce6cf513dd1a047193ed0a13110f727fef4421102" +2021-07-02 15:07:53,041 root show_store_upp() INFO UPP written to "upp.bin" +``` +Keep in mind that when using chained UPPs (`--version 0x23`) you should anchor each UPP, or the signature chain will be broken. This won't cause any errors, but defeat the purpose of chaining UPPs. + +### Sending a UPP +After creating the UPP, it can be sent to the uBirch backend where it will be verified (the backend will use the registered public/verifying key to check the signature) and anchored into the blockchain. The `upp-sender.py` script can be used for that. +``` +$ python3 upp-sender.py --help +usage: upp-sender.py [-h] [--env ENV] [--input INPUT] [--output OUTPUT] UUID AUTH + +Send a uBirch Protocol Package (UPP) to uBirch Niomon + +positional arguments: + UUID UUID to work with; e.g.: 56bd9b85-6c6e-4a24-bf71-f2ac2de10183 + AUTH uBirch device authentication token + +optional arguments: + -h, --help show this help message and exit + --env ENV, -e ENV environment to operate in; dev, demo or prod (default: dev) + --input INPUT, -i INPUT + UPP input file path; e.g. upp.bin or /dev/stdin (default: upp.bin) + --output OUTPUT, -o OUTPUT + response UPP output file path (ignored for key registration UPPs); e.g. response_upp.bin (default: response_upp.bin) +``` +For this script there aren't that many parameters, since the task is rather easy and straight forward. +- `--env/-e` The env to operate on. This parameter decides wether the UPP will be sent to `niomon.prod.ubirch.com`, `niomon.demo.ubirch.com` or `niomon.dev.ubirch.com`. The value can either be `prod`, `demo` or `dev`. It must match the stage the UUID is registered on. +- `--input/-i` Specifies where to read the UPP to be sent from. This can be a normal file path or also `/dev/stdin` if, for example, the UPP will be piped to this script from another script (like `upp-creator.py`). In most cases the UPP will just be read from some file. +- `--output/-o` Normally the uBirch backend will respond to the UPP with another UPP. This parameter sets the location to write that response-UPP to. +- `UUID` The UUID of the device that generated the UPP as a hex-string. +- `AUTH` The auth token for the device on the specified stage as a hex-string. +Continuing from the example above (see [Creating a UPP](#creating-a-upp)), the send-command might look like this: +``` +$ python upp-sender.py --env demo --input upp.bin --output response_upp.bin f5ded8a3-d462-41c4-a8dc-af3fd072a217 xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx +2021-07-02 15:21:36,966 root read_upp() INFO Reading the input UPP for "f5ded8a3-d462-41c4-a8dc-af3fd072a217" from "upp.bin" +2021-07-02 15:21:37,722 root send_upp() INFO The UPP for "f5ded8a3-d462-41c4-a8dc-af3fd072a217" was accepted +2021-07-02 15:21:37,723 root send_upp() INFO 9623c4109d3c78ff22f34441a5d185c636d486ffc440ccc7e39d9a1acbf39d307d08d5b5f74218016e0b9e74d1efc7640c540c4cda1bf182b389a7ed9fd3fefb047ce6cf513dd1a047193ed0a13110f727fef442110200c42049950c5d778045a7b20c5e4db820c38100000000000000000000000000000000c440577f3679edbf96120066b9d1a794651817ec36fe6d1728841a7110ef0a2692c1e72827e8a48f98eefb42777b4fafd47c6bd7931e21c3c983c6f0c8a99144f90c +2021-07-02 15:21:37,723 root store_response_upp() INFO The response UPP has been written to "response_upp.bin" +``` + +### Verifying a UPP +To make sure that the response UPP actually was sent by the uBirch backend, its signature can be checked. The example script for that is `upp-verifier.py`. It knows the UUID and verifying/public key for each uBirch Niomon stage end checks if the signature of the response UPP is valid. + +``` +$ python3 upp-verifier.py --help +usage: upp-verifier.py [-h] [--verifying-key VK] [--verifying-key-uuid UUID] [--input INPUT] + +Check if a UPP is valid/properly signed + +optional arguments: + -h, --help show this help message and exit + --verifying-key VK, -k VK + key to be used for verification; any verifying key in hex like "b12a906051f102881bbb487ee8264aa05d8d0fcc51218f2a47f562ceb9b0d068" + --verifying-key-uuid UUID, -u UUID + the UUID for the key supplied via -k (only needed when -k is specified); e.g.: 6eac4d0b-16e6-4508-8c46-22e7451ea5a1 + --input INPUT, -i INPUT + UPP input file path; e.g. upp.bin or /dev/stdin (default: upp.bin) + +Note that when trying to verify a UPP sent by the uBirch backend (Niomon) a verifying key doesn't have to be provided via the -k option. Instead, this script will try to pick the correct stage key based on the UUID which is contained in the UPP, identifying the creator. If the UUID doesn't match any Niomon stage and no key was specified using -k, an error message will be printed. +``` +- `--verifying-key/-k` If not trying to verify a UPP coming from uBirch Niomon but from another source, the verifying key for that source needs to be provided. This parameter expects the key as a hex-string like `b12a906051f102881bbb487ee8264aa05d8d0fcc51218f2a47f562ceb9b0d068`. +- `--verifying-key-uuid/-u` The UUID for the verifying key from `--verifying-key`. This parameter will be ignored when `--verifying-key` is not set. Not setting this parameter when `--verifying-key` is set will cause an error. +- The file path to read the UPP from. + +``` +$ python3 upp-verifier.py --input response_upp.bin +2021-07-02 15:43:36,273 root read_upp() INFO Reading the input UPP from "response_upp.bin" +2021-07-02 15:43:36,274 ubirch.ubirch_ks _load_keys() WARNING creating new key store: -- temporary -- +2021-07-02 15:43:36,274 root get_upp_uuid() INFO UUID of the UPP creator: "07104235-1892-4020-9042-00003c94b60b" +2021-07-02 15:43:36,275 root verify_upp() INFO Signature verified - the UPP is valid! +``` + +### Examining a UPP +To examine the contents (Version Field, Type Field, UUID, Signature, Payload, Previous Signature) of an UPP, the `upp-unpacker.py` script can be used like this: +``` +$ python3 upp-unpacker.py response_upp.bin +- Version: 0x23 +- UUID: 9d3c78ff-22f3-4441-a5d1-85c636d486ff +- prev.Sign.: zMfjnZoay/OdMH0I1bX3QhgBbguedNHvx2QMVAxM2hvxgrOJp+2f0/77BHzmz1E90aBHGT7QoTEQ9yf+9EIRAg== + [hex]: ccc7e39d9a1acbf39d307d08d5b5f74218016e0b9e74d1efc7640c540c4cda1bf182b389a7ed9fd3fefb047ce6cf513dd1a047193ed0a13110f727fef4421102 (64 bytes) +- Type: 0x00 +- Payload: SZUMXXeARaeyDF5NuCDDgQAAAAAAAAAAAAAAAAAAAAA= + [hex]: 49950c5d778045a7b20c5e4db820c38100000000000000000000000000000000 (32 bytes) +- Signature: V382ee2/lhIAZrnRp5RlGBfsNv5tFyiEGnEQ7womksHnKCfopI+Y7vtCd3tPr9R8a9eTHiHDyYPG8MipkUT5DA== + [hex]: 577f3679edbf96120066b9d1a794651817ec36fe6d1728841a7110ef0a2692c1e72827e8a48f98eefb42777b4fafd47c6bd7931e21c3c983c6f0c8a99144f90c (64 bytes) +``` +_The UUID in this response UPP doesn't match the one from examples above because the UPP was sent from Niomon-Dev._ + +### Checking the anchoring status of an UPP +uBirch Niomon accepting the UPP doesn't mean that it is anchored yet. This process takes place in certain intervals, so one might have to wait a short while. The script for that task is `upp-anchoring-status.py`. +``` +$ python3 upp-anchoring-status.py -h +usage: upp-anchoring-status.py [-h] [--ishash ISHASH] [--env ENV] INPUT + +Requests the verification/anchoring of a UPP from the uBirch backend + +positional arguments: + INPUT input hash or upp path (depends on --ishash) + +optional arguments: + -h, --help show this help message and exit + --ishash ISHASH, -i ISHASH + sets if INPUT is being treated as a hash or upp path; true or false (default: False) + --env ENV, -e ENV the environment to operate in; dev, demo or prod (default: dev) + +When --ishash/-i is set to true, the input argument is treated as a base64 payload hash. Otherwise, it is expected to be some kind of path to read a UPP from. This can be a file path or also /dev/stdin if the UPP is piped to this program via standard input. +``` +- `--ishash/-i` A boolean specifying wether the input data is a payload hash or an UPP. The payload hash is what is actually used to look up anchoring information about the UPP. This script can either extract it from a given UPP or just use the hash directly if provided. If directly provided, it must be base64 encoded (see the last example of this sub-section). `true` or `false`. +- `--env/-e` The stage to check on. Should be the one the UPP was sent to. `prod`, `demo` or `dev`. +- `INPUT` The input UPP file path or payload hash, depending on `--ishash`. + +One example might be: +``` +python3 upp-anchoring-status.py --env demo upp.bin +2021-07-02 16:01:46,761 root read_upp() INFO Reading the input UPP from "upp.bin" +2021-07-02 16:01:46,761 root get_hash_from_upp() INFO Extracted UPP hash: "ToBgV89kXaWU0YHblha7qUXn0gohzpKoIS515cmSl4Y=" +2021-07-02 16:01:46,761 root get_status() INFO Requesting anchoring information from: "https://verify.demo.ubirch.com/api/upp/verify/anchor" +2021-07-02 16:01:46,950 root get_status() INFO The UPP is known to the uBirch backend! (code: 200) +Curr. UPP: "liPEEPXe2KPUYkHEqNyvP9ByohfEQMzH452aGsvznTB9CNW190IYAW4LnnTR78dkDFQMTNob8YKziaftn9P++wR85s9RPdGgRxk+0KExEPcn/vRCEQIAxCBOgGBXz2RdpZTRgduWFrupRefSCiHOkqghLnXlyZKXhsRAZw4gMp5Wlq5Sij9UQrjMfhxdmeoY6IsVS7Aq8MLZyUT5CvTeEK/4kt4N55tE8pYVN7G+FxEYwvYfwDLZPqViBw==" +Prev. UPP: "liPEEPXe2KPUYkHEqNyvP9ByohfEQMvoTzPB2AqaKmjxDGHIQ1ZwNdGReacDu14K/06SDZuFNayxcfH9VScTcdGZ/JhfM88LMfPG7Pp75oS1YaxtkA8AxCB19C7vAEIvZoK4CpYuTIcRcLNOUod18yvs8yvERzBjoMRAzMfjnZoay/OdMH0I1bX3QhgBbguedNHvx2QMVAxM2hvxgrOJp+2f0/77BHzmz1E90aBHGT7QoTEQ9yf+9EIRAg==" +2021-07-02 16:01:46,950 root get_status() INFO The UPP has NOT been anchored into any blockchains yet! Please retry later +``` +Here it is visible that the backend knows the UPP and that it is valid, but it hasn't been anchored yet. Additionally the output shows that The backend knows the previous UPP, indicating that the UPP is a chained UPP and not the first UPP in the chain. When using unchained UPPs the line will change to: `Prev. UPP: "None"`. After waiting some time and running the script again with the same parameters: +``` +$ python3 upp-anchoring-status.py --env demo upp.bin +2021-07-02 16:09:34,521 root read_upp() INFO Reading the input UPP from "upp.bin" +2021-07-02 16:09:34,521 root get_hash_from_upp() INFO Extracted UPP hash: "ToBgV89kXaWU0YHblha7qUXn0gohzpKoIS515cmSl4Y=" +2021-07-02 16:09:34,521 root get_status() INFO Requesting anchoring information from: "https://verify.demo.ubirch.com/api/upp/verify/anchor" +2021-07-02 16:09:34,727 root get_status() INFO The UPP is known to the uBirch backend! (code: 200) +Curr. UPP: "liPEEPXe2KPUYkHEqNyvP9ByohfEQMzH452aGsvznTB9CNW190IYAW4LnnTR78dkDFQMTNob8YKziaftn9P++wR85s9RPdGgRxk+0KExEPcn/vRCEQIAxCBOgGBXz2RdpZTRgduWFrupRefSCiHOkqghLnXlyZKXhsRAZw4gMp5Wlq5Sij9UQrjMfhxdmeoY6IsVS7Aq8MLZyUT5CvTeEK/4kt4N55tE8pYVN7G+FxEYwvYfwDLZPqViBw==" +Prev. UPP: "liPEEPXe2KPUYkHEqNyvP9ByohfEQMvoTzPB2AqaKmjxDGHIQ1ZwNdGReacDu14K/06SDZuFNayxcfH9VScTcdGZ/JhfM88LMfPG7Pp75oS1YaxtkA8AxCB19C7vAEIvZoK4CpYuTIcRcLNOUod18yvs8yvERzBjoMRAzMfjnZoay/OdMH0I1bX3QhgBbguedNHvx2QMVAxM2hvxgrOJp+2f0/77BHzmz1E90aBHGT7QoTEQ9yf+9EIRAg==" +2021-07-02 16:09:34,727 root get_status() INFO The UPP has been fully anchored! +[{'label': 'PUBLIC_CHAIN', 'properties': {'timestamp': '2021-07-02T14:03:22.093Z', 'hash': '0xd1fe1f27a315089e5522eb7c8124962774b335c24d1ed7281091b447a8d3bca2', 'public_chain': 'ETHEREUM_TESTNET_RINKEBY_TESTNET_NETWORK', 'prev_hash': '06700cdb7b196292eceac71520fad2e46890e2d8f74510f1bc4296c6a0e16a631cff533989c9b83363f72051105b8f0bfaf59706a5258d8d275abc93d67d5b4d'}}] +``` +The UPP has been anchored. **Note** that when running on `prod` the output regarding the anchoring status will be significantly longer: +``` +$ python3 upp-anchoring-status.py --env prod --ishash true "dfQu7wBCL2aCuAqWLkyHEXCzTlKHdfMr7PMrxEcwY6A=" +2021-07-02 16:13:47,509 root get_hash_from_input() INFO Extracted hash from input: "dfQu7wBCL2aCuAqWLkyHEXCzTlKHdfMr7PMrxEcwY6A=" +2021-07-02 16:13:47,509 root get_status() INFO Requesting anchoring information from: "https://verify.prod.ubirch.com/api/upp/verify/anchor" +2021-07-02 16:13:47,631 root get_status() INFO The UPP is known to the uBirch backend! (code: 200) +Curr. UPP: "liPEEAi4JX1SUEaGhzAnqUf6pn3EQA1F2OFz+pQfCw7yxznodtsSf5ubCXPjHOFNWPexyiNFVHouv4m2mcDHzu8icxoD1U8pXFtXscsFrYy3+oCfPgoAxCB19C7vAEIvZoK4CpYuTIcRcLNOUod18yvs8yvERzBjoMRADipQZBD9bOdYezTD49h8MuAGBspO+PCkHFAMor8H3OZGRKXs0i4Fa4ICG0VV8B6PtVzoKz5vf8m6pWGFAb/wBQ==" +Prev. UPP: "liPEEAi4JX1SUEaGhzAnqUf6pn3EQOeEqF4lo+xT9RF2ygDx9+anv14fykUolJ9gmKuTTjmzc05qXnjhs+sdQtwN7To21DBrCeDDmK4MYFixx/umBAoAxEAns4128paQWJKi9D+W9UzQqpWtOtg1474cDWvxHTMSGnP87f6IllmqA+DHJ3Xe6LZ47hlbjUsLJtAiYtS1u1hBxEANRdjhc/qUHwsO8sc56HbbEn+bmwlz4xzhTVj3scojRVR6Lr+JtpnAx87vInMaA9VPKVxbV7HLBa2Mt/qAnz4K" +2021-07-02 16:13:47,631 root get_status() INFO The UPP has been fully anchored! +[{'label': 'PUBLIC_CHAIN', 'properties': {'timestamp': '2021-07-02T13:46:45.949Z', 'hash': '0xc32ccbe70ed727a1842f998d56d9928a9a30e201aef91bb08d9ac7faf931dac6', 'public_chain': 'ETHEREUM-CLASSIC_MAINNET_ETHERERUM_CLASSIC_MAINNET_NETWORK', 'prev_hash': '4d79e0d331b1fe057b3c9ee7cb595c371ec0ea764147029a862b3cffce808ae049ec40a6e5cddbd6ee90e4d36955cd2e08ab1f4ef1ccc8c013710617bd689cfe'}}, {'label': 'PUBLIC_CHAIN', 'properties': {'timestamp': '2021-07-02T13:58:24.677Z', 'hash': '0x65438aab8904c467ceb5b22e6b1a4198eeb056647a8b7a9ece0d03739a79a0bb', 'public_chain': 'GOV-DIGITAL_MAINNET_GOV_DIGITAL_MAINNET_NETWORK', 'prev_hash': 'e98a502253ec3aebd9af29cd368e3db899e5c04d6e8214be17856055b9301b63ce2ddb34038c4d3366ab9b4be28f755041f3a089d36c3516d84b0a95741289e3'}}] +``` + +### Verifying a measurement +Being able to check if a UPP is anchored and valid is nice, but in a real usecase it might not be that useful. That's because usually the goal is the check if some data is valid. Verifying a UPP doesn't really help in that case, since it is not possible to reverse the hash contained in the UPP back to the original data. That's why one shouldn't bother to store sent UPPs, but rather the measurements these UPPs are based on. Those can then be hashed, and the hash can be looked up. The `data-verifier.py` script does exactly that. It has similar behaviour to the `upp-anchoring-status.py` script, see [Checking the anchoring status of an UPP](#checking-the-anchoring-status-of-an-upp). +``` +$ python data-verifier.py -h +usage: data-verifier.py [-h] [--ispath ISHASH] [--env ENV] [--isjson ISJSON] [--hash HASH] INPUT + +Check if the hash of given input data is known to the uBirch backend (verify it) + +positional arguments: + INPUT input data or data file path (depends on --ispath) + +optional arguments: + -h, --help show this help message and exit + --ispath ISPATH, -i ISPATH + sets if INPUT is being treated as data or data file path; true or false (default: False) + --env ENV, -e ENV the environment to operate in; dev, demo or prod (default: dev) + --isjson ISJSON, -j ISJSON + tells the script to treat the input data as json and serealize it (see EXAMPLES.md for more information); true or false (default: True) + --hash HASH, -a HASH sets the hash algorithm to use; sha256, sha512 or OFF to treat the input data as hash (default: sha256) + +When --ispath/-i is set to true, the input data is treated as a file path to read the actual input data from. When setting --hash/-a to off, the input argument is expected to be a valid base64 encoded hash. +``` +- `--ispath/-i` Specifies wether the input is to be treated as a data-file path or direct input data. `true` or `false`. +- `--env-e` The stage to check on. Should be the one the UPP corresponding to the data was sent to. `prod`, `demo` or `dev`. +- `--isjson/-j` A binary flag that indicates that the input data is in JSON format. The script will serialize the JSON object before calculating the hash. This has the advantage one doesn't have to remember the order in which fields are listed in a JSON object to still be able to reconstruct the hash later on. Serializing the JSON is done like this: `json.dumps(self.data, separators=(',', ':'), sort_keys=True, ensure_ascii=False)` where `self.data` contains the JSON object which was loaded like this: `self.data = json.loads(self.dataStr)` where `dataStr` contains the input string which should represent a JSON object. This flag can have two values: `true` or `false`. It should only be set to `true` if the data represents a JSON object and if it also was serialized when creating the UPP. +- `--hash/-a` Sets the hashing algorithm to use. `sha256`, `sha512` or `off`. It should match the algorithm used when creating the corresponding UPP. Setting it to `off` means that the input data actually already is the hash of the data. In this case this script will simply look up the hash. + +Example: +``` +python data-verifier.py --env demo --isjson true --hash sha256 '{ + "ts": 1625163338, + "T": 11.2, + "H": 35.8, + "S": "OK" +}' +2021-07-02 16:21:41,178 root serialize_json() INFO Serialized JSON: "{"H":35.8,"S":"OK","T":11.2,"ts":1625163338}" +2021-07-02 16:21:41,178 root get_hash_from_data() INFO Calculated hash: "dfQu7wBCL2aCuAqWLkyHEXCzTlKHdfMr7PMrxEcwY6A=" +2021-07-02 16:21:41,178 root get_status() INFO Requesting anchoring information from: "https://verify.demo.ubirch.com/api/upp/verify/anchor" +2021-07-02 16:21:41,599 root get_status() INFO The hash is known to the uBirch backend! (code: 200) +Curr. UPP: "liPEEPXe2KPUYkHEqNyvP9ByohfEQMvoTzPB2AqaKmjxDGHIQ1ZwNdGReacDu14K/06SDZuFNayxcfH9VScTcdGZ/JhfM88LMfPG7Pp75oS1YaxtkA8AxCB19C7vAEIvZoK4CpYuTIcRcLNOUod18yvs8yvERzBjoMRAzMfjnZoay/OdMH0I1bX3QhgBbguedNHvx2QMVAxM2hvxgrOJp+2f0/77BHzmz1E90aBHGT7QoTEQ9yf+9EIRAg==" +Prev. UPP: "None" +2021-07-02 16:21:41,600 root get_status() INFO The corresponding UPP has been fully anchored! +[{'label': 'PUBLIC_CHAIN', 'properties': {'timestamp': '2021-07-02T13:23:30.076Z', 'hash': '0x6e5956b4ac53bcaf58664e189673d4f8c7043488cf05009cc96868b146220604', 'public_chain': 'ETHEREUM-CLASSIC_TESTNET_ETHERERUM_CLASSIC_KOTTI_TESTNET_NETWORK', 'prev_hash': '644b41eee9043de5bda4b58bda1136fa0229712953678fe26486651338109b7a7135211f2b2cb646ab25d3b25198549e3c662c4791d60d343f08349b51ccc92b'}}] +``` +Just like with `upp-anchoring-status.py`, it might take a short while after sending the corresponding UPP to the backend before it will be anchored. + +## Sending data to the Simple Data Service +The `data-sender.py` example-script allows sending of data to the simple data service. This is only used for demo purposes. +``` +$ python3 data-sender.py --help +usage: data-sender.py [-h] [--env ENV] UUID AUTH INPUT + +Send some data to the uBirch Simple Data Service + +positional arguments: + UUID UUID to work with; e.g.: 56bd9b85-6c6e-4a24-bf71-f2ac2de10183 + AUTH uBirch device authentication token + INPUT data to be sent to the simple data service + +optional arguments: + -h, --help show this help message and exit + --env ENV, -e ENV environment to operate in; dev, demo or prod (default: dev) + +Note that the input data should follow this pattern: {"timestamp": TIMESTAMP, "uuid": "UUID", "msg_type": 0, "data": DATA, "hash": "UPP_HASH"}. For more information take a look at the EXAMPLES.md file. +``` + +## Example uBirch client implementation +`example-client.py` implements a full example uBirch client. From 4636086dcc99a92e3f9b9002ee797d6af2536179 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Fri, 2 Jul 2021 17:08:11 +0200 Subject: [PATCH 18/40] remove old comment --- examples/data-verifier.py | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/examples/data-verifier.py b/examples/data-verifier.py index 87f8c61..ff1c77f 100644 --- a/examples/data-verifier.py +++ b/examples/data-verifier.py @@ -1,32 +1,3 @@ - -# import hashlib -# import json - -# import requests - -# VERIFICATION_SERVICE = "https://verify.prod.ubirch.com/api/upp/verify/anchor" - -# with open("data_to_verify.json") as f: -# message = json.load(f) - -# # create a compact rendering of the message to ensure determinism when creating the hash -# serialized = json.dumps(message, separators=(',', ':'), sort_keys=True, ensure_ascii=False).encode() -# print("rendered data:\n\t{}\n".format(serialized.decode())) - -# # calculate hash of message -# data_hash = hashlib.sha256(serialized).digest() -# print("hash [base64]:\n\t{}\n".format(base64.b64encode(data_hash).decode())) - -# # verify existence of the hash in the UBIRCH backend -# r = requests.post(url=VERIFICATION_SERVICE, -# headers={'Accept': 'application/json', 'Content-Type': 'text/plain'}, -# data=base64.b64encode(data_hash).decode().rstrip('\n')) - -# if 200 <= r.status_code < 300: -# print("verification successful:\n\t{}\n".format(r.content.decode())) -# else: -# print("verification FAIL: ({})\n\tdata hash could not be verified\n".format(r.status_code)) - import sys import argparse import base64 From c47c537fdaffd711b2179e59602cfd1258a0b6cd Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Fri, 2 Jul 2021 17:09:21 +0200 Subject: [PATCH 19/40] fix typo --- examples/upp-anchoring-status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/upp-anchoring-status.py b/examples/upp-anchoring-status.py index 1c39f81..6234dbf 100644 --- a/examples/upp-anchoring-status.py +++ b/examples/upp-anchoring-status.py @@ -189,7 +189,7 @@ def run(self) -> int: # get the anchoring status if self.get_status() != True: - logger.error("Errors occured while requesting the anchring status - exiting!\n") + logger.error("Errors occured while requesting the anchoring status - exiting!\n") self.argparser.print_usage() From 2bec84bfe14474b73d2bbb5eabd4651d1735c852 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Fri, 2 Jul 2021 22:07:13 +0200 Subject: [PATCH 20/40] add documentation for keystore dumper script --- examples/EXAMPLES.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/examples/EXAMPLES.md b/examples/EXAMPLES.md index 38d2f71..b20da32 100644 --- a/examples/EXAMPLES.md +++ b/examples/EXAMPLES.md @@ -47,6 +47,25 @@ To create, or more precisely, to _sign_ a UPP, a device will need a keypair. Thi **NOTE** that losing access to the signing key, especially if it is already registered at the uBirch backend, will take away the ability to create and send any new UPPs from that device/UUID, since there is no way of creating a valid signature that would be accepted by the backend. +A keystore can be read out with the `keystore-dumper.py` script. + +``` +$ python keystore-dumper.py --help +usage: keystore-dumper.py [-h] [--show-sk SHOW_SIGNING_KET] KEYSTORE KEYSTORE_PASS + +Dump the contents of a keystore (.jks) + +positional arguments: + KEYSTORE keystore file path; e.g.: test.jks + KEYSTORE_PASS keystore password; e.g.: secret + +optional arguments: + -h, --help show this help message and exit + --show-sk SHOW_SIGNING_KET, -s SHOW_SIGNING_KET + enables/disables showing of signing keys; e.g.: true, false (default: False) +``` +By default, only UUIDs und public keys (verifying keys) will be displayed. Displaying of private keys (signing keys) can be enabled by passing `-s true`. + ### Registering a public key To enable the uBirch backend to verify a UPP, it needs to know the corresponding verifying key. Therefore, the device needs to send this key to the backend before starting to send UPPs supposed to be verified and anchored. Registering a verifying key is also done by sending a special kind of UPP containing this key. This can be done by using two scripts: ``` From 887a82a70f7d04b18eb185b9176ebfed9d582ebd Mon Sep 17 00:00:00 2001 From: wowa Date: Tue, 6 Jul 2021 00:21:28 +0200 Subject: [PATCH 21/40] Reviewed and corrected EXAMPLES.md --- examples/EXAMPLES.md | 78 +++++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 34 deletions(-) diff --git a/examples/EXAMPLES.md b/examples/EXAMPLES.md index b20da32..dcf0808 100644 --- a/examples/EXAMPLES.md +++ b/examples/EXAMPLES.md @@ -13,13 +13,13 @@ This file documents how to use the examples provided alongside the [uBirch-Proto - [Sending a UPP](#sending-a-upp) - [Verifying a UPP](#verifying-a-upp) - [Examining a UPP](#examining-a-upp) - - [Checking the anchoring status of an UPP](#checking-the-anchoring-status-of-an-upp) - - [Verifying a measurement](#verifying-a-measurement) + - [Checking the anchoring status of a UPP](#checking-the-anchoring-status-of-an-upp) + - [Verifying data](#verifying-data) - [Sending data to the Simple Data Service](#sending-data-to-the-simple-data-service) - [Example uBirch client implementation](#example-ubirch-client-implementation) ## From measurement to blockchain-anchored UPP -The process needed to get a UPP to be anchored in the blockchain can be cut down into multiple steps. For each of those steps there is an example in this directory, demonstrating how one could handle them. There also are examples showing a full example-client implementation. +The process needed to get a UPP to be anchored in the blockchain can be cut down into multiple steps. For each of those steps there is an example in this directory, demonstrating how to handle them. There are also examples showing a full example-client implementation. 0. [Setup](#setup) 1. [Gathering Data](#gathering-data) @@ -31,8 +31,8 @@ Before anything, you will need to do/get a couple of things: - https://console.prod.ubirch.com for the `prod` stage - https://console.demo.ubirch.com for the `demo` stage - https://console.dev.ubirch.com for the `dev` stage -- Get a UUID (can be generated randomly or on the basis of certain device properties like MAC-Addresses) -- Create a "Thing" at the uBirch-Console; remember/note down the used UUID and the generated Auth-Token +- Get a UUID (can be generated randomly, for example [here](https://www.uuidgenerator.net/), or on the basis of certain device properties like MAC-Addresses) +- Create a "Thing" at the uBirch-Console; remember/note down the used UUID and the generated Auth-Token. For details on how to create a Thing, check the[Ubirch console documentation](https://developer.ubirch.com/console.html). You should now have the following information at hand: - The stage you want to work on (later referred to as `env`) @@ -43,7 +43,7 @@ The values used below are `f5ded8a3-d462-41c4-a8dc-af3fd072a217` for the UUID, ` `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` for the auth token. ### Generating and managing a keypair -To create, or more precisely, to _sign_ a UPP, a device will need a keypair. This keypair consist of a private key (_signing key_) and public key (_verifying key_). The signing key is used to sign UPPs and the verifying key can be used by the uBirch backend to check if the signature is valid and belongs to the correct sender/signer. So logically, it doesn't matter who knows to verifying key, but the signing key must be kept secret all the time. In a real usecase, a device might store it in a TPM ([Trusted platform module](https://en.wikipedia.org/wiki/Trusted_Platform_Module)) or use other counter measures against an attacker reading the key from the device. For this demo, keypairs will be stored in a [JKS Keystore](https://en.wikipedia.org/wiki/Java_KeyStore) using the [`pyjks`](https://pypi.org/project/pyjks/) library. For that you will have to chose and remember a file path for that keystore and a password used to encrypt it. The process of actually generating the keypair is exlpained [bellow](#registering-a-public-key). +To create, or more precisely, to _sign_ a UPP, a device will need a keypair. This keypair consist of a private key (_signing key_) and public key (_verifying key_). The signing key is used to sign UPPs and the verifying key can be used by the uBirch backend to check, if the signature is valid and belongs to the correct sender/signer. So, logically it doesn't matter who knows to verifying key, but the signing key must be kept secret all the time. In a real use case a device might store it in a TPM ([Trusted platform module](https://en.wikipedia.org/wiki/Trusted_Platform_Module)) or use other counter measures against attackers reading the key from the device. For this demo, keypairs will be stored in a [JKS Keystore](https://en.wikipedia.org/wiki/Java_KeyStore) using the [`pyjks`](https://pypi.org/project/pyjks/) library. Therefore, you will have to choose and remember a file path for that keystore and a password used to encrypt it. The process of actually generating the keypair is explained [below](#registering-a-public-key). **NOTE** that losing access to the signing key, especially if it is already registered at the uBirch backend, will take away the ability to create and send any new UPPs from that device/UUID, since there is no way of creating a valid signature that would be accepted by the backend. @@ -67,7 +67,7 @@ optional arguments: By default, only UUIDs und public keys (verifying keys) will be displayed. Displaying of private keys (signing keys) can be enabled by passing `-s true`. ### Registering a public key -To enable the uBirch backend to verify a UPP, it needs to know the corresponding verifying key. Therefore, the device needs to send this key to the backend before starting to send UPPs supposed to be verified and anchored. Registering a verifying key is also done by sending a special kind of UPP containing this key. This can be done by using two scripts: +To enable the uBirch backend to verify a UPP, it needs to know the corresponding verifying key. Therefore, the device needs to send this key to the backend, before starting to send UPPs, which are supposed to be verified and anchored. Registering a verifying key is done by sending a special kind of UPP containing this key. This can be done by using two scripts: ``` upp-creator.py upp-sender.py @@ -106,10 +106,19 @@ UPPs are usually used to anchor the hash of some kind of data. This data, in the "S": "OK" } ``` -Translated to a hypothetical usecase this could be a measurement taken at `1625163338` (Unix-Timestamp), stating that the sensor measured `11.2 C` in temperature (`T`) and `35.8 %H` in humidity (`H`). The status - `S` - is `'OK'`. There is no script for this step, since it can easily done by hand. +Translated to a hypothetical use case this could be a measurement taken at `1625163338` (Unix-Timestamp), stating that the sensor measured `11.2 C` in temperature (`T`) and `35.8 %H` in humidity (`H`). The status - `S` - is `'OK'`. There is no script for this step, since it can easily be done by hand. + +**Note: _If you use a JSON format for your data, the data has to be alphabetically sorted, all whitespace removed and +serialized into a simple string, before the hash of the data is generated. This ensures, that you can always regenerate +the same hash for your data. This is already implemented in the examples, like the following line of code shows:_** +```python +serialized = json.dumps(message, separators=(',', ':'), sort_keys=True, ensure_ascii=False).encode() +``` +This will create a string from the above examplary JSON object: +`{"H":35.8,"S":"OK","T":11.2,"ts":1625163338}` ### Creating a UPP -After gathering some measurement data a UPP can be created. The won't contain the actual measurement data, but a hash of it. The script used to create UPPs in this example is `upp-creator.py`. +After gathering some measurement data a UPP can be created. The UPP won't contain the actual measurement data, but a hash of it. The example script to create UPPs in is `upp-creator.py`. ``` $ python3 upp-creator.py --help @@ -139,18 +148,18 @@ optional arguments: --nostdout nostdout, -n nostdout do not output anything to stdout; can be combined with --output /dev/stdout; e.g.: true, false (default: False) -Note that when using chained UPPs (--version 0x23), this tool will try to load/save signatures to UUID.sig, where UUID will be replaced with the actual UUID. Make sure that the UUID.sig file is in your current working directory if you try to continue a UPP chain +Note that, when using chained UPPs (--version 0x23), this tool will try to load/save signatures from/to .sig, where UUID will be replaced with the actual UUID. Make sure that the UUID.sig file is in your current working directory if you try to continue a UPP chain using this tool.Also beware that you will only be able to access the contents of a keystore when you use the same password you used when creating it. Otherwise all contents are lost. When --hash off is set, contents of the DATA argument will be copied into the payload field of the UPP. Normally used for special messages (e.g. key registration). For more information on possible values for --type and --version see https://github.com/ubirch/ubirch-protocol. ``` -The script allows multiple modes of operation which can be set through different command line arguments. Some of those directly set fields in the resulting UPP. Please consult the [uBirch Protocol Readme](https://github.com/ubirch/ubirch-protocol#basic-message-format) for further information on those fields and their possible values. -- `--version/-v` This flag sets the version field of the UPP. The version field actually consists of two sub-fields. The higher four bits set the actual version (`1` or `2`) and the "mode". The higher four bits will be set to two (`0010`) set in almost all usecases. The mode can eather be a simple UPP without a signature, an UPP with a signature and an UPP with a signature + the signature of the previous UPP embedded into it. The one would be called a _Chained UPP_. Unsigned UPPs (`-v 0x21`) are not implemented. Signed UPPs have `-v 0x22` and chained ones `-v 0x23`. -- `--type/-t` This flag sets the type field of the UPP. It it used to indicate what the UPP contains/should be used for. It can be set to `0x00` in most cases. One of the cases in which a specific value is required are Key Registration Messages, as described in [Registering a Public Key](#registering-a-public-key). -- `--k/-k` The path to the keystore that contains the keypair for the device or should be used to store a newly generated keypair. If the keystore pointed to by this parameter doesn't exist, the script will simple create it. -- `--kspwd/-p` The password to decrypt/encrypt the keystore. You must remember this or you will lose access to the keystore and all its contents. +The script allows multiple modes of operation, which can be set through different command line arguments. Some of those directly set fields in the resulting UPP. Please consult the [uBirch Protocol Readme](https://github.com/ubirch/ubirch-protocol#basic-message-format) for further information on those fields and their possible values. +- `--version/-v` This flag sets the version field of the UPP. The version field actually consists of two sub-fields. The higher four bits set the actual version (`1` or `2`) and the "mode". The higher four bits will be set to two(`0010`) in almost all use cases. The mode can either be a simple UPP without a signature, a UPP with a signature and a UPP with a signature + the signature of the previous UPP embedded into it. The latter would be called a_Chained UPP_. Unsigned UPPs (`-v 0x21`) are not implemented. Signed UPPs have `-v 0x22` and chained ones `-v 0x23`. +- `--type/-t` This flag sets the type field of the UPP. It is used to indicate what the UPP contains/should be used for. It can be set to `0x00` in most cases. One of the cases where a specific value is required, is a Key Registration Messages, as described in [Registering a Public Key](#registering-a-public-key). +- `--k/-k` The path to the keystore that contains the keypair for the device or should be used to store a newly generated keypair. If the keystore, pointed to by this parameter, doesn't exist, the script will simply create it. +- `--kspwd/-p` The password to decrypt/encrypt the keystore. You must remember this, or you will lose access to the keystore and all its contents. - `--keyreg/-k` Tells the script that the UPP that should be generated is a key registration UPP. The effect of that is that the script will ignore any custom input data and the `--hash` parameter (below). Instead, the UPP will contain the public key certificate. This parameter is a binary flag which can have two values: `true` or `false`. -- `--hash` Sets the hash algorithm to used to hash the input data. The produced hash will then be inserted into the payload field of the UPP. This parameter can have three values: `sha512`, `sha256` and `off`. When set to off, the input data will be directly put into the UPP without hashing it. This is only useful in some special cases like when manually assembling key registration messages (normally the `--keyreg` option should be used for that). -- `--isjson/-j` A binary flag that indicates that the input data is in JSON format. The script will serialize the JSON object before calculating the hash. This has the advantage one doesn't have to remember the order in which fields are listed in a JSON object to still be able to reconstruct the hash later on. Serializing the JSON is done like this: `json.dumps(self.data, separators=(',', ':'), sort_keys=True, ensure_ascii=False)` where `self.data` contains the JSON object which was loaded like this: `self.data = json.loads(self.dataStr)` where `dataStr` contains the input string which should represent a JSON object. This flag can have two values: `true` or `false`. +- `--hash` Sets the hash algorithm to be used to generate the hash of the input data. The produced hash will then be inserted into the payload field of the UPP. This parameter can have three values: `sha512`, `sha256` and `off`. When set to off, the input data will be directly put into the UPP without hashing it. This is only useful in some special cases like when manually assembling key registration messages (normally the `--keyreg` option should be used for that). +- `--isjson/-j` A binary flag that indicates that the input data is in JSON format. The script will serialize the JSON object before calculating the hash. This has the advantage one doesn't have to remember the order in which fields are listed in a JSON object to still be able to reconstruct the hash later on. Serializing the JSON is done like this: `json.dumps(self.data, separators=(',', ':'), sort_keys=True, ensure_ascii=False)` where `self.data` contains the JSON object which was loaded like this: `self.data = json.loads(self.dataStr)` where `dataStr` contains the input string, which should represent a JSON object. This flag can have two values: `true` or `false`. - `--output/-o` Tells the script where to write the generated UPP to. - `--nostdout/-n` Binary flag to disable printing of any log messages to standard output. This can be used for piping a created UPP to another program. For this `--output /dev/stdout` would have to be set. - `UUID` The UUID of the device as a hex-string, like `f5ded8a3-d462-41c4-a8dc-af3fd072a217`. @@ -171,10 +180,10 @@ $ python3 upp-creator.py --version 0x23 --isjson true --output upp.bin --hash sh 2021-07-02 15:07:53,041 root show_store_upp() INFO UPP [hex]: "9623c410f5ded8a3d46241c4a8dcaf3fd072a217c440cbe84f33c1d80a9a2a68f10c61c843567035d19179a703bb5e0aff4e920d9b8535acb171f1fd55271371d199fc985f33cf0b31f3c6ecfa7be684b561ac6d900f00c42075f42eef00422f6682b80a962e4c871170b34e528775f32becf32bc4473063a0c440ccc7e39d9a1acbf39d307d08d5b5f74218016e0b9e74d1efc7640c540c4cda1bf182b389a7ed9fd3fefb047ce6cf513dd1a047193ed0a13110f727fef4421102" 2021-07-02 15:07:53,041 root show_store_upp() INFO UPP written to "upp.bin" ``` -Keep in mind that when using chained UPPs (`--version 0x23`) you should anchor each UPP, or the signature chain will be broken. This won't cause any errors, but defeat the purpose of chaining UPPs. +Keep in mind that if you use chained UPPs (`--version 0x23`) you should anchor each UPP, or the signature chain will be broken. This won't cause any errors, but the advantage of chaining UPPs and thereby knowing the correct order of them, will get lost. ### Sending a UPP -After creating the UPP, it can be sent to the uBirch backend where it will be verified (the backend will use the registered public/verifying key to check the signature) and anchored into the blockchain. The `upp-sender.py` script can be used for that. +After creating the UPP, it can be sent to the uBirch backend where it will be verified and anchored in the blockchain. The ubirch backend will use the registered public/verifying key to check the signature. The `upp-sender.py` script can be used for that. ``` $ python3 upp-sender.py --help usage: upp-sender.py [-h] [--env ENV] [--input INPUT] [--output OUTPUT] UUID AUTH @@ -193,9 +202,9 @@ optional arguments: --output OUTPUT, -o OUTPUT response UPP output file path (ignored for key registration UPPs); e.g. response_upp.bin (default: response_upp.bin) ``` -For this script there aren't that many parameters, since the task is rather easy and straight forward. -- `--env/-e` The env to operate on. This parameter decides wether the UPP will be sent to `niomon.prod.ubirch.com`, `niomon.demo.ubirch.com` or `niomon.dev.ubirch.com`. The value can either be `prod`, `demo` or `dev`. It must match the stage the UUID is registered on. -- `--input/-i` Specifies where to read the UPP to be sent from. This can be a normal file path or also `/dev/stdin` if, for example, the UPP will be piped to this script from another script (like `upp-creator.py`). In most cases the UPP will just be read from some file. +For this script the parameters are: +- `--env/-e` The env to operate on. This parameter decides wether the UPP will be sent to `niomon.prod.ubirch.com`, `niomon.demo.ubirch.com` or `niomon.dev.ubirch.com`. The value can either be `prod`, `demo` or `dev`. It must match the stage, the UUID is registered on. +- `--input/-i` Specifies where to read the UPP to be sent from. This can be a normal file path or also `/dev/stdin`, if for example the UPP will be piped to this script from another script (like `upp-creator.py`). In most cases the UPP will just be read from some file. - `--output/-o` Normally the uBirch backend will respond to the UPP with another UPP. This parameter sets the location to write that response-UPP to. - `UUID` The UUID of the device that generated the UPP as a hex-string. - `AUTH` The auth token for the device on the specified stage as a hex-string. @@ -209,7 +218,7 @@ $ python upp-sender.py --env demo --input upp.bin --output response_upp.bin f5de ``` ### Verifying a UPP -To make sure that the response UPP actually was sent by the uBirch backend, its signature can be checked. The example script for that is `upp-verifier.py`. It knows the UUID and verifying/public key for each uBirch Niomon stage end checks if the signature of the response UPP is valid. +To make sure, that the response UPP actually was sent by the uBirch backend, its signature can be checked. The example script for that is `upp-verifier.py`. It knows the UUID and verifying/public key for each uBirch Niomon stage end checks, if the signature of the response UPP is valid. ``` $ python3 upp-verifier.py --help @@ -226,11 +235,11 @@ optional arguments: --input INPUT, -i INPUT UPP input file path; e.g. upp.bin or /dev/stdin (default: upp.bin) -Note that when trying to verify a UPP sent by the uBirch backend (Niomon) a verifying key doesn't have to be provided via the -k option. Instead, this script will try to pick the correct stage key based on the UUID which is contained in the UPP, identifying the creator. If the UUID doesn't match any Niomon stage and no key was specified using -k, an error message will be printed. +Note that, when trying to verify a UPP, sent by the uBirch backend (Niomon), a verifying key doesn't have to be provided via the -k option. Instead, this script will try to pick the correct stage key based on the UUID which is contained in the UPP, identifying the creator. If the UUID doesn't match any Niomon stage and no key was specified using -k, an error message will be printed. ``` - `--verifying-key/-k` If not trying to verify a UPP coming from uBirch Niomon but from another source, the verifying key for that source needs to be provided. This parameter expects the key as a hex-string like `b12a906051f102881bbb487ee8264aa05d8d0fcc51218f2a47f562ceb9b0d068`. - `--verifying-key-uuid/-u` The UUID for the verifying key from `--verifying-key`. This parameter will be ignored when `--verifying-key` is not set. Not setting this parameter when `--verifying-key` is set will cause an error. -- The file path to read the UPP from. +- `--input/-i` The file path to read the UPP from. ``` $ python3 upp-verifier.py --input response_upp.bin @@ -241,7 +250,7 @@ $ python3 upp-verifier.py --input response_upp.bin ``` ### Examining a UPP -To examine the contents (Version Field, Type Field, UUID, Signature, Payload, Previous Signature) of an UPP, the `upp-unpacker.py` script can be used like this: +To examine the contents (Version Field, Type Field, UUID, Signature, Payload, Previous Signature) of a UPP, the `upp-unpacker.py` script can be used like this: ``` $ python3 upp-unpacker.py response_upp.bin - Version: 0x23 @@ -256,8 +265,8 @@ $ python3 upp-unpacker.py response_upp.bin ``` _The UUID in this response UPP doesn't match the one from examples above because the UPP was sent from Niomon-Dev._ -### Checking the anchoring status of an UPP -uBirch Niomon accepting the UPP doesn't mean that it is anchored yet. This process takes place in certain intervals, so one might have to wait a short while. The script for that task is `upp-anchoring-status.py`. +### Checking the anchoring status of a UPP +uBirch Niomon accepting the UPP doesn't mean that it is anchored yet. This process takes place in certain intervals, so one might have to wait a short while. The script to check if a UPP was already anchored is `upp-anchoring-status.py`. ``` $ python3 upp-anchoring-status.py -h usage: upp-anchoring-status.py [-h] [--ishash ISHASH] [--env ENV] INPUT @@ -275,7 +284,7 @@ optional arguments: When --ishash/-i is set to true, the input argument is treated as a base64 payload hash. Otherwise, it is expected to be some kind of path to read a UPP from. This can be a file path or also /dev/stdin if the UPP is piped to this program via standard input. ``` -- `--ishash/-i` A boolean specifying wether the input data is a payload hash or an UPP. The payload hash is what is actually used to look up anchoring information about the UPP. This script can either extract it from a given UPP or just use the hash directly if provided. If directly provided, it must be base64 encoded (see the last example of this sub-section). `true` or `false`. +- `--ishash/-i` A boolean specifying whether the input data is a payload hash or a UPP. The payload hash is what is actually used to look up anchoring information about the UPP. This script can either extract it from a given UPP or just use the hash directly if provided. If directly provided, it must be base64 encoded (see the last example of this sub-section). `true` or `false`. - `--env/-e` The stage to check on. Should be the one the UPP was sent to. `prod`, `demo` or `dev`. - `INPUT` The input UPP file path or payload hash, depending on `--ishash`. @@ -290,7 +299,7 @@ Curr. UPP: "liPEEPXe2KPUYkHEqNyvP9ByohfEQMzH452aGsvznTB9CNW190IYAW4LnnTR78dkDFQM Prev. UPP: "liPEEPXe2KPUYkHEqNyvP9ByohfEQMvoTzPB2AqaKmjxDGHIQ1ZwNdGReacDu14K/06SDZuFNayxcfH9VScTcdGZ/JhfM88LMfPG7Pp75oS1YaxtkA8AxCB19C7vAEIvZoK4CpYuTIcRcLNOUod18yvs8yvERzBjoMRAzMfjnZoay/OdMH0I1bX3QhgBbguedNHvx2QMVAxM2hvxgrOJp+2f0/77BHzmz1E90aBHGT7QoTEQ9yf+9EIRAg==" 2021-07-02 16:01:46,950 root get_status() INFO The UPP has NOT been anchored into any blockchains yet! Please retry later ``` -Here it is visible that the backend knows the UPP and that it is valid, but it hasn't been anchored yet. Additionally the output shows that The backend knows the previous UPP, indicating that the UPP is a chained UPP and not the first UPP in the chain. When using unchained UPPs the line will change to: `Prev. UPP: "None"`. After waiting some time and running the script again with the same parameters: +Here it is visible that the backend knows the UPP and that it is valid, but it hasn't been anchored yet. Additionally, the output shows that the backend knows the previous UPP, indicating that the UPP is a chained UPP and not the first UPP in the chain. When using unchained UPPs, the line will change to: `Prev. UPP: "None"`. After waiting some time and running the script again with the same parameters: ``` $ python3 upp-anchoring-status.py --env demo upp.bin 2021-07-02 16:09:34,521 root read_upp() INFO Reading the input UPP from "upp.bin" @@ -314,8 +323,8 @@ Prev. UPP: "liPEEAi4JX1SUEaGhzAnqUf6pn3EQOeEqF4lo+xT9RF2ygDx9+anv14fykUolJ9gmKuT [{'label': 'PUBLIC_CHAIN', 'properties': {'timestamp': '2021-07-02T13:46:45.949Z', 'hash': '0xc32ccbe70ed727a1842f998d56d9928a9a30e201aef91bb08d9ac7faf931dac6', 'public_chain': 'ETHEREUM-CLASSIC_MAINNET_ETHERERUM_CLASSIC_MAINNET_NETWORK', 'prev_hash': '4d79e0d331b1fe057b3c9ee7cb595c371ec0ea764147029a862b3cffce808ae049ec40a6e5cddbd6ee90e4d36955cd2e08ab1f4ef1ccc8c013710617bd689cfe'}}, {'label': 'PUBLIC_CHAIN', 'properties': {'timestamp': '2021-07-02T13:58:24.677Z', 'hash': '0x65438aab8904c467ceb5b22e6b1a4198eeb056647a8b7a9ece0d03739a79a0bb', 'public_chain': 'GOV-DIGITAL_MAINNET_GOV_DIGITAL_MAINNET_NETWORK', 'prev_hash': 'e98a502253ec3aebd9af29cd368e3db899e5c04d6e8214be17856055b9301b63ce2ddb34038c4d3366ab9b4be28f755041f3a089d36c3516d84b0a95741289e3'}}] ``` -### Verifying a measurement -Being able to check if a UPP is anchored and valid is nice, but in a real usecase it might not be that useful. That's because usually the goal is the check if some data is valid. Verifying a UPP doesn't really help in that case, since it is not possible to reverse the hash contained in the UPP back to the original data. That's why one shouldn't bother to store sent UPPs, but rather the measurements these UPPs are based on. Those can then be hashed, and the hash can be looked up. The `data-verifier.py` script does exactly that. It has similar behaviour to the `upp-anchoring-status.py` script, see [Checking the anchoring status of an UPP](#checking-the-anchoring-status-of-an-upp). +### Verifying data +In a real use case, not the UPP, but rather the original data itself has to be verified. The original data can be hashed, and the hash can be looked up by the `data-verifier.py` script. It has similar behaviour to the `upp-anchoring-status.py` script, see [Checking the anchoring status of a UPP](#checking-the-anchoring-status-of-an-upp). ``` $ python data-verifier.py -h usage: data-verifier.py [-h] [--ispath ISHASH] [--env ENV] [--isjson ISJSON] [--hash HASH] INPUT @@ -353,7 +362,8 @@ python data-verifier.py --env demo --isjson true --hash sha256 '{ 2021-07-02 16:21:41,178 root get_hash_from_data() INFO Calculated hash: "dfQu7wBCL2aCuAqWLkyHEXCzTlKHdfMr7PMrxEcwY6A=" 2021-07-02 16:21:41,178 root get_status() INFO Requesting anchoring information from: "https://verify.demo.ubirch.com/api/upp/verify/anchor" 2021-07-02 16:21:41,599 root get_status() INFO The hash is known to the uBirch backend! (code: 200) -Curr. UPP: "liPEEPXe2KPUYkHEqNyvP9ByohfEQMvoTzPB2AqaKmjxDGHIQ1ZwNdGReacDu14K/06SDZuFNayxcfH9VScTcdGZ/JhfM88LMfPG7Pp75oS1YaxtkA8AxCB19C7vAEIvZoK4CpYuTIcRcLNOUod18yvs8yvERzBjoMRAzMfjnZoay/OdMH0I1bX3QhgBbguedNHvx2QMVAxM2hvxgrOJp+2f0/77BHzmz1E90aBHGT7QoTEQ9yf+9EIRAg==" +Curr. UPP: "liPEEPXe2KPUYkHEqNyvP9ByohfEQMvoTzPB2AqaKmjxDGHIQ1ZwNdGReacDu14K/06SDZuFNayxcfH9VScTcdGZ/JhfM88LMfPG7Pp75oS1YaxtkA8AxCB19C7vAEIvZoK4CpYuTIcRcLNOUod18yvs8yvERzBjoMRAzMfjnZoay + /OdMH0I1bX3QhgBbguedNHvx2QMVAxM2hvxgrOJp+2f0/77BHzmz1E90aBHGT7QoTEQ9yf+9EIRAg==" Prev. UPP: "None" 2021-07-02 16:21:41,600 root get_status() INFO The corresponding UPP has been fully anchored! [{'label': 'PUBLIC_CHAIN', 'properties': {'timestamp': '2021-07-02T13:23:30.076Z', 'hash': '0x6e5956b4ac53bcaf58664e189673d4f8c7043488cf05009cc96868b146220604', 'public_chain': 'ETHEREUM-CLASSIC_TESTNET_ETHERERUM_CLASSIC_KOTTI_TESTNET_NETWORK', 'prev_hash': '644b41eee9043de5bda4b58bda1136fa0229712953678fe26486651338109b7a7135211f2b2cb646ab25d3b25198549e3c662c4791d60d343f08349b51ccc92b'}}] @@ -361,7 +371,7 @@ Prev. UPP: "None" Just like with `upp-anchoring-status.py`, it might take a short while after sending the corresponding UPP to the backend before it will be anchored. ## Sending data to the Simple Data Service -The `data-sender.py` example-script allows sending of data to the simple data service. This is only used for demo purposes. +The `data-sender.py` example-script allows sending of data to the simple data service. This should only be used for demo purposes. Ubirch will not guarantee, to keep all data, which is sent to this endpoint. ``` $ python3 data-sender.py --help usage: data-sender.py [-h] [--env ENV] UUID AUTH INPUT From 4b58c5576523ab1746a2a87339a59015b2b63048 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Tue, 6 Jul 2021 14:57:14 +0200 Subject: [PATCH 22/40] . --- examples/EXAMPLES.md | 90 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 75 insertions(+), 15 deletions(-) diff --git a/examples/EXAMPLES.md b/examples/EXAMPLES.md index dcf0808..5ab6dd4 100644 --- a/examples/EXAMPLES.md +++ b/examples/EXAMPLES.md @@ -43,11 +43,11 @@ The values used below are `f5ded8a3-d462-41c4-a8dc-af3fd072a217` for the UUID, ` `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` for the auth token. ### Generating and managing a keypair -To create, or more precisely, to _sign_ a UPP, a device will need a keypair. This keypair consist of a private key (_signing key_) and public key (_verifying key_). The signing key is used to sign UPPs and the verifying key can be used by the uBirch backend to check, if the signature is valid and belongs to the correct sender/signer. So, logically it doesn't matter who knows to verifying key, but the signing key must be kept secret all the time. In a real use case a device might store it in a TPM ([Trusted platform module](https://en.wikipedia.org/wiki/Trusted_Platform_Module)) or use other counter measures against attackers reading the key from the device. For this demo, keypairs will be stored in a [JKS Keystore](https://en.wikipedia.org/wiki/Java_KeyStore) using the [`pyjks`](https://pypi.org/project/pyjks/) library. Therefore, you will have to choose and remember a file path for that keystore and a password used to encrypt it. The process of actually generating the keypair is explained [below](#registering-a-public-key). +To create, or more precisely, to _sign_ a UPP, a device will need a keypair. This keypair consist of a private key (_signing key_) and public key (_verifying key_). The signing key is used to sign UPPs and the verifying key can be used by the uBirch backend to check, if the signature is valid and belongs to the correct sender/signer. So, logically it doesn't matter who knows to verifying key, but the signing key must be kept secret all the time. In a real use case a device might store it in a TPM ([Trusted platform module](https://en.wikipedia.org/wiki/Trusted_Platform_Module)) or use other counter measures against attackers reading the key from the device. For this demo, keypairs will be stored in a [JKS Keystore](https://en.wikipedia.org/wiki/Java_KeyStore) using the [`pyjks`](https://pypi.org/project/pyjks/) library. Therefore, you will have to choose and remember a file path for that keystore and a password used to encrypt it. The process of actually generating the keypair is handled by the [upp-creator.py] script and explained [below](#registering-a-public-key). **NOTE** that losing access to the signing key, especially if it is already registered at the uBirch backend, will take away the ability to create and send any new UPPs from that device/UUID, since there is no way of creating a valid signature that would be accepted by the backend. -A keystore can be read out with the `keystore-dumper.py` script. +A keystore can be read out with the [`keystore-dumper.py`](keystore-dumper.py) script. ``` $ python keystore-dumper.py --help @@ -61,7 +61,7 @@ positional arguments: optional arguments: -h, --help show this help message and exit - --show-sk SHOW_SIGNING_KET, -s SHOW_SIGNING_KET + --show-sk SHOW_SIGNING_KEY, -s SHOW_SIGNING_KEY enables/disables showing of signing keys; e.g.: true, false (default: False) ``` By default, only UUIDs und public keys (verifying keys) will be displayed. Displaying of private keys (signing keys) can be enabled by passing `-s true`. @@ -85,7 +85,7 @@ $ python upp-creator.py -t 1 --ks devices.jks --kspwd keystore --keyreg true --o 2021-07-02 11:51:50,485 root show_store_upp() INFO UPP [hex]: "9522c410f5ded8a3d46241c4a8dcaf3fd072a2170187a9616c676f726974686dab4543435f45443235353139a763726561746564ce60dec596aa68774465766963654964c410f5ded8a3d46241c4a8dcaf3fd072a216a67075624b6579c420e0264e7d9428149cef59ccecb8813b214d8f94c62e3e836d7546d3f8bd884a4ca87075624b65794964c420e0264e7d9428149cef59ccecb8813b214d8f94c62e3e836d7546d3f8bd884a4cad76616c69644e6f744166746572ce62bff916ae76616c69644e6f744265666f7265ce60dec596c440cadb70d30250a5a2dd2eb44b645e54b56387f228607fbf6f59a11493befa118f0e9c79da1f7d85ba5a4076c134f8b4aff04173adfc4b858ec491be2366988900" 2021-07-02 11:51:50,485 root show_store_upp() INFO UPP written to "keyreg_upp.bin" ``` -The generated key registration UPP has been saved to `keyreg_upp.bin`. Sending the UPP can be done like following (remember to put the correct value for the env of your choice): +The [upp-creator.py](upp-creator.py) script will check if the keystore specified with `--ks` contains an entry for the device with the given UUID. If it doesn´t, the script will generate a new keypair and store it. This can be seen when examining the two first log messages starting with `No keys found for` and `inserted new keypair for`. Otherwsise (if there already is a keypair for the device) the script will simple use the existent keypair. The generated key registration UPP has been saved to `keyreg_upp.bin`. Sending the UPP can be done like following (remember to put the correct value for the env of your choice): ``` $ python upp-sender.py --env demo --input keyreg_upp.bin --output response_upp.bin f5ded8a3-d462-41c4-a8dc-af3fd072a217 xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx @@ -118,7 +118,7 @@ This will create a string from the above examplary JSON object: `{"H":35.8,"S":"OK","T":11.2,"ts":1625163338}` ### Creating a UPP -After gathering some measurement data a UPP can be created. The UPP won't contain the actual measurement data, but a hash of it. The example script to create UPPs in is `upp-creator.py`. +After gathering some measurement data a UPP can be created. The UPP won't contain the actual measurement data, but a hash of it. The example script to create UPPs is [`upp-creator.py`](upp-creator.py). ``` $ python3 upp-creator.py --help @@ -183,7 +183,7 @@ $ python3 upp-creator.py --version 0x23 --isjson true --output upp.bin --hash sh Keep in mind that if you use chained UPPs (`--version 0x23`) you should anchor each UPP, or the signature chain will be broken. This won't cause any errors, but the advantage of chaining UPPs and thereby knowing the correct order of them, will get lost. ### Sending a UPP -After creating the UPP, it can be sent to the uBirch backend where it will be verified and anchored in the blockchain. The ubirch backend will use the registered public/verifying key to check the signature. The `upp-sender.py` script can be used for that. +After creating the UPP, it can be sent to the uBirch backend where it will be verified and anchored in the blockchain. The ubirch backend will use the registered public/verifying key to check the signature. The [`upp-sender.py`](upp-sender.py) script can be used for that. ``` $ python3 upp-sender.py --help usage: upp-sender.py [-h] [--env ENV] [--input INPUT] [--output OUTPUT] UUID AUTH @@ -204,7 +204,7 @@ optional arguments: ``` For this script the parameters are: - `--env/-e` The env to operate on. This parameter decides wether the UPP will be sent to `niomon.prod.ubirch.com`, `niomon.demo.ubirch.com` or `niomon.dev.ubirch.com`. The value can either be `prod`, `demo` or `dev`. It must match the stage, the UUID is registered on. -- `--input/-i` Specifies where to read the UPP to be sent from. This can be a normal file path or also `/dev/stdin`, if for example the UPP will be piped to this script from another script (like `upp-creator.py`). In most cases the UPP will just be read from some file. +- `--input/-i` Specifies where to read the UPP to be sent from. This can be a normal file path or also `/dev/stdin`, if for example the UPP will be piped to this script from another script (like [`upp-creator.py`](upp-creator.py)). In most cases the UPP will just be read from some file. - `--output/-o` Normally the uBirch backend will respond to the UPP with another UPP. This parameter sets the location to write that response-UPP to. - `UUID` The UUID of the device that generated the UPP as a hex-string. - `AUTH` The auth token for the device on the specified stage as a hex-string. @@ -218,7 +218,7 @@ $ python upp-sender.py --env demo --input upp.bin --output response_upp.bin f5de ``` ### Verifying a UPP -To make sure, that the response UPP actually was sent by the uBirch backend, its signature can be checked. The example script for that is `upp-verifier.py`. It knows the UUID and verifying/public key for each uBirch Niomon stage end checks, if the signature of the response UPP is valid. +To make sure, that the response UPP actually was sent by the uBirch backend, its signature can be checked. The example script for that is [`upp-verifier.py`](upp-verifier.py). It knows the UUID and verifying/public key for each uBirch Niomon stage end checks, if the signature of the response UPP is valid. ``` $ python3 upp-verifier.py --help @@ -250,7 +250,7 @@ $ python3 upp-verifier.py --input response_upp.bin ``` ### Examining a UPP -To examine the contents (Version Field, Type Field, UUID, Signature, Payload, Previous Signature) of a UPP, the `upp-unpacker.py` script can be used like this: +To examine the contents (Version Field, Type Field, UUID, Signature, Payload, Previous Signature) of a UPP, the [`upp-unpacker.py`](upp-unpacker.py) script can be used like this: ``` $ python3 upp-unpacker.py response_upp.bin - Version: 0x23 @@ -266,7 +266,7 @@ $ python3 upp-unpacker.py response_upp.bin _The UUID in this response UPP doesn't match the one from examples above because the UPP was sent from Niomon-Dev._ ### Checking the anchoring status of a UPP -uBirch Niomon accepting the UPP doesn't mean that it is anchored yet. This process takes place in certain intervals, so one might have to wait a short while. The script to check if a UPP was already anchored is `upp-anchoring-status.py`. +uBirch Niomon accepting the UPP doesn't mean that it is anchored yet. This process takes place in certain intervals, so one might have to wait a short while. The script to check if a UPP was already anchored is [`upp-anchoring-status.py`](upp-anchoring-status.py). ``` $ python3 upp-anchoring-status.py -h usage: upp-anchoring-status.py [-h] [--ishash ISHASH] [--env ENV] INPUT @@ -324,7 +324,7 @@ Prev. UPP: "liPEEAi4JX1SUEaGhzAnqUf6pn3EQOeEqF4lo+xT9RF2ygDx9+anv14fykUolJ9gmKuT ``` ### Verifying data -In a real use case, not the UPP, but rather the original data itself has to be verified. The original data can be hashed, and the hash can be looked up by the `data-verifier.py` script. It has similar behaviour to the `upp-anchoring-status.py` script, see [Checking the anchoring status of a UPP](#checking-the-anchoring-status-of-an-upp). +In a real use case, not the UPP, but rather the original data itself has to be verified. The original data can be hashed, and the hash can be looked up by the [`data-verifier.py`](data-verifier.py) script. It has similar behaviour to the [`upp-anchoring-status.py`](upp-anchoring-status.py) script, see [Checking the anchoring status of a UPP](#checking-the-anchoring-status-of-an-upp). ``` $ python data-verifier.py -h usage: data-verifier.py [-h] [--ispath ISHASH] [--env ENV] [--isjson ISJSON] [--hash HASH] INPUT @@ -350,7 +350,7 @@ When --ispath/-i is set to true, the input data is treated as a file path to rea - `--isjson/-j` A binary flag that indicates that the input data is in JSON format. The script will serialize the JSON object before calculating the hash. This has the advantage one doesn't have to remember the order in which fields are listed in a JSON object to still be able to reconstruct the hash later on. Serializing the JSON is done like this: `json.dumps(self.data, separators=(',', ':'), sort_keys=True, ensure_ascii=False)` where `self.data` contains the JSON object which was loaded like this: `self.data = json.loads(self.dataStr)` where `dataStr` contains the input string which should represent a JSON object. This flag can have two values: `true` or `false`. It should only be set to `true` if the data represents a JSON object and if it also was serialized when creating the UPP. - `--hash/-a` Sets the hashing algorithm to use. `sha256`, `sha512` or `off`. It should match the algorithm used when creating the corresponding UPP. Setting it to `off` means that the input data actually already is the hash of the data. In this case this script will simply look up the hash. -Example: +Example for CLI-Input data: ``` python data-verifier.py --env demo --isjson true --hash sha256 '{ "ts": 1625163338, @@ -368,10 +368,24 @@ Prev. UPP: "None" 2021-07-02 16:21:41,600 root get_status() INFO The corresponding UPP has been fully anchored! [{'label': 'PUBLIC_CHAIN', 'properties': {'timestamp': '2021-07-02T13:23:30.076Z', 'hash': '0x6e5956b4ac53bcaf58664e189673d4f8c7043488cf05009cc96868b146220604', 'public_chain': 'ETHEREUM-CLASSIC_TESTNET_ETHERERUM_CLASSIC_KOTTI_TESTNET_NETWORK', 'prev_hash': '644b41eee9043de5bda4b58bda1136fa0229712953678fe26486651338109b7a7135211f2b2cb646ab25d3b25198549e3c662c4791d60d343f08349b51ccc92b'}}] ``` -Just like with `upp-anchoring-status.py`, it might take a short while after sending the corresponding UPP to the backend before it will be anchored. +Example for File-Input data: +``` +python data-verifier.py --ispath true -j true data_to_verify.json -e prod +2021-07-06 12:17:31,261 root read_data() INFO Reading the input data from "data_to_verify.json" +2021-07-06 12:17:31,262 root serialize_json() INFO Serialized JSON: "{"data":{"AccPitch":"-11.52","AccRoll":"1.26","AccX":"-0.02","AccY":"0.20","AccZ":"0.99","H":"64.85","L_blue":232,"L_red":275,"P":"100934.00","T":"20.69","V":"4.62"},"msg_type":1,"timestamp":1599203876,"uuid":"07104235-1892-4020-9042-00003c94b60b"}" +2021-07-06 12:17:31,262 root get_hash_from_data() INFO Calculated hash: "/hHAPCT60m0/pnsB2z4Y4TYNcALrBnKb8h1ZR429fuY=" +2021-07-06 12:17:31,262 root get_status() INFO Requesting anchoring information from: "https://verify.prod.ubirch.com/api/upp/verify/anchor" +2021-07-06 12:17:31,784 root get_status() INFO The hash is known to the uBirch backend! (code: 200) +Curr. UPP: "liPEEAcQQjUYkkAgkEIAADyUtgvEQHy+eJ38aa7R6A1K+5ZLqYxoP7EraPYBo9cTllip+FCVm3OkzfDNB36/yMkJT5GqyopDs1mBJu8Y3kYczX8VM8oAxCD+EcA8JPrSbT+mewHbPhjhNg1wAusGcpvyHVlHjb1+5sRAsp7YwQtGxGBXX/PgbjEd1JQP1qDWOfDDsYc0oJ0jrZcjLvJv6SGnIgnZvmF1YSYewnHe56Fb3GApTw7Ybs43SQ==" +Prev. UPP: "liPEEAcQQjUYkkAgkEIAADyUtgvEQJViO08kxDSmJWebjNDFAVFwqxGUANe9XkNqi549sVLSlCcNd1lLFWGfXUttolDlENsSgjejqH7Iwf2QxAJWqmsAxCDbSx12E4W489A0oKaaFm+cpCqp9ShhfPJockqU/axOgMRAfL54nfxprtHoDUr7lkupjGg/sSto9gGj1xOWWKn4UJWbc6TN8M0Hfr/IyQlPkarKikOzWYEm7xjeRhzNfxUzyg==" +2021-07-06 12:17:31,784 root get_status() INFO The corresponding UPP has been fully anchored! +[{'label': 'PUBLIC_CHAIN', 'properties': {'timestamp': '2020-09-04T07:27:59.196Z', 'hash': 'FNMJQLBQXPRDAG9ZDPEDZEOQOXEUPJQOFOOBBEUZRXA9BBY9FRZRSYABEAFTYFCDFWJYDMXTZWVXZ9999', 'public_chain': 'IOTA_MAINNET_IOTA_MAINNET_NETWORK', 'prev_hash': 'bc318054140e1f4014977ebd37058807cba5c7c369cebe14daf8fbccdacb24ee135a0773764cb0ad6530fd0d8392d77f7f9d669b2ca973f13c683d1a8930d61b'}}, {'label': 'PUBLIC_CHAIN', 'properties': {'timestamp': '2020-09-04T07:28:11.886Z', 'hash': '0x2b1b940d98d35522a326396625690397ef9aab9c8dfb2b8f63a7e7a297559ce9', 'public_chain': 'ETHEREUM-CLASSIC_MAINNET_ETHERERUM_CLASSIC_MAINNET_NETWORK', 'prev_hash': 'bc318054140e1f4014977ebd37058807cba5c7c369cebe14daf8fbccdacb24ee135a0773764cb0ad6530fd0d8392d77f7f9d669b2ca973f13c683d1a8930d61b'}}, {'label': 'PUBLIC_CHAIN', 'properties': {'timestamp': '2020-09-04T07:28:19.414Z', 'hash': '0x133627d9effaa40186d0bab8331cff05242c87178a32ca370f5fa7512716c361', 'public_chain': 'GOV-DIGITAL_MAINNET_GOV_DIGITAL_MAINNET_NETWORK', 'prev_hash': 'bc318054140e1f4014977ebd37058807cba5c7c369cebe14daf8fbccdacb24ee135a0773764cb0ad6530fd0d8392d77f7f9d669b2ca973f13c683d1a8930d61b'}}] +``` + +Just like with [`upp-anchoring-status.py`](upp-anchoring-status.py), it might take a short while after sending the corresponding UPP to the backend before it will be anchored. ## Sending data to the Simple Data Service -The `data-sender.py` example-script allows sending of data to the simple data service. This should only be used for demo purposes. Ubirch will not guarantee, to keep all data, which is sent to this endpoint. +The [`data-sender.py`](data-sender.py) example-script allows sending of data to the simple data service. This should only be used for demo purposes. Ubirch will not guarantee, to keep all data, which is sent to this endpoint. ``` $ python3 data-sender.py --help usage: data-sender.py [-h] [--env ENV] UUID AUTH INPUT @@ -391,4 +405,50 @@ Note that the input data should follow this pattern: {"timestamp": TIMESTAMP, "u ``` ## Example uBirch client implementation -`example-client.py` implements a full example uBirch client. +[`example-client.py`](example-client.py) implements a full example uBirch client. It generates a keypair if needed, registers it at the uBirch backend if it doesn't know it yet, creates and sends an UPP and handles/verfies the response from the uBirch backend. The used message format looks like this: +``` +{ + "id": "UUID", + "ts": TIMESTAMP, + "data": "DATA" +} +``` +It has two positional and one optional command line parameters. +``` +usage: python3 example-client.py [ubirch-env] +``` +- `UUID` is the UUID as hex-string like `f5ded8a3-d462-41c4-a8dc-af3fd072a217` +- `ubirch-auth-token` is the uBirch authentication token for the specified UUID, e.g.: `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` +- `ubirch-env` (optional) specifies the environment/stage to operator on. `dev`, `demo` or `prod` (default). +Keys are loaded from/stored to `demo-device.jks`. The keystore-password can be read from the [script](example-client.py) itself. + + +## Create a hash from an JSON object +[`create-hash.py`](create-hash.py) takes a string representing a JSON object as input, serializes it, and calculates the corresponding SHA256 hash. +``` +$ python3 create-hash.py '{"ts": 1625163338, "T": 11.2, "H": 35.8, "S": "OK"}' + input: {"ts": 1625163338, "T": 11.2, "H": 35.8, "S": "OK"} +rendered: {"H":35.8,"S":"OK","T":11.2,"ts":1625163338} + hash: dfQu7wBCL2aCuAqWLkyHEXCzTlKHdfMr7PMrxEcwY6A= +``` + +## [`test-identity.py`](test-identity.py) +The [`test-identity.py`](test-identity.py) script tests registering and de-registering a public key of a device at the uBirch backend. To function it needs the following variables to be set using the environment: +```sh +export UBIRCH_UUID= +export UBIRCH_AUTH= +export UBIRCH_ENV=[dev|demo|prod] +``` +It uses `test-identity.jsk` as a place to store/look for keypairs. The keystore-password can be read from the [script](test-identity.py) itself. + +## [`test-protocol.py`](test-protocol.py) +The [`test-protocol.py`](test-protocol.py) script sends a couple of UPPs to uBirch Niomon and verifies the backend response. It reads all information it needs interactively from the terminal. Once entered, all device information (UUID, ENV, AUTH TOKEN) are stored in a file called `demo-device.ini`. Devices keys are stored in `demo-device.jks` and the keystore-password can be read from the [script](test-protocol.py) itself. If no keys for the given UUID are found, the script will generated a keypair and stores it in the keystore file. + +## [`test-web-of-trust.py`](test-web-of-trust.py) +**TODO** + +## [`verify-ecdsa.py`](verify-ecdsa.py) +The [`verify-ecdsa.py`](verify-ecdsa.py) script verifies a hard-coded UPP which was signed with an ECDSA signing key using a ECDSA verifying key. All the information are contained in the script. + +## [`verify-ed25519.py`](verify-25519.py) +The [`verify-25519.py`](verify-25519.py) script verifies a hard-coded UPP which was signed with an ED25519 signing key using a ED25519 verifying key. All the information are contained in the script. This mode is normally used (in all other examples). \ No newline at end of file From e7812ab157ed9f51a5a814e56d88b46341d8983a Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Wed, 7 Jul 2021 08:33:51 +0200 Subject: [PATCH 23/40] fix incomplete toc --- examples/EXAMPLES.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/examples/EXAMPLES.md b/examples/EXAMPLES.md index 5ab6dd4..e174a97 100644 --- a/examples/EXAMPLES.md +++ b/examples/EXAMPLES.md @@ -17,12 +17,25 @@ This file documents how to use the examples provided alongside the [uBirch-Proto - [Verifying data](#verifying-data) - [Sending data to the Simple Data Service](#sending-data-to-the-simple-data-service) - [Example uBirch client implementation](#example-ubirch-client-implementation) + - [Create a hash from a JSON object](#create-a-hash-from-an-json-object) + - [test-identiy.py](#test-identity.py) + - [test-web-of-trust.py](#test-web-of-trust.py) + - [verify-ecdsa.py](#verify-ecdsa.py) + - [verify-ed25519.py](#verify-ed25519.py) ## From measurement to blockchain-anchored UPP The process needed to get a UPP to be anchored in the blockchain can be cut down into multiple steps. For each of those steps there is an example in this directory, demonstrating how to handle them. There are also examples showing a full example-client implementation. 0. [Setup](#setup) -1. [Gathering Data](#gathering-data) +1. [Generating and managing a keypair](#generating-and-managing-a-keypair) +2. [Registering a public key](#registering-a-public-key) +3. [Gathering Data](#gathering-data) +4. [Creating a UPP](#creating-a-upp) +5. [Sending a UPP](#sending-a-upp) +6. [Verifying a UPP](#verifying-a-upp) +7. [Examining a UPP](#examining-a-upp) +8. [Checking the anchoring status of a UPP](#checking-the-anchoring-status-of-an-upp) +9. [Verifying data](#verifying-data) ### Setup Before anything, you will need to do/get a couple of things: From 44d64a2079db0582052939330f38abf65da44c5c Mon Sep 17 00:00:00 2001 From: wowa Date: Thu, 8 Jul 2021 08:24:44 +0200 Subject: [PATCH 24/40] test-identity try to solve broken reference --- examples/EXAMPLES.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/EXAMPLES.md b/examples/EXAMPLES.md index e174a97..514c609 100644 --- a/examples/EXAMPLES.md +++ b/examples/EXAMPLES.md @@ -18,7 +18,7 @@ This file documents how to use the examples provided alongside the [uBirch-Proto - [Sending data to the Simple Data Service](#sending-data-to-the-simple-data-service) - [Example uBirch client implementation](#example-ubirch-client-implementation) - [Create a hash from a JSON object](#create-a-hash-from-an-json-object) - - [test-identiy.py](#test-identity.py) + - [test-identity.py](#test-identity) - [test-web-of-trust.py](#test-web-of-trust.py) - [verify-ecdsa.py](#verify-ecdsa.py) - [verify-ed25519.py](#verify-ed25519.py) @@ -56,7 +56,7 @@ The values used below are `f5ded8a3-d462-41c4-a8dc-af3fd072a217` for the UUID, ` `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` for the auth token. ### Generating and managing a keypair -To create, or more precisely, to _sign_ a UPP, a device will need a keypair. This keypair consist of a private key (_signing key_) and public key (_verifying key_). The signing key is used to sign UPPs and the verifying key can be used by the uBirch backend to check, if the signature is valid and belongs to the correct sender/signer. So, logically it doesn't matter who knows to verifying key, but the signing key must be kept secret all the time. In a real use case a device might store it in a TPM ([Trusted platform module](https://en.wikipedia.org/wiki/Trusted_Platform_Module)) or use other counter measures against attackers reading the key from the device. For this demo, keypairs will be stored in a [JKS Keystore](https://en.wikipedia.org/wiki/Java_KeyStore) using the [`pyjks`](https://pypi.org/project/pyjks/) library. Therefore, you will have to choose and remember a file path for that keystore and a password used to encrypt it. The process of actually generating the keypair is handled by the [upp-creator.py] script and explained [below](#registering-a-public-key). +To create, or more precisely, to _sign_ a UPP, a device will need a keypair. This keypair consist of a private key (_signing key_) and public key (_verifying key_). The signing key is used to sign UPPs and the verifying key can be used by the uBirch backend to check, if the signature is valid and belongs to the correct sender/signer. So, logically it doesn't matter who knows to verifying key, but the signing key must be kept secret all the time. In a real use case a device might store it in a TPM ([Trusted platform module](https://en.wikipedia.org/wiki/Trusted_Platform_Module)) or use other counter measures against attackers reading the key from the device. For this demo, keypairs will be stored in a [JKS Keystore](https://en.wikipedia.org/wiki/Java_KeyStore) using the [`pyjks`](https://pypi.org/project/pyjks/) library. Therefore, you will have to choose and remember a file path for that keystore and a password used to encrypt it. The process of actually generating the keypair is handled by the [upp-creator.py](upp-creator.py) script and explained [below](#registering-a-public-key). **NOTE** that losing access to the signing key, especially if it is already registered at the uBirch backend, will take away the ability to create and send any new UPPs from that device/UUID, since there is no way of creating a valid signature that would be accepted by the backend. @@ -445,7 +445,7 @@ rendered: {"H":35.8,"S":"OK","T":11.2,"ts":1625163338} hash: dfQu7wBCL2aCuAqWLkyHEXCzTlKHdfMr7PMrxEcwY6A= ``` -## [`test-identity.py`](test-identity.py) +## [`test-identity.py`](test-identity.py){#test-identity} The [`test-identity.py`](test-identity.py) script tests registering and de-registering a public key of a device at the uBirch backend. To function it needs the following variables to be set using the environment: ```sh export UBIRCH_UUID= From 5c6debc3bdfde13d7750c44f4bce3e96573914ad Mon Sep 17 00:00:00 2001 From: wowa Date: Thu, 8 Jul 2021 08:43:51 +0200 Subject: [PATCH 25/40] solve broken reference links --- examples/EXAMPLES.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/examples/EXAMPLES.md b/examples/EXAMPLES.md index 514c609..5c222e1 100644 --- a/examples/EXAMPLES.md +++ b/examples/EXAMPLES.md @@ -18,10 +18,11 @@ This file documents how to use the examples provided alongside the [uBirch-Proto - [Sending data to the Simple Data Service](#sending-data-to-the-simple-data-service) - [Example uBirch client implementation](#example-ubirch-client-implementation) - [Create a hash from a JSON object](#create-a-hash-from-an-json-object) - - [test-identity.py](#test-identity) - - [test-web-of-trust.py](#test-web-of-trust.py) - - [verify-ecdsa.py](#verify-ecdsa.py) - - [verify-ed25519.py](#verify-ed25519.py) + - [Test identity of the device](#test-identity-of-the-device) + - [Test the complete protocol](#test-the-complete-protocol) + - [Test the web of trust](#test-the-web-of-trust) + - [Verify ECDSA signed UPP](#verify-ecdsa-signed-upp) + - [Verify ED25519 signed UPP](#verify-ed25519-signed-upp) ## From measurement to blockchain-anchored UPP The process needed to get a UPP to be anchored in the blockchain can be cut down into multiple steps. For each of those steps there is an example in this directory, demonstrating how to handle them. There are also examples showing a full example-client implementation. @@ -445,7 +446,7 @@ rendered: {"H":35.8,"S":"OK","T":11.2,"ts":1625163338} hash: dfQu7wBCL2aCuAqWLkyHEXCzTlKHdfMr7PMrxEcwY6A= ``` -## [`test-identity.py`](test-identity.py){#test-identity} +## Test identity of the device The [`test-identity.py`](test-identity.py) script tests registering and de-registering a public key of a device at the uBirch backend. To function it needs the following variables to be set using the environment: ```sh export UBIRCH_UUID= @@ -454,14 +455,15 @@ export UBIRCH_ENV=[dev|demo|prod] ``` It uses `test-identity.jsk` as a place to store/look for keypairs. The keystore-password can be read from the [script](test-identity.py) itself. -## [`test-protocol.py`](test-protocol.py) +## Test the complete protocol The [`test-protocol.py`](test-protocol.py) script sends a couple of UPPs to uBirch Niomon and verifies the backend response. It reads all information it needs interactively from the terminal. Once entered, all device information (UUID, ENV, AUTH TOKEN) are stored in a file called `demo-device.ini`. Devices keys are stored in `demo-device.jks` and the keystore-password can be read from the [script](test-protocol.py) itself. If no keys for the given UUID are found, the script will generated a keypair and stores it in the keystore file. -## [`test-web-of-trust.py`](test-web-of-trust.py) +## Test the web of trust +[`test-web-of-trust.py`](test-web-of-trust.py) **TODO** -## [`verify-ecdsa.py`](verify-ecdsa.py) +## Verify ECDSA signed UPP The [`verify-ecdsa.py`](verify-ecdsa.py) script verifies a hard-coded UPP which was signed with an ECDSA signing key using a ECDSA verifying key. All the information are contained in the script. -## [`verify-ed25519.py`](verify-25519.py) +## Verify ED25519 signed UPP The [`verify-25519.py`](verify-25519.py) script verifies a hard-coded UPP which was signed with an ED25519 signing key using a ED25519 verifying key. All the information are contained in the script. This mode is normally used (in all other examples). \ No newline at end of file From 6b48d137c1ffeac5bc012dae633d878f339c2205 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Sat, 4 Sep 2021 14:27:27 +0200 Subject: [PATCH 26/40] add option to only generate data hash without verifying it --- examples/data-verifier.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/examples/data-verifier.py b/examples/data-verifier.py index ff1c77f..1bd273f 100644 --- a/examples/data-verifier.py +++ b/examples/data-verifier.py @@ -13,6 +13,7 @@ DEFAULT_ENV = "dev" DEFAULT_ISJSON = "True" DEFAULT_HASH = "sha256" +DEFAULT_NOSEND = "False" logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) @@ -31,6 +32,8 @@ def __init__(self): self.isjson_str : str = None self.isjson : bool = None self.hashalg : str = None + self.nosend_str : str = None + self.nosend : bool = None self.ishash : bool = False self.hasher : object = None @@ -66,6 +69,9 @@ def setup_argparse(self): self.argparser.add_argument("--hash", "-a", metavar="HASH", type=str, default=DEFAULT_HASH, help="sets the hash algorithm to use; sha256, sha512 or OFF to treat the input data as hash (default: %s)" % DEFAULT_HASH ) + self.argparser.add_argument("--no-send", "-n", metavar="NOSEND", type=str, default=DEFAULT_NOSEND, + help="if set to true, the script will only generate the hash of the input data without sending it; true or false (default: %s)" % DEFAULT_NOSEND + ) def process_args(self) -> bool: # parse cli arguments (exists on err) @@ -77,6 +83,7 @@ def process_args(self) -> bool: self.isjson_str = self.args.isjson self.env = self.args.env self.hashalg = self.args.hash + self.nosend_str = self.args.no_send # check the value for --hash if self.hashalg.lower() == "off": @@ -110,6 +117,12 @@ def process_args(self) -> bool: else: self.isjson = False + # get the bool for nosend + if self.nosend_str.lower() in ["1", "yes", "y", "true"]: + self.nosend = True + else: + self.nosend = False + return True def read_data(self) -> bool: @@ -143,7 +156,7 @@ def serialize_json(self) -> bool: def get_hash_from_data(self) -> bool: try: # calculate the hash - self.hasher.update(self.data) + self.hasher.update(self.data if type(self.data) == bytes else self.data.encode()) self.hash = self.hasher.digest() self.hash = base64.b64encode(self.hash).decode().rstrip("\n") @@ -244,13 +257,14 @@ def run(self) -> int: return 1 - # get the anchoring status - if self.get_status() != True: - logger.error("Errors occured while requesting the anchring status - exiting!\n") + if self.nosend == False: + # get the anchoring status + if self.get_status() != True: + logger.error("Errors occured while requesting the anchring status - exiting!\n") - self.argparser.print_usage() + self.argparser.print_usage() - return 1 + return 1 return 0 From 76c7ce8e228124314296a2fb4714f55e7c5b9a90 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Tue, 12 Oct 2021 12:53:08 +0200 Subject: [PATCH 27/40] fix typo in usage info --- examples/keystore-dumper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/keystore-dumper.py b/examples/keystore-dumper.py index 0fa8060..3ae90b4 100644 --- a/examples/keystore-dumper.py +++ b/examples/keystore-dumper.py @@ -44,7 +44,7 @@ def setup_argparse(self): self.argparser.add_argument("keystore_pass", metavar="KEYSTORE_PASS", type=str, help="keystore password; e.g.: secret" ) - self.argparser.add_argument("--show-sk", "-s", metavar="SHOW_SIGNING_KET", type=str, default=DEFAULT_SHOW_SIGN, + self.argparser.add_argument("--show-sk", "-s", metavar="SHOW_SIGNING_KEY", type=str, default=DEFAULT_SHOW_SIGN, help="enables/disables showing of signing keys; e.g.: true, false (default: %s)" % DEFAULT_SHOW_SIGN ) From 701a60c7923e3f17986f421f81f9e4f1d3978807 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Tue, 12 Oct 2021 13:01:01 +0200 Subject: [PATCH 28/40] add hashlink functionality to data verifier and upp creator scripts --- examples/data-verifier.py | 84 ++++++++++++++++++++++++++++++++++++++- examples/upp-creator.py | 83 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 166 insertions(+), 1 deletion(-) diff --git a/examples/data-verifier.py b/examples/data-verifier.py index 1bd273f..07b3f66 100644 --- a/examples/data-verifier.py +++ b/examples/data-verifier.py @@ -14,6 +14,7 @@ DEFAULT_ISJSON = "True" DEFAULT_HASH = "sha256" DEFAULT_NOSEND = "False" +DEFAULT_ISHL = "False" logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) @@ -34,7 +35,8 @@ def __init__(self): self.hashalg : str = None self.nosend_str : str = None self.nosend : bool = None - + self.ishl_str : str = None + self.ishl : bool = None self.ishash : bool = False self.hasher : object = None @@ -72,6 +74,9 @@ def setup_argparse(self): self.argparser.add_argument("--no-send", "-n", metavar="NOSEND", type=str, default=DEFAULT_NOSEND, help="if set to true, the script will only generate the hash of the input data without sending it; true or false (default: %s)" % DEFAULT_NOSEND ) + self.argparser.add_argument("--ishl", "-l", metavar="ISHASHLINK", type=str, default=DEFAULT_ISHL, + help="implied --isjson to be true; if set to true, the script will look for a hashlink list in the json object and use it to decide which fields to hash; true or false (default: %s)" % DEFAULT_ISHL + ) def process_args(self) -> bool: # parse cli arguments (exists on err) @@ -84,6 +89,7 @@ def process_args(self) -> bool: self.env = self.args.env self.hashalg = self.args.hash self.nosend_str = self.args.no_send + self.ishl_str = self.args.ishl # check the value for --hash if self.hashalg.lower() == "off": @@ -123,6 +129,18 @@ def process_args(self) -> bool: else: self.nosend = False + # get the bool for ishl + if self.ishl_str.lower() in ["1", "yes", "y", "true"]: + self.ishl = True + else: + self.ishl = False + + # check if ishl is true + if (self.ishl == True and self.isjson == False): + logger.warning("Overwriting '--isjson false' because --ishl is true") + + self.isjson = True + return True def read_data(self) -> bool: @@ -139,6 +157,61 @@ def read_data(self) -> bool: return True + def _getValueFromDict(self, keyPath : list, currentObj : dict) -> object: + """ this function gets an object from the config object: config[path[0]][path[1]][path[n]] """ + if len(keyPath) == 0 or not currentObj: + return currentObj + elif type(currentObj) == list and type(keyPath[0]) == int: + return self._getValueFromDict(keyPath[1:], currentObj[keyPath[0]]) + elif type(currentObj) != dict: + return None + else: + return self._getValueFromDict(keyPath[1:], currentObj.get(keyPath[0])) + + def _addValueToDict(self, keyPath : list, value : object) -> dict: + if len(keyPath) == 0: + return {} + elif len(keyPath) == 1: + return { + keyPath[0]: value + } + else: + return { + keyPath[0]: self._addValueToDict(keyPath[1:], value) + } + + def extract_relevant_fields(self) -> bool: + try: + # load the string as data + dataDict = json.loads(self.data) + + newDict = {} + + # check whether the hashlink array exists + if dataDict.get("hashLink") != None and type(dataDict.get("hashLink")) == list: + for hl in dataDict.get("hashLink"): + v = self._getValueFromDict(hl.split("."), dataDict) + + if v == None: + logger.error("Hashlink array contains entries that aren't present in the JSON: %s" % hl) + + return False + + newDict.update(self._addValueToDict(hl.split("."), v)) + else: + logger.warning("No hashLink array found in data but hashlink is enabled") + + newDict = dataDict + + # write back the filtered data + self.data = json.dumps(newDict) + except Exception as e: + logger.exception(e) + + return False + + return True + def serialize_json(self) -> bool: try: # load the string as json and put it back into a string, serealizing it @@ -240,6 +313,15 @@ def run(self) -> int: else: self.data = self.input + # check if hashlink is enabled + if self.ishl: + if self.extract_relevant_fields() != True: + logger.error("Error occured while getting relevant fields from the JSON data - exiting!\n") + + self.argparser.print_usage() + + return 1 + # check if the input data is json/should be serialized if self.isjson == True: if self.serialize_json() != True: diff --git a/examples/upp-creator.py b/examples/upp-creator.py index d089b6c..2151972 100644 --- a/examples/upp-creator.py +++ b/examples/upp-creator.py @@ -19,6 +19,7 @@ DEFAULT_ISJSON = "False" DEFAULT_OUTPUT = "upp.bin" DEFAULT_NOSTDOUT = "False" +DEFAULT_ISHL = "False" logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) logger = logging.getLogger() @@ -74,6 +75,8 @@ def __init__(self): self.nostdout_str : str = None self.nostdout : bool = None self.payload : bytes = None + self.ishl : bool = None + self.ishash : bool = False self.hasher : object = None self.keystore : ubirch.KeyStore = None @@ -129,6 +132,9 @@ def setup_argparse(self): self.argparser.add_argument("--nostdout", "-n", metavar="nostdout", type=str, default=DEFAULT_NOSTDOUT, help="do not output anything to stdout; can be combined with --output /dev/stdout; e.g.: true, false (default: %s)" % DEFAULT_NOSTDOUT ) + self.argparser.add_argument("--ishl", "-l", metavar="ISHASHLINK", type=str, default=DEFAULT_ISHL, + help="implied --isjson to be true; if set to true, the script will look for a hashlink list in the json object and use it to decide which fields to hash; true or false (default: %s)" % DEFAULT_ISHL + ) def process_args(self) -> bool: # parse cli arguments (exists on err) @@ -146,6 +152,7 @@ def process_args(self) -> bool: self.keystore_pass = self.args.kspwd self.output = self.args.output self.nostdout_str = self.args.nostdout + self.ishl_str = self.args.ishl # get the keyreg value if self.keyreg_str.lower() in ["1", "yes", "y", "true"]: @@ -202,6 +209,18 @@ def process_args(self) -> bool: else: self.isjson = False + # get the bool for ishl + if self.ishl_str.lower() in ["1", "yes", "y", "true"]: + self.ishl = True + else: + self.ishl = False + + # check if ishl is true + if (self.ishl == True and self.isjson == False): + logger.warning("Overwriting '--isjson false' because --ishl is true") + + self.isjson = True + # success return True @@ -237,6 +256,61 @@ def init_proto(self) -> bool: return True + def _getValueFromDict(self, keyPath : list, currentObj : dict) -> object: + """ this function gets an object from the config object: config[path[0]][path[1]][path[n]] """ + if len(keyPath) == 0 or not currentObj: + return currentObj + elif type(currentObj) == list and type(keyPath[0]) == int: + return self._getValueFromDict(keyPath[1:], currentObj[keyPath[0]]) + elif type(currentObj) != dict: + return None + else: + return self._getValueFromDict(keyPath[1:], currentObj.get(keyPath[0])) + + def _addValueToDict(self, keyPath : list, value : object) -> dict: + if len(keyPath) == 0: + return {} + elif len(keyPath) == 1: + return { + keyPath[0]: value + } + else: + return { + keyPath[0]: self._addValueToDict(keyPath[1:], value) + } + + def extract_relevant_fields(self) -> bool: + try: + # load the string as data + dataDict = json.loads(self.data) + + newDict = {} + + # check whether the hashlink array exists + if dataDict.get("hashLink") != None and type(dataDict.get("hashLink")) == list: + for hl in dataDict.get("hashLink"): + v = self._getValueFromDict(hl.split("."), dataDict) + + if v == None: + logger.error("Hashlink array contains entries that aren't present in the JSON: %s" % hl) + + return False + + newDict.update(self._addValueToDict(hl.split("."), v)) + else: + logger.warning("No hashLink array found in data but hashlink is enabled") + + newDict = dataDict + + # write back the filtered data + self.data = json.dumps(newDict) + except Exception as e: + logger.exception(e) + + return False + + return True + def prepare_payload(self) -> bool: try: if self.hasher == None: @@ -339,6 +413,15 @@ def run(self) -> int: return 1 + # check if hashlink is enabled + if self.ishl == True and self.ishash == False: + if self.extract_relevant_fields() != True: + logger.error("Error occured while getting relevant fields from the JSON data - exiting!\n") + + self.argparser.print_usage() + + return 1 + # prepare the UPP payload if self.prepare_payload() != True: logger.error("Errors occured while preparing the UPP payload - exiting!\n") From d0d510c8d70dece41a24ffb1d91ffb6cbf0771eb Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Fri, 15 Oct 2021 11:48:34 +0200 Subject: [PATCH 29/40] print used hashing method to prevent confusion --- examples/data-verifier.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/data-verifier.py b/examples/data-verifier.py index 07b3f66..b41793e 100644 --- a/examples/data-verifier.py +++ b/examples/data-verifier.py @@ -141,6 +141,9 @@ def process_args(self) -> bool: self.isjson = True + # show the user which hashing method is used + logger.info(("Using %s as hashing algorithm" % self.hashalg.lower())) + return True def read_data(self) -> bool: From 06b1b44a76084c54ae0b88f72fac45e418b8fdbb Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Fri, 15 Oct 2021 17:55:53 +0200 Subject: [PATCH 30/40] update the readme to have some info about hashlink --- examples/EXAMPLES.md | 54 ++++++++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/examples/EXAMPLES.md b/examples/EXAMPLES.md index 5c222e1..02eea01 100644 --- a/examples/EXAMPLES.md +++ b/examples/EXAMPLES.md @@ -13,11 +13,11 @@ This file documents how to use the examples provided alongside the [uBirch-Proto - [Sending a UPP](#sending-a-upp) - [Verifying a UPP](#verifying-a-upp) - [Examining a UPP](#examining-a-upp) - - [Checking the anchoring status of a UPP](#checking-the-anchoring-status-of-an-upp) + - [Checking the anchoring status of a UPP](#checking-the-anchoring-status-of-a-upp) - [Verifying data](#verifying-data) - [Sending data to the Simple Data Service](#sending-data-to-the-simple-data-service) - [Example uBirch client implementation](#example-ubirch-client-implementation) - - [Create a hash from a JSON object](#create-a-hash-from-an-json-object) + - [Create a hash from an JSON object](#create-a-hash-from-an-json-object) - [Test identity of the device](#test-identity-of-the-device) - [Test the complete protocol](#test-the-complete-protocol) - [Test the web of trust](#test-the-web-of-trust) @@ -27,17 +27,25 @@ This file documents how to use the examples provided alongside the [uBirch-Proto ## From measurement to blockchain-anchored UPP The process needed to get a UPP to be anchored in the blockchain can be cut down into multiple steps. For each of those steps there is an example in this directory, demonstrating how to handle them. There are also examples showing a full example-client implementation. -0. [Setup](#setup) -1. [Generating and managing a keypair](#generating-and-managing-a-keypair) -2. [Registering a public key](#registering-a-public-key) -3. [Gathering Data](#gathering-data) -4. [Creating a UPP](#creating-a-upp) -5. [Sending a UPP](#sending-a-upp) -6. [Verifying a UPP](#verifying-a-upp) -7. [Examining a UPP](#examining-a-upp) -8. [Checking the anchoring status of a UPP](#checking-the-anchoring-status-of-an-upp) -9. [Verifying data](#verifying-data) +1. [Setup](#setup) + +2. [Generating and managing a keypair](#generating-and-managing-a-keypair) +3. [Registering a public key](#registering-a-public-key) + +4. [Gathering Data](#gathering-data) + +5. [Creating a UPP](#creating-a-upp) + +6. [Sending a UPP](#sending-a-upp) + +7. [Verifying a UPP](#verifying-a-upp) + +8. [Examining a UPP](#examining-a-upp) + +9. [Checking the anchoring status of a UPP](#checking-the-anchoring-status-of-a-upp) + +10. [Verifying data](#verifying-data) ### Setup Before anything, you will need to do/get a couple of things: - Choose a stage to work on @@ -339,9 +347,9 @@ Prev. UPP: "liPEEAi4JX1SUEaGhzAnqUf6pn3EQOeEqF4lo+xT9RF2ygDx9+anv14fykUolJ9gmKuT ### Verifying data In a real use case, not the UPP, but rather the original data itself has to be verified. The original data can be hashed, and the hash can be looked up by the [`data-verifier.py`](data-verifier.py) script. It has similar behaviour to the [`upp-anchoring-status.py`](upp-anchoring-status.py) script, see [Checking the anchoring status of a UPP](#checking-the-anchoring-status-of-an-upp). -``` -$ python data-verifier.py -h -usage: data-verifier.py [-h] [--ispath ISHASH] [--env ENV] [--isjson ISJSON] [--hash HASH] INPUT +```txt +$ python data-verifier.py --help +usage: data-verifier.py [-h] [--ispath ISPATH] [--env ENV] [--isjson ISJSON] [--hash HASH] [--no-send NOSEND] [--ishl ISHASHLINK] INPUT Check if the hash of given input data is known to the uBirch backend (verify it) @@ -354,18 +362,26 @@ optional arguments: sets if INPUT is being treated as data or data file path; true or false (default: False) --env ENV, -e ENV the environment to operate in; dev, demo or prod (default: dev) --isjson ISJSON, -j ISJSON - tells the script to treat the input data as json and serealize it (see EXAMPLES.md for more information); true or false (default: True) + tells the script to treat the input data as json and serealize it (see EXAMPLES.md for more information); true or false + (default: True) --hash HASH, -a HASH sets the hash algorithm to use; sha256, sha512 or OFF to treat the input data as hash (default: sha256) + --no-send NOSEND, -n NOSEND + if set to true, the script will only generate the hash of the input data without sending it; true or false (default: False) + --ishl ISHASHLINK, -l ISHASHLINK + implied --isjson to be true; if set to true, the script will look for a hashlink list in the json object and use it to + decide which fields to hash; true or false (default: False) -When --ispath/-i is set to true, the input data is treated as a file path to read the actual input data from. When setting --hash/-a to off, the input argument is expected to be a valid base64 encoded hash. +When --ispath/-i is set to true, the input data is treated as a file path to read the actual input data from. When setting --hash/-a to off, the +input argument is expected to be a valid base64 encoded hash. ``` - `--ispath/-i` Specifies wether the input is to be treated as a data-file path or direct input data. `true` or `false`. - `--env-e` The stage to check on. Should be the one the UPP corresponding to the data was sent to. `prod`, `demo` or `dev`. - `--isjson/-j` A binary flag that indicates that the input data is in JSON format. The script will serialize the JSON object before calculating the hash. This has the advantage one doesn't have to remember the order in which fields are listed in a JSON object to still be able to reconstruct the hash later on. Serializing the JSON is done like this: `json.dumps(self.data, separators=(',', ':'), sort_keys=True, ensure_ascii=False)` where `self.data` contains the JSON object which was loaded like this: `self.data = json.loads(self.dataStr)` where `dataStr` contains the input string which should represent a JSON object. This flag can have two values: `true` or `false`. It should only be set to `true` if the data represents a JSON object and if it also was serialized when creating the UPP. - `--hash/-a` Sets the hashing algorithm to use. `sha256`, `sha512` or `off`. It should match the algorithm used when creating the corresponding UPP. Setting it to `off` means that the input data actually already is the hash of the data. In this case this script will simply look up the hash. +- `--ishl/-l` enables Hashlink functionality. This means that the script will expect the input data to be a valid JSON object and to contain a list called `hashLink` at root-level. This list contains the names of all fields that should be taken into account when calculating the hash. Different JSON-levels can are represented like this: `[..., "a.b", ...]`. Example for CLI-Input data: -``` +```txt python data-verifier.py --env demo --isjson true --hash sha256 '{ "ts": 1625163338, "T": 11.2, @@ -383,7 +399,7 @@ Prev. UPP: "None" [{'label': 'PUBLIC_CHAIN', 'properties': {'timestamp': '2021-07-02T13:23:30.076Z', 'hash': '0x6e5956b4ac53bcaf58664e189673d4f8c7043488cf05009cc96868b146220604', 'public_chain': 'ETHEREUM-CLASSIC_TESTNET_ETHERERUM_CLASSIC_KOTTI_TESTNET_NETWORK', 'prev_hash': '644b41eee9043de5bda4b58bda1136fa0229712953678fe26486651338109b7a7135211f2b2cb646ab25d3b25198549e3c662c4791d60d343f08349b51ccc92b'}}] ``` Example for File-Input data: -``` +```txt python data-verifier.py --ispath true -j true data_to_verify.json -e prod 2021-07-06 12:17:31,261 root read_data() INFO Reading the input data from "data_to_verify.json" 2021-07-06 12:17:31,262 root serialize_json() INFO Serialized JSON: "{"data":{"AccPitch":"-11.52","AccRoll":"1.26","AccX":"-0.02","AccY":"0.20","AccZ":"0.99","H":"64.85","L_blue":232,"L_red":275,"P":"100934.00","T":"20.69","V":"4.62"},"msg_type":1,"timestamp":1599203876,"uuid":"07104235-1892-4020-9042-00003c94b60b"}" From ec56fad1f9b68d1087db795903fa010ebbbe9a07 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Thu, 21 Oct 2021 17:27:59 +0200 Subject: [PATCH 31/40] add ability to read hex encoded upps as input to the upp verifier --- examples/upp-verifier.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/examples/upp-verifier.py b/examples/upp-verifier.py index 3bda61b..adccb51 100644 --- a/examples/upp-verifier.py +++ b/examples/upp-verifier.py @@ -9,7 +9,8 @@ import ubirch -DEFAULT_INPUT = "upp.bin" +DEFAULT_INPUT = "/dev/stdin" +DEFAULT_ISHEX = "false" logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) @@ -61,6 +62,9 @@ def __init__(self): self.upp_uuid : uuid.UUID = None self.upp_uuid_str : str = None + self.ishex : bool = None + self.ishex_str : str = None + # initialize the argument parser self.setup_argparse() @@ -80,6 +84,9 @@ def setup_argparse(self): self.argparser.add_argument("--verifying-key-uuid", "-u", metavar="UUID", type=str, default="EMPTY", help="the UUID for the key supplied via -k (only needed when -k is specified); e.g.: 6eac4d0b-16e6-4508-8c46-22e7451ea5a1" ) + self.argparser.add_argument("--ishex", "-x", metavar="ISHEX", type=str, default=DEFAULT_ISHEX, + help="Sets whether the UPP input data is a hex string or binary; e.g. true, false (default: %s)" % DEFAULT_ISHEX + ) self.argparser.add_argument("--input", "-i", metavar="INPUT", type=str, default=DEFAULT_INPUT, help="UPP input file path; e.g. upp.bin or /dev/stdin (default: %s)" % DEFAULT_INPUT ) @@ -94,6 +101,7 @@ def process_args(self) -> bool: self.vk_str = self.args.verifying_key self.vk_uuid_str = self.args.verifying_key_uuid self.input = self.args.input + self.ishex_str = self.args.ishex # check if a verifying key was supplied if self.vk_str != "AUTO": @@ -119,6 +127,12 @@ def process_args(self) -> bool: return False + # get the ishex value + if self.ishex_str.lower() in ["1", "yes", "y", "true"]: + self.ishex = True + else: + self.ishex = False + return True def read_upp(self) -> bool: @@ -128,6 +142,10 @@ def read_upp(self) -> bool: with open(self.input, "rb") as fd: self.upp = fd.read() + + # check whether hex decoding is needed + if self.ishex == True: + self.upp = binascii.unhexlify(self.upp) except Exception as e: logger.exception(e) From f0d94e8ba63784103262e5ccf3e888dfb433ceee Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Sun, 7 Nov 2021 12:01:03 +0100 Subject: [PATCH 32/40] add hex-input support for the anchoring status script --- examples/EXAMPLES.md | 14 ++++++++++---- examples/upp-anchoring-status.py | 21 ++++++++++++++++++++- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/examples/EXAMPLES.md b/examples/EXAMPLES.md index 02eea01..7dca234 100644 --- a/examples/EXAMPLES.md +++ b/examples/EXAMPLES.md @@ -13,7 +13,7 @@ This file documents how to use the examples provided alongside the [uBirch-Proto - [Sending a UPP](#sending-a-upp) - [Verifying a UPP](#verifying-a-upp) - [Examining a UPP](#examining-a-upp) - - [Checking the anchoring status of a UPP](#checking-the-anchoring-status-of-a-upp) + - [Checking the anchoring status of an UPP](#checking-the-anchoring-status-of-an-upp) - [Verifying data](#verifying-data) - [Sending data to the Simple Data Service](#sending-data-to-the-simple-data-service) - [Example uBirch client implementation](#example-ubirch-client-implementation) @@ -287,11 +287,11 @@ $ python3 upp-unpacker.py response_upp.bin ``` _The UUID in this response UPP doesn't match the one from examples above because the UPP was sent from Niomon-Dev._ -### Checking the anchoring status of a UPP +### Checking the anchoring status of an UPP uBirch Niomon accepting the UPP doesn't mean that it is anchored yet. This process takes place in certain intervals, so one might have to wait a short while. The script to check if a UPP was already anchored is [`upp-anchoring-status.py`](upp-anchoring-status.py). ``` $ python3 upp-anchoring-status.py -h -usage: upp-anchoring-status.py [-h] [--ishash ISHASH] [--env ENV] INPUT +usage: upp-anchoring-status.py [-h] [--ishash ISHASH] [--env ENV] [--ishex ISHEX] INPUT Requests the verification/anchoring of a UPP from the uBirch backend @@ -303,11 +303,17 @@ optional arguments: --ishash ISHASH, -i ISHASH sets if INPUT is being treated as a hash or upp path; true or false (default: False) --env ENV, -e ENV the environment to operate in; dev, demo or prod (default: dev) + --ishex ISHEX, -x ISHEX + Sets whether the UPP input data is a hex string or binary; e.g. true, false (default: false) -When --ishash/-i is set to true, the input argument is treated as a base64 payload hash. Otherwise, it is expected to be some kind of path to read a UPP from. This can be a file path or also /dev/stdin if the UPP is piped to this program via standard input. +When --ishash/-i is set to true, the input argument is treated as a base64 payload hash. Otherwise, it is expected to be some kind of path to read a +UPP from. This can be a file path or also /dev/stdin if the UPP is piped to this program via standard input. ``` - `--ishash/-i` A boolean specifying whether the input data is a payload hash or a UPP. The payload hash is what is actually used to look up anchoring information about the UPP. This script can either extract it from a given UPP or just use the hash directly if provided. If directly provided, it must be base64 encoded (see the last example of this sub-section). `true` or `false`. - `--env/-e` The stage to check on. Should be the one the UPP was sent to. `prod`, `demo` or `dev`. +- `--ishex/-x` A boolean which controls how the input UPP data is interpreted. By default, the data will +be interpreted as normale binary data. When this flag is set to `true`, it will be considered +hex-encoded binary data and de-hexlified before parsing it. - `INPUT` The input UPP file path or payload hash, depending on `--ishash`. One example might be: diff --git a/examples/upp-anchoring-status.py b/examples/upp-anchoring-status.py index 6234dbf..df0f5dc 100644 --- a/examples/upp-anchoring-status.py +++ b/examples/upp-anchoring-status.py @@ -7,12 +7,14 @@ import base64 import json import requests +import binascii VERIFICATION_SERVICE = "https://verify.%s.ubirch.com/api/upp/verify/anchor" DEFAULT_ISHASH = "False" DEFAULT_ENV = "dev" +DEFAULT_ISHEX = "false" logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) @@ -31,6 +33,9 @@ def __init__(self): self.upp : bytes = None self.hash : str = None + + self.ishex : bool = None + self.ishex_str : str = None # initialize the argument parser self.setup_argparse() @@ -54,6 +59,9 @@ def setup_argparse(self): self.argparser.add_argument("--env", "-e", metavar="ENV", type=str, default=DEFAULT_ENV, help="the environment to operate in; dev, demo or prod (default: %s)" % DEFAULT_ENV ) + self.argparser.add_argument("--ishex", "-x", metavar="ISHEX", type=str, default=DEFAULT_ISHEX, + help="Sets whether the UPP input data is a hex string or binary; e.g. true, false (default: %s)" % DEFAULT_ISHEX + ) def process_args(self) -> bool: # parse cli arguments (exists on err) @@ -63,13 +71,20 @@ def process_args(self) -> bool: self.input = self.args.input self.ishash_str = self.args.ishash self.env = self.args.env + self.ishex_str = self.args.ishex # get the bool for ishash if self.ishash_str.lower() in ["1", "yes", "y", "true"]: self.ishash = True else: self.ishash = False - + + # get the ishex value + if self.ishex_str.lower() in ["1", "yes", "y", "true"]: + self.ishex = True + else: + self.ishex = False + return True def read_upp(self) -> bool: @@ -79,6 +94,10 @@ def read_upp(self) -> bool: with open(self.input, "rb") as fd: self.upp = fd.read() + + # check whether hex decoding is needed + if self.ishex == True: + self.upp = binascii.unhexlify(self.upp) except Exception as e: logger.exception(e) From ede1d65182a35b570b284af1277da3b9102e08e6 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Sun, 7 Nov 2021 15:46:44 +0100 Subject: [PATCH 33/40] add a script to check chains of upps (upp-chaining) --- examples/upp-chain-checker.py | 316 ++++++++++++++++++++++++++++++++++ 1 file changed, 316 insertions(+) create mode 100644 examples/upp-chain-checker.py diff --git a/examples/upp-chain-checker.py b/examples/upp-chain-checker.py new file mode 100644 index 0000000..b692124 --- /dev/null +++ b/examples/upp-chain-checker.py @@ -0,0 +1,316 @@ +import sys +import logging +import argparse +import msgpack +import binascii +import uuid +import ed25519 +import json + +import ubirch +from ubirch.ubirch_protocol import UNPACKED_UPP_FIELD_UUID, UNPACKED_UPP_FIELD_PREV_SIG, UNPACKED_UPP_FIELD_SIG + + +DEFAULT_ISJSON = "false" +DEFAULT_ISHEX = "false" + + +logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) +logger = logging.getLogger() + + +class Proto(ubirch.Protocol): + def __init__(self, ks : ubirch.KeyStore): + super().__init__() + + self.ks : ubirch.KeyStore = ks + + def _verify(self, uuid: uuid.UUID, message: bytes, signature: bytes): + return self.ks.find_verifying_key(uuid).verify(signature, message) + + +class Main: + def __init__(self): + self.argparser : argparse.ArgumentParser = None + self.args : argparse.Namespace = None + + self.vk_str : str = None + self.vk : ed25519.VerifyingKey = None + self.vk_uuid : uuid.UUID = None + self.vk_uuid_str : str = None + + self.input : str = None + + self.keystore : ubirch.KeyStore = None + self.proto : ubirch.Protocol = None + + self.upps : [bytes] = None + self.upp_uuid : uuid.UUID = None + self.upp_uuid_str : str = None + + self.isjson : bool = None + self.isjson_str : str = None + + # initialize the argument parser + self.setup_argparse() + + return + + def setup_argparse(self): + self.argparser = argparse.ArgumentParser( + description="Check if a sequence of chained UPPs is valid/properly signed and correctly chained", + epilog="The JSON file (when using --is-json true) es expected to contain a single field called \"upps\", which is a list of " + "hex-encoded UPPs. Otherwise (--is-json false). If --is-hex is true, it expects a sequence of hex-encoded UPPs " + "separated by newlines. The third (default) scenario is that the script expects a sequence of binary UPPs separated " + "by newlines.\n\nIf --is-json true is set, --is-hex will be ignored." + ) + + self.argparser.add_argument("inputfile", metavar="INPUTFILE", type=str, + help="Input file path; e.g. upp_list.bin, upp_list.json or /dev/stdin" + ) + self.argparser.add_argument("verifying_key", metavar="VK", type=str, + help="key to be used for verification; any verifying key in hex like \"b12a906051f102881bbb487ee8264aa05d8d0fcc51218f2a47f562ceb9b0d068\"" + ) + self.argparser.add_argument("verifying_key_uuid", metavar="UUID", type=str, + help="the UUID for the verifying key; e.g.: 6eac4d0b-16e6-4508-8c46-22e7451ea5a1" + ) + self.argparser.add_argument("--is-json", "-j", metavar="ISJSON", type=str, default=DEFAULT_ISJSON, + help="If true, the script expects a JSON file for INPUTFILE (see below); e.g. true, false (default: %s)" % DEFAULT_ISHEX + ) + self.argparser.add_argument("--is-hex", "-x", metavar="ISHEX", type=str, default=DEFAULT_ISHEX, + help="If true, the script hex-encoded UPPs from the input file; e.g. true, false (default: %s)" % DEFAULT_ISHEX + ) + + return + + def process_args(self) -> bool: + # parse cli arguments (exists on err) + self.args = self.argparser.parse_args() + + # get all needed args + self.vk_str = self.args.verifying_key + self.vk_uuid_str = self.args.verifying_key_uuid + self.input = self.args.inputfile + self.ishex_str = self.args.is_hex + self.isjson_str = self.args.is_json + + # check the VK + try: + # convert the string + self.vk = ed25519.VerifyingKey(self.vk_str, encoding="hex") + except Exception as e: + logger.error("Invalid verifying key: \"%s\"" % self.vk_str) + logger.exception(e) + + return False + + # check the UUID + try: + self.vk_uuid = uuid.UUID(hex=self.vk_uuid_str) + except Exception as e: + logger.error("Invalid UUID: \"%s\"" % self.vk_uuid_str) + logger.exception(e) + + return False + + # get the ishex value + if self.ishex_str.lower() in ["1", "yes", "y", "true"]: + self.ishex = True + else: + self.ishex = False + + # get the ishex value + if self.isjson_str.lower() in ["1", "yes", "y", "true"]: + self.isjson = True + else: + self.isjson = False + + return True + + def read_upps(self) -> bool: + # check whether json is enabled or not + if self.isjson == True: + try: + # read the json file and get the upps from the contained list + logger.info("Reading the input UPP json from \"%s\"" % self.input) + + with open(self.input, "rb") as fd: + input_json : dict = json.load(fd) + + # get the upps field + upps_hex = input_json.get("upps") + + if upps_hex == None or type(upps_hex) != list: + raise Exception("input json must contain a \"upps\" filed which must be a list of hex-encoded UPPs!") + + # decode the hex-upps + self.upps = list(map(lambda x: binascii.unhexlify(x), upps_hex)) + except Exception as e: + logger.exception(e) + + return False + else: + # read the UPP from the input path + try: + logger.info("Reading the input UPPs from \"%s\"" % self.input) + + with open(self.input, "rb") as fd: + upp_list_raw = fd.read().splitlines() + + # check whether hex decoding is needed + if self.ishex == True: + self.upps = list(map(lambda x: binascii.unhexlify(x), upp_list_raw)) + else: + self.upps = upp_list_raw + except Exception as e: + logger.exception(e) + + return False + + logger.info("Read %d UPPs" % len(self.upps)) + + return True + + def init_keystore(self) -> bool: + try: + self.keystore = ubirch.KeyStore("-- temporary --", None) + except Exception as e: + logger.exception(e) + + return False + + return True + + def check_cli_vk(self) -> bool: + try: + if self.vk != None: + self.keystore.insert_ed25519_verifying_key(self.vk_uuid, self.vk) + + logger.info("Inserted \"%s\": \"%s\" (UUID/VK) into the keystore" % (self.vk_uuid_str, self.vk_str)) + except Exception as e: + logger.exception(e) + + return False + + return True + + def init_proto(self) -> bool: + try: + self.proto = Proto(self.keystore) + except Exception as e: + logger.exception(e) + + return False + + return True + + def verify_upps(self) -> bool: + # store the signature of the last checked upp + prev_sig = None + + try: + for i in range(0, len(self.upps)): + # check the signature + if self.proto.verfiy_signature(self.vk_uuid, self.upps[i]) == False: + raise Exception("The signature cannot be verified with the given VK - the UPP is invalid - Aborting at UPP %d" % (i + 1)) + + + + # unpack the upp + upp_unpacked = self.proto.unpack_upp(self.upps[i]) + + # check whether the UUID matches with the given vk_uuid + uuid_index = self.proto.get_unpacked_index(upp_unpacked[0], UNPACKED_UPP_FIELD_UUID) + + if upp_unpacked[uuid_index] != self.vk_uuid.bytes: + raise Exception("The UUID contained in UPP %s doesn't match the VK-UUID (%s vs. %s) - Aborting at UPP %d" % + (i + 1, uuid.UUID(bytes=upp_unpacked[uuid_index]), self.vk_uuid_str, i + 1) + ) + + # check whether a prevsig check should be done + if prev_sig != None: + # get the index of the previous signature + prevsig_index = self.proto.get_unpacked_index(upp_unpacked[0], UNPACKED_UPP_FIELD_PREV_SIG) + + # check the return value - -1 means, that the UPP is not chained/doesn't contain a prevsig + if prevsig_index == -1: + raise Exception("UPP %d is NOT a chained UPP/doesn't contain a prevsig - Aborting at UPP %d" % (i + 1, i + 1)) + + # compare the signatures + if prev_sig != upp_unpacked[prevsig_index]: + raise Exception("The prevsig of UPP %d doesn't match the sig of UPP %d - Aborting at UPP %d" % (i + 1, i, i + 1)) + + # set the new prevsig + sig_index = self.proto.get_unpacked_index(upp_unpacked[0], UNPACKED_UPP_FIELD_SIG) + + if sig_index == -1: + raise Exception("UPP %d is NEITHER chained NOR signed/doesn't contain a signature - Aborting at UPP %d" % (i + 1, i + 1)) + + prev_sig = upp_unpacked[sig_index] + + logger.info("All signatures verified and prevsigs compared - the UPP chain is valid!") + except KeyError: + logger.error("No verifying key found for UUID \"%s\" - can't verify the UPP!" % self.upp_uuid_str) + + return False + except Exception as e: + logger.exception(e) + + return False + + return True + + def run(self): + # process all args + if self.process_args() != True: + logger.error("Errors occured during argument processing - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # read the upp + if self.read_upps() != True: + logger.error("Errors occured while reading the UPPs from \"%s\" - exiting!\n" % self.input) + + self.argparser.print_usage() + + return 1 + + # initialize the keystore + if self.init_keystore() != True: + logger.error("Errors occured while initializing the keystore - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # check/insert the cli-provided verifying key + if self.check_cli_vk() != True: + logger.error("Errorc occured while inserting the verifying key into the keystore - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # initialize the Protocol + if self.init_proto() != True: + logger.error("Erros occured while initializing the Protocol - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # try to verify the message + if self.verify_upps() != True: + logger.error("Errors occured while verifying the UPPs - exiting!\n") + + self.argparser.print_usage() + + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(Main().run()) \ No newline at end of file From b33e3a68c70db00464540b896ae3e6f322dcec71 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Sun, 7 Nov 2021 15:47:15 +0100 Subject: [PATCH 34/40] add docs for the chain-checker script --- examples/EXAMPLES.md | 122 +++++++++++++++++++++++++++++++++---------- 1 file changed, 93 insertions(+), 29 deletions(-) diff --git a/examples/EXAMPLES.md b/examples/EXAMPLES.md index 7dca234..5c06c50 100644 --- a/examples/EXAMPLES.md +++ b/examples/EXAMPLES.md @@ -9,10 +9,11 @@ This file documents how to use the examples provided alongside the [uBirch-Proto - [Generating and managing a keypair](#generating-and-managing-a-keypair) - [Registering a public key](#registering-a-public-key) - [Gathering Data](#gathering-data) - - [Creating a UPP](#creating-a-upp) - - [Sending a UPP](#sending-a-upp) - - [Verifying a UPP](#verifying-a-upp) - - [Examining a UPP](#examining-a-upp) + - [Creating an UPP](#creating-an-upp) + - [Sending an UPP](#sending-an-upp) + - [Verifying an UPP](#verifying-an-upp) + - [Verifying an UPP chain](#verifying-an-upp-chain) + - [Examining an UPP](#examining-an-upp) - [Checking the anchoring status of an UPP](#checking-the-anchoring-status-of-an-upp) - [Verifying data](#verifying-data) - [Sending data to the Simple Data Service](#sending-data-to-the-simple-data-service) @@ -25,7 +26,7 @@ This file documents how to use the examples provided alongside the [uBirch-Proto - [Verify ED25519 signed UPP](#verify-ed25519-signed-upp) ## From measurement to blockchain-anchored UPP -The process needed to get a UPP to be anchored in the blockchain can be cut down into multiple steps. For each of those steps there is an example in this directory, demonstrating how to handle them. There are also examples showing a full example-client implementation. +The process needed to get an UPP to be anchored in the blockchain can be cut down into multiple steps. For each of those steps there is an example in this directory, demonstrating how to handle them. There are also examples showing a full example-client implementation. 1. [Setup](#setup) @@ -35,15 +36,15 @@ The process needed to get a UPP to be anchored in the blockchain can be cut down 4. [Gathering Data](#gathering-data) -5. [Creating a UPP](#creating-a-upp) +5. [Creating an UPP](#creating-an-upp) -6. [Sending a UPP](#sending-a-upp) +6. [Sending an UPP](#sending-an-upp) -7. [Verifying a UPP](#verifying-a-upp) +7. [Verifying an UPP](#verifying-an-upp) -8. [Examining a UPP](#examining-a-upp) +8. [Examining an UPP](#examining-an-upp) -9. [Checking the anchoring status of a UPP](#checking-the-anchoring-status-of-a-upp) +9. [Checking the anchoring status of an UPP](#checking-the-anchoring-status-of-a-upp) 10. [Verifying data](#verifying-data) ### Setup @@ -65,7 +66,7 @@ The values used below are `f5ded8a3-d462-41c4-a8dc-af3fd072a217` for the UUID, ` `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` for the auth token. ### Generating and managing a keypair -To create, or more precisely, to _sign_ a UPP, a device will need a keypair. This keypair consist of a private key (_signing key_) and public key (_verifying key_). The signing key is used to sign UPPs and the verifying key can be used by the uBirch backend to check, if the signature is valid and belongs to the correct sender/signer. So, logically it doesn't matter who knows to verifying key, but the signing key must be kept secret all the time. In a real use case a device might store it in a TPM ([Trusted platform module](https://en.wikipedia.org/wiki/Trusted_Platform_Module)) or use other counter measures against attackers reading the key from the device. For this demo, keypairs will be stored in a [JKS Keystore](https://en.wikipedia.org/wiki/Java_KeyStore) using the [`pyjks`](https://pypi.org/project/pyjks/) library. Therefore, you will have to choose and remember a file path for that keystore and a password used to encrypt it. The process of actually generating the keypair is handled by the [upp-creator.py](upp-creator.py) script and explained [below](#registering-a-public-key). +To create, or more precisely, to _sign_ an UPP, a device will need a keypair. This keypair consist of a private key (_signing key_) and public key (_verifying key_). The signing key is used to sign UPPs and the verifying key can be used by the uBirch backend to check, if the signature is valid and belongs to the correct sender/signer. So, logically it doesn't matter who knows to verifying key, but the signing key must be kept secret all the time. In a real use case a device might store it in a TPM ([Trusted platform module](https://en.wikipedia.org/wiki/Trusted_Platform_Module)) or use other counter measures against attackers reading the key from the device. For this demo, keypairs will be stored in a [JKS Keystore](https://en.wikipedia.org/wiki/Java_KeyStore) using the [`pyjks`](https://pypi.org/project/pyjks/) library. Therefore, you will have to choose and remember a file path for that keystore and a password used to encrypt it. The process of actually generating the keypair is handled by the [upp-creator.py](upp-creator.py) script and explained [below](#registering-a-public-key). **NOTE** that losing access to the signing key, especially if it is already registered at the uBirch backend, will take away the ability to create and send any new UPPs from that device/UUID, since there is no way of creating a valid signature that would be accepted by the backend. @@ -89,12 +90,12 @@ optional arguments: By default, only UUIDs und public keys (verifying keys) will be displayed. Displaying of private keys (signing keys) can be enabled by passing `-s true`. ### Registering a public key -To enable the uBirch backend to verify a UPP, it needs to know the corresponding verifying key. Therefore, the device needs to send this key to the backend, before starting to send UPPs, which are supposed to be verified and anchored. Registering a verifying key is done by sending a special kind of UPP containing this key. This can be done by using two scripts: +To enable the uBirch backend to verify an UPP, it needs to know the corresponding verifying key. Therefore, the device needs to send this key to the backend, before starting to send UPPs, which are supposed to be verified and anchored. Registering a verifying key is done by sending a special kind of UPP containing this key. This can be done by using two scripts: ``` upp-creator.py upp-sender.py ``` -Both of these scripts will be explained in more detail in [Creating a UPP](#creating-a-upp) and [Sending a UPP](#sending-a-upp). To generate a _Public Key Registration UPP_ this command can be used: +Both of these scripts will be explained in more detail in [Creating an UPP](#creating-a-upp) and [Sending an UPP](#sending-a-upp). To generate a _Public Key Registration UPP_ this command can be used: ``` $ python upp-creator.py -t 1 --ks devices.jks --kspwd keystore --keyreg true --output keyreg_upp.bin f5ded8a3-d462-41c4-a8dc-af3fd072a217 none @@ -139,8 +140,8 @@ serialized = json.dumps(message, separators=(',', ':'), sort_keys=True, ensure_a This will create a string from the above examplary JSON object: `{"H":35.8,"S":"OK","T":11.2,"ts":1625163338}` -### Creating a UPP -After gathering some measurement data a UPP can be created. The UPP won't contain the actual measurement data, but a hash of it. The example script to create UPPs is [`upp-creator.py`](upp-creator.py). +### Creating an UPP +After gathering some measurement data an UPP can be created. The UPP won't contain the actual measurement data, but a hash of it. The example script to create UPPs is [`upp-creator.py`](upp-creator.py). ``` $ python3 upp-creator.py --help @@ -170,12 +171,12 @@ optional arguments: --nostdout nostdout, -n nostdout do not output anything to stdout; can be combined with --output /dev/stdout; e.g.: true, false (default: False) -Note that, when using chained UPPs (--version 0x23), this tool will try to load/save signatures from/to .sig, where UUID will be replaced with the actual UUID. Make sure that the UUID.sig file is in your current working directory if you try to continue a UPP chain +Note that, when using chained UPPs (--version 0x23), this tool will try to load/save signatures from/to .sig, where UUID will be replaced with the actual UUID. Make sure that the UUID.sig file is in your current working directory if you try to continue an UPP chain using this tool.Also beware that you will only be able to access the contents of a keystore when you use the same password you used when creating it. Otherwise all contents are lost. When --hash off is set, contents of the DATA argument will be copied into the payload field of the UPP. Normally used for special messages (e.g. key registration). For more information on possible values for --type and --version see https://github.com/ubirch/ubirch-protocol. ``` The script allows multiple modes of operation, which can be set through different command line arguments. Some of those directly set fields in the resulting UPP. Please consult the [uBirch Protocol Readme](https://github.com/ubirch/ubirch-protocol#basic-message-format) for further information on those fields and their possible values. -- `--version/-v` This flag sets the version field of the UPP. The version field actually consists of two sub-fields. The higher four bits set the actual version (`1` or `2`) and the "mode". The higher four bits will be set to two(`0010`) in almost all use cases. The mode can either be a simple UPP without a signature, a UPP with a signature and a UPP with a signature + the signature of the previous UPP embedded into it. The latter would be called a_Chained UPP_. Unsigned UPPs (`-v 0x21`) are not implemented. Signed UPPs have `-v 0x22` and chained ones `-v 0x23`. +- `--version/-v` This flag sets the version field of the UPP. The version field actually consists of two sub-fields. The higher four bits set the actual version (`1` or `2`) and the "mode". The higher four bits will be set to two(`0010`) in almost all use cases. The mode can either be a simple UPP without a signature, an UPP with a signature and an UPP with a signature + the signature of the previous UPP embedded into it. The latter would be called a_Chained UPP_. Unsigned UPPs (`-v 0x21`) are not implemented. Signed UPPs have `-v 0x22` and chained ones `-v 0x23`. - `--type/-t` This flag sets the type field of the UPP. It is used to indicate what the UPP contains/should be used for. It can be set to `0x00` in most cases. One of the cases where a specific value is required, is a Key Registration Messages, as described in [Registering a Public Key](#registering-a-public-key). - `--k/-k` The path to the keystore that contains the keypair for the device or should be used to store a newly generated keypair. If the keystore, pointed to by this parameter, doesn't exist, the script will simply create it. - `--kspwd/-p` The password to decrypt/encrypt the keystore. You must remember this, or you will lose access to the keystore and all its contents. @@ -204,7 +205,7 @@ $ python3 upp-creator.py --version 0x23 --isjson true --output upp.bin --hash sh ``` Keep in mind that if you use chained UPPs (`--version 0x23`) you should anchor each UPP, or the signature chain will be broken. This won't cause any errors, but the advantage of chaining UPPs and thereby knowing the correct order of them, will get lost. -### Sending a UPP +### Sending an UPP After creating the UPP, it can be sent to the uBirch backend where it will be verified and anchored in the blockchain. The ubirch backend will use the registered public/verifying key to check the signature. The [`upp-sender.py`](upp-sender.py) script can be used for that. ``` $ python3 upp-sender.py --help @@ -230,7 +231,7 @@ For this script the parameters are: - `--output/-o` Normally the uBirch backend will respond to the UPP with another UPP. This parameter sets the location to write that response-UPP to. - `UUID` The UUID of the device that generated the UPP as a hex-string. - `AUTH` The auth token for the device on the specified stage as a hex-string. -Continuing from the example above (see [Creating a UPP](#creating-a-upp)), the send-command might look like this: +Continuing from the example above (see [Creating an UPP](#creating-a-upp)), the send-command might look like this: ``` $ python upp-sender.py --env demo --input upp.bin --output response_upp.bin f5ded8a3-d462-41c4-a8dc-af3fd072a217 xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx 2021-07-02 15:21:36,966 root read_upp() INFO Reading the input UPP for "f5ded8a3-d462-41c4-a8dc-af3fd072a217" from "upp.bin" @@ -239,14 +240,14 @@ $ python upp-sender.py --env demo --input upp.bin --output response_upp.bin f5de 2021-07-02 15:21:37,723 root store_response_upp() INFO The response UPP has been written to "response_upp.bin" ``` -### Verifying a UPP +### Verifying an UPP To make sure, that the response UPP actually was sent by the uBirch backend, its signature can be checked. The example script for that is [`upp-verifier.py`](upp-verifier.py). It knows the UUID and verifying/public key for each uBirch Niomon stage end checks, if the signature of the response UPP is valid. ``` $ python3 upp-verifier.py --help usage: upp-verifier.py [-h] [--verifying-key VK] [--verifying-key-uuid UUID] [--input INPUT] -Check if a UPP is valid/properly signed +Check if an UPP is valid/properly signed optional arguments: -h, --help show this help message and exit @@ -257,9 +258,9 @@ optional arguments: --input INPUT, -i INPUT UPP input file path; e.g. upp.bin or /dev/stdin (default: upp.bin) -Note that, when trying to verify a UPP, sent by the uBirch backend (Niomon), a verifying key doesn't have to be provided via the -k option. Instead, this script will try to pick the correct stage key based on the UUID which is contained in the UPP, identifying the creator. If the UUID doesn't match any Niomon stage and no key was specified using -k, an error message will be printed. +Note that, when trying to verify an UPP, sent by the uBirch backend (Niomon), a verifying key doesn't have to be provided via the -k option. Instead, this script will try to pick the correct stage key based on the UUID which is contained in the UPP, identifying the creator. If the UUID doesn't match any Niomon stage and no key was specified using -k, an error message will be printed. ``` -- `--verifying-key/-k` If not trying to verify a UPP coming from uBirch Niomon but from another source, the verifying key for that source needs to be provided. This parameter expects the key as a hex-string like `b12a906051f102881bbb487ee8264aa05d8d0fcc51218f2a47f562ceb9b0d068`. +- `--verifying-key/-k` If not trying to verify an UPP coming from uBirch Niomon but from another source, the verifying key for that source needs to be provided. This parameter expects the key as a hex-string like `b12a906051f102881bbb487ee8264aa05d8d0fcc51218f2a47f562ceb9b0d068`. - `--verifying-key-uuid/-u` The UUID for the verifying key from `--verifying-key`. This parameter will be ignored when `--verifying-key` is not set. Not setting this parameter when `--verifying-key` is set will cause an error. - `--input/-i` The file path to read the UPP from. @@ -271,8 +272,71 @@ $ python3 upp-verifier.py --input response_upp.bin 2021-07-02 15:43:36,275 root verify_upp() INFO Signature verified - the UPP is valid! ``` -### Examining a UPP -To examine the contents (Version Field, Type Field, UUID, Signature, Payload, Previous Signature) of a UPP, the [`upp-unpacker.py`](upp-unpacker.py) script can be used like this: +### Verifying an UPP chain +When working with `chained UPPs` it can be useful to check whether the chain is in order and valid. For +this task, the [`upp-chain-checker.py`](upp-chain-checke.py) can be used. It reads in a list of UPPs, +checks the signature of each UPP and compares the `prevsig` field with the `signature` of the last UPP. +If at any point something doesn't match up, it will print an error message alonge with the number of the +UPP at which the chain broke/something went wrong. The UPP list can either be read directly from a file +which contains them in binary or hex-encoded, separated by newlines, or from a JSON file which contains +a list of hex-encoded UPPs. +``` +$ python3 upp-chain-checker.py -h +usage: upp-chain-checker.py [-h] [--is-json ISJSON] [--is-hex ISHEX] INPUTFILE VK UUID + +Check if a sequence of chained UPPs is valid/properly signed and correctly chained + +positional arguments: + INPUTFILE Input file path; e.g. upp_list.bin, upp_list.json or /dev/stdin + VK key to be used for verification; any verifying key in hex like + "b12a906051f102881bbb487ee8264aa05d8d0fcc51218f2a47f562ceb9b0d068" + UUID the UUID for the verifying key; e.g.: 6eac4d0b-16e6-4508-8c46-22e7451ea5a1 + +optional arguments: + -h, --help show this help message and exit + --is-json ISJSON, -j ISJSON + If true, the script expects a JSON file for INPUTFILE (see below); e.g. true, false (default: false) + --is-hex ISHEX, -x ISHEX + If true, the script hex-encoded UPPs from the input file; e.g. true, false (default: false) + +The JSON file (when using --is-json true) es expected to contain a single field called "upps", which is a list of hex-encoded UPPs. Otherwise (--is-json false). If --is-hex is true, it expects a sequence of hex-encoded UPPs separated by newlines. The third (default) scenario is that the script expects a sequence of binary UPPs separated by newlines. If --is-json true is set, --is-hex will be ignored. +``` +The `VK` and `UUID` arguments work like in the [UPP-Verifier](#verifying-an-upp), with the difference +that they aren't optional and must be provided. Here is an example JSON file containing four UPPs +```json +{ + "upps": [ + "9623c410ee8c4cfe9b3a43e29e9f8875cb02cec3c44025152e18f42352a7fba90b3fa30fc245587c8af1e681a77a25107137926a9ce88287e804b7989f60d9d9ea5673bc1531437fe147281b18071ac0adbe40d27d0b00c440f30a0ee67fc6f5ae5a133a012ab3931198752ee8e13084d473c1d1bd7dd000423b5ede36e5c217a2b8fe0512c5bfb3e8959f6773b812ddf98e45895ee9a7ac06c4406502e436d33edbfa8c1f82f9644344307e79dfd46c2a766083a238bfd6edca2ec6d83b2329a5b302516839bfac36b199c7593dded5bc4f0531f233ce53f94903", + "9623c410ee8c4cfe9b3a43e29e9f8875cb02cec3c4406502e436d33edbfa8c1f82f9644344307e79dfd46c2a766083a238bfd6edca2ec6d83b2329a5b302516839bfac36b199c7593dded5bc4f0531f233ce53f9490300c440a27146b7aa7bc0194468a1e5eee816dd07861bd4036654b74812f36e721b98615aedb84bb8700b5aede01207994c20b1bac759da95a3b41f4614c975a0668883c440596c4b7b840681ce89bb1d6dbb2ccf1108e2007a68ed39fce71783c6d1e8b39ba78769866bacbc281a64d8f7d9ff20fd5dc6a1cf998104395e2018ad49a15a08", + "9623c410ee8c4cfe9b3a43e29e9f8875cb02cec3c440596c4b7b840681ce89bb1d6dbb2ccf1108e2007a68ed39fce71783c6d1e8b39ba78769866bacbc281a64d8f7d9ff20fd5dc6a1cf998104395e2018ad49a15a0800c44038b971a62ce01cbbf302cb635c4c6f2faa266a5d78aa1edbda28ac8945ed51ac651b3fac2aa85b1d1685cf4424b7fbb1845a09e47b9ce69b957ceff2bcddf61dc4409c2f20ece86519f541b45b4e2aea4ea51b98c3d12014e513c303c8c9b0af7c0caab39894419dac6e4bf601c27273f9bc58c22ab9e93879fc472f381da00c1d03", + "9623c410ee8c4cfe9b3a43e29e9f8875cb02cec3c4409c2f20ece86519f541b45b4e2aea4ea51b98c3d12014e513c303c8c9b0af7c0caab39894419dac6e4bf601c27273f9bc58c22ab9e93879fc472f381da00c1d0300c440553470115df2e2bc5d1044fa3cec93f95c9e0d9df2daa394daca465d75e3dc91d34c6cfa0d7b29081f0dd58d79deae541e890d6ef04f6cf4a32031a8e855d93bc44040f79e9feb28e4086489431b5650b74849308f5a1911f3d630711e226eef03a0b48185964b753a63e44b36d5a9794f5f3df2af0e613545c063b81c7005f9d400" + ] +} +``` +If stored in `UPP_LIST.json`, it can be used like this: +``` +$ python3 upp-chain-checker.py --is-json true UPP_LIST.json 286401c523ebbfb5f6a4044e62af8ef66775f9a76a2ff2af0067ecfb4563df21 ee8c4cfe-9b3a-43e2-9e9f-8875cb02cec3 +2021-11-07 15:40:58,168 root read_upps() INFO Reading the input UPP json from "UPP_LIST.json" +2021-11-07 15:40:58,168 root read_upps() INFO Read 4 UPPs +2021-11-07 15:40:58,168 ubirch.ubirch_ks _load_keys() WARNING creating new key store: -- temporary -- +2021-11-07 15:40:58,168 root check_cli_vk() INFO Inserted "ee8c4cfe-9b3a-43e2-9e9f-8875cb02cec3": "286401c523ebbfb5f6a4044e62af8ef66775f9a76a2ff2af0067ecfb4563df21" (UUID/VK) into the keystore +2021-11-07 15:40:58,173 root verify_upps() INFO All signatures verified and prevsigs compared - the UPP chain is valid! +``` +Another way of using the script is this: +``` +$ echo -n -e "9623c410ee8c4cfe9b3a43e29e9f8875cb02cec3c44025152e18f42352a7fba90b3fa30fc245587c8af1e681a77a25107137926a9ce88287e804b7989f60d9d9ea5673bc1531437fe147281b18071ac0adbe40d27d0b00c440f30a0ee67fc6f5ae5a133a012ab3931198752ee8e13084d473c1d1bd7dd000423b5ede36e5c217a2b8fe0512c5bfb3e8959f6773b812ddf98e45895ee9a7ac06c4406502e436d33edbfa8c1f82f9644344307e79dfd46c2a766083a238bfd6edca2ec6d83b2329a5b302516839bfac36b199c7593dded5bc4f0531f233ce53f94903\n9623c410ee8c4cfe9b3a43e29e9f8875cb02cec3c4406502e436d33edbfa8c1f82f9644344307e79dfd46c2a766083a238bfd6edca2ec6d83b2329a5b302516839bfac36b199c7593dded5bc4f0531f233ce53f9490300c440a27146b7aa7bc0194468a1e5eee816dd07861bd4036654b74812f36e721b98615aedb84bb8700b5aede01207994c20b1bac759da95a3b41f4614c975a0668883c440596c4b7b840681ce89bb1d6dbb2ccf1108e2007a68ed39fce71783c6d1e8b39ba78769866bacbc281a64d8f7d9ff20fd5dc6a1cf998104395e2018ad49a15a08\n9623c410ee8c4cfe9b3a43e29e9f8875cb02cec3c440596c4b7b840681ce89bb1d6dbb2ccf1108e2007a68ed39fce71783c6d1e8b39ba78769866bacbc281a64d8f7d9ff20fd5dc6a1cf998104395e2018ad49a15a0800c44038b971a62ce01cbbf302cb635c4c6f2faa266a5d78aa1edbda28ac8945ed51ac651b3fac2aa85b1d1685cf4424b7fbb1845a09e47b9ce69b957ceff2bcddf61dc4409c2f20ece86519f541b45b4e2aea4ea51b98c3d12014e513c303c8c9b0af7c0caab39894419dac6e4bf601c27273f9bc58c22ab9e93879fc472f381da00c1d03\n9623c410ee8c4cfe9b3a43e29e9f8875cb02cec3c4409c2f20ece86519f541b45b4e2aea4ea51b98c3d12014e513c303c8c9b0af7c0caab39894419dac6e4bf601c27273f9bc58c22ab9e93879fc472f381da00c1d0300c440553470115df2e2bc5d1044fa3cec93f95c9e0d9df2daa394daca465d75e3dc91d34c6cfa0d7b29081f0dd58d79deae541e890d6ef04f6cf4a32031a8e855d93bc44040f79e9feb28e4086489431b5650b74849308f5a1911f3d630711e226eef03a0b48185964b753a63e44b36d5a9794f5f3df2af0e613545c063b81c7005f9d400\n" | python3 upp-chain-checker.py --is-hex true /dev/stdin 286401c523ebbfb5f6a4044e62af8ef66775f9a76a2ff2af0067ecfb4563df21 ee8c4cfe-9b3a-43e2-9e9f-8875cb02cec3 + +2021-11-07 15:45:00,121 root read_upps() INFO Reading the input UPPs from "/dev/stdin" +2021-11-07 15:45:00,121 root read_upps() INFO Read 4 UPPs +2021-11-07 15:45:00,121 ubirch.ubirch_ks _load_keys() WARNING creating new key store: -- temporary -- +2021-11-07 15:45:00,121 root check_cli_vk() INFO Inserted "ee8c4cfe-9b3a-43e2-9e9f-8875cb02cec3": "286401c523ebbfb5f6a4044e62af8ef66775f9a76a2ff2af0067ecfb4563df21" (UUID/VK) into the keystore +2021-11-07 15:45:00,127 root verify_upps() INFO All signatures verified and prevsigs compared - the UPP chain is valid! +``` +The UPPs are piped as hex-encoded strings separated by newlines (`\n`) to the script which has the input +file path set to `/dev/stdin`. + +### Examining an UPP +To examine the contents (Version Field, Type Field, UUID, Signature, Payload, Previous Signature) of an UPP, the [`upp-unpacker.py`](upp-unpacker.py) script can be used like this: ``` $ python3 upp-unpacker.py response_upp.bin - Version: 0x23 @@ -288,12 +352,12 @@ $ python3 upp-unpacker.py response_upp.bin _The UUID in this response UPP doesn't match the one from examples above because the UPP was sent from Niomon-Dev._ ### Checking the anchoring status of an UPP -uBirch Niomon accepting the UPP doesn't mean that it is anchored yet. This process takes place in certain intervals, so one might have to wait a short while. The script to check if a UPP was already anchored is [`upp-anchoring-status.py`](upp-anchoring-status.py). +uBirch Niomon accepting the UPP doesn't mean that it is anchored yet. This process takes place in certain intervals, so one might have to wait a short while. The script to check if an UPP was already anchored is [`upp-anchoring-status.py`](upp-anchoring-status.py). ``` $ python3 upp-anchoring-status.py -h usage: upp-anchoring-status.py [-h] [--ishash ISHASH] [--env ENV] [--ishex ISHEX] INPUT -Requests the verification/anchoring of a UPP from the uBirch backend +Requests the verification/anchoring of an UPP from the uBirch backend positional arguments: INPUT input hash or upp path (depends on --ishash) @@ -309,7 +373,7 @@ optional arguments: When --ishash/-i is set to true, the input argument is treated as a base64 payload hash. Otherwise, it is expected to be some kind of path to read a UPP from. This can be a file path or also /dev/stdin if the UPP is piped to this program via standard input. ``` -- `--ishash/-i` A boolean specifying whether the input data is a payload hash or a UPP. The payload hash is what is actually used to look up anchoring information about the UPP. This script can either extract it from a given UPP or just use the hash directly if provided. If directly provided, it must be base64 encoded (see the last example of this sub-section). `true` or `false`. +- `--ishash/-i` A boolean specifying whether the input data is a payload hash or an UPP. The payload hash is what is actually used to look up anchoring information about the UPP. This script can either extract it from a given UPP or just use the hash directly if provided. If directly provided, it must be base64 encoded (see the last example of this sub-section). `true` or `false`. - `--env/-e` The stage to check on. Should be the one the UPP was sent to. `prod`, `demo` or `dev`. - `--ishex/-x` A boolean which controls how the input UPP data is interpreted. By default, the data will be interpreted as normale binary data. When this flag is set to `true`, it will be considered @@ -352,7 +416,7 @@ Prev. UPP: "liPEEAi4JX1SUEaGhzAnqUf6pn3EQOeEqF4lo+xT9RF2ygDx9+anv14fykUolJ9gmKuT ``` ### Verifying data -In a real use case, not the UPP, but rather the original data itself has to be verified. The original data can be hashed, and the hash can be looked up by the [`data-verifier.py`](data-verifier.py) script. It has similar behaviour to the [`upp-anchoring-status.py`](upp-anchoring-status.py) script, see [Checking the anchoring status of a UPP](#checking-the-anchoring-status-of-an-upp). +In a real use case, not the UPP, but rather the original data itself has to be verified. The original data can be hashed, and the hash can be looked up by the [`data-verifier.py`](data-verifier.py) script. It has similar behaviour to the [`upp-anchoring-status.py`](upp-anchoring-status.py) script, see [Checking the anchoring status of an UPP](#checking-the-anchoring-status-of-an-upp). ```txt $ python data-verifier.py --help usage: data-verifier.py [-h] [--ispath ISPATH] [--env ENV] [--isjson ISJSON] [--hash HASH] [--no-send NOSEND] [--ishl ISHASHLINK] INPUT From 56bff975ba39f425fbf69f267e10564d1d57fb57 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Tue, 16 Nov 2021 16:17:57 +0100 Subject: [PATCH 35/40] adapt changes from review to the readme --- examples/EXAMPLES.md | 146 +++++++------------------------------------ 1 file changed, 23 insertions(+), 123 deletions(-) diff --git a/examples/EXAMPLES.md b/examples/EXAMPLES.md index 5c06c50..15e4563 100644 --- a/examples/EXAMPLES.md +++ b/examples/EXAMPLES.md @@ -74,20 +74,27 @@ A keystore can be read out with the [`keystore-dumper.py`](keystore-dumper.py) s ``` $ python keystore-dumper.py --help -usage: keystore-dumper.py [-h] [--show-sk SHOW_SIGNING_KET] KEYSTORE KEYSTORE_PASS - -Dump the contents of a keystore (.jks) - -positional arguments: - KEYSTORE keystore file path; e.g.: test.jks - KEYSTORE_PASS keystore password; e.g.: secret - -optional arguments: - -h, --help show this help message and exit - --show-sk SHOW_SIGNING_KEY, -s SHOW_SIGNING_KEY - enables/disables showing of signing keys; e.g.: true, false (default: False) -``` -By default, only UUIDs und public keys (verifying keys) will be displayed. Displaying of private keys (signing keys) can be enabled by passing `-s true`. +usage: keystore-dumper.py [-h] [--show-sk SHOW_SIGNING_KEY] KEYSTORE KEYSTORE_PASS +``` +By default, only UUIDs und public keys (verifying keys) will be displayed. Displaying of private keys (signing keys) can be enabled by passing `-s true`. `KEYSTORE` is the path to the keystore file and `KEYSTORE_PASS` is the password needed to open/decrypt it. Below is an example with `-s` being set to `false`. +``` +====================================================================================================================================== +UUID: 292e2f0b-1d9e-407f-9b1a-bda5a9560797 + VK : 2dc6fdf373c7e13abfe1f51ff0fe45417ad713c5fb8df90b76c3077355c4b5e9 + SK : ████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ +====================================================================================================================================== +====================================================================================================================================== +UUID: 832e885c-be2a-44b1-ade6-c6c6dd718a3b + VK : 43536e9793ab5b205ba62e614d12104cbb6f906b2a44b79c1cd01e87a724d5b1 + SK : ████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ +====================================================================================================================================== +====================================================================================================================================== +UUID: 622e3933-8fdd-446f-b645-d3aa7ed8b638 + VK : cfd32f687366679d7ef73795bafa3bfca43421e092fbc2501f7f4bfe339bb15a + SK : ████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ +====================================================================================================================================== +``` +As you can see, the secret key is not printed and `█`s are used as placeholders. ### Registering a public key To enable the uBirch backend to verify an UPP, it needs to know the corresponding verifying key. Therefore, the device needs to send this key to the backend, before starting to send UPPs, which are supposed to be verified and anchored. Registering a verifying key is done by sending a special kind of UPP containing this key. This can be done by using two scripts: @@ -147,33 +154,7 @@ $ python3 upp-creator.py --help usage: upp-creator.py [-h] [--version VERISON] [--type TYPE] [--ks KS] [--kspwd KSPWD] [--keyreg KEYREG] [--hash HASH] [--isjson ISJSON] [--output OUTPUT] [--nostdout nostdout] UUID DATA -Create a uBirch Protocol Package (UPP) - -positional arguments: - UUID UUID to work with; e.g.: 56bd9b85-6c6e-4a24-bf71-f2ac2de10183 - DATA data to be packed into the UPP or hashed; e.g.: {"t": 23.4, "ts": 1624624140} - -optional arguments: - -h, --help show this help message and exit - --version VERISON, -v VERISON - version of the UPP; 0x21 (unsigned; NOT IMPLEMENTED), 0x22 (signed) or 0x23 (chained) (default: 0x23) - --type TYPE, -t TYPE type of the UPP (0 < type < 256); e.g.: 0x00 (unknown), 0x32 (msgpack), 0x53 (generic), ... (default and recommended: 0x00) - --ks KS, -k KS keystore file path; e.g.: test.jks (default: devices.jks) - --kspwd KSPWD, -p KSPWD - keystore password; e.g.: secret (default: keystore) - --keyreg KEYREG, -r KEYREG - generate a key registration UPP (data and --hash will be ignored); e.g.: true, false (default: False) - --hash HASH hash algorithm for hashing the data; sha256, sha512 or off (disable hashing), ... (default and recommended: sha512) - --isjson ISJSON, -j ISJSON - tells the script to treat the input data as json and serealize it (see EXAMPLES.md for more information); true or false (default: False) - --output OUTPUT, -o OUTPUT - file to write the generated UPP to (aside from standard output); e.g. upp.bin (default: upp.bin) - --nostdout nostdout, -n nostdout - do not output anything to stdout; can be combined with --output /dev/stdout; e.g.: true, false (default: False) - -Note that, when using chained UPPs (--version 0x23), this tool will try to load/save signatures from/to .sig, where UUID will be replaced with the actual UUID. Make sure that the UUID.sig file is in your current working directory if you try to continue an UPP chain -using this tool.Also beware that you will only be able to access the contents of a keystore when you use the same password you used when creating it. Otherwise all contents are lost. When --hash off is set, contents of the DATA argument will be copied into the -payload field of the UPP. Normally used for special messages (e.g. key registration). For more information on possible values for --type and --version see https://github.com/ubirch/ubirch-protocol. +Note that, when using chained UPPs (--version 0x23), this tool will try to load/save signatures from/to .sig, where UUID will be replaced with the actual UUID. Make sure that the UUID.sig file is in your current working directory if you try to continue an UPP chain using this tool. Also beware that you will only be able to access the contents of a keystore when you use the same password you used when creating it. Otherwise all contents are lost. When --hash off is set, contents of the DATA argument will be copied into the payload field of the UPP. Normally used for special messages (e.g. key registration). For more information on possible values for --type and --version see https://github.com/ubirch/ubirch-protocol. ``` The script allows multiple modes of operation, which can be set through different command line arguments. Some of those directly set fields in the resulting UPP. Please consult the [uBirch Protocol Readme](https://github.com/ubirch/ubirch-protocol#basic-message-format) for further information on those fields and their possible values. - `--version/-v` This flag sets the version field of the UPP. The version field actually consists of two sub-fields. The higher four bits set the actual version (`1` or `2`) and the "mode". The higher four bits will be set to two(`0010`) in almost all use cases. The mode can either be a simple UPP without a signature, an UPP with a signature and an UPP with a signature + the signature of the previous UPP embedded into it. The latter would be called a_Chained UPP_. Unsigned UPPs (`-v 0x21`) are not implemented. Signed UPPs have `-v 0x22` and chained ones `-v 0x23`. @@ -210,20 +191,6 @@ After creating the UPP, it can be sent to the uBirch backend where it will be ve ``` $ python3 upp-sender.py --help usage: upp-sender.py [-h] [--env ENV] [--input INPUT] [--output OUTPUT] UUID AUTH - -Send a uBirch Protocol Package (UPP) to uBirch Niomon - -positional arguments: - UUID UUID to work with; e.g.: 56bd9b85-6c6e-4a24-bf71-f2ac2de10183 - AUTH uBirch device authentication token - -optional arguments: - -h, --help show this help message and exit - --env ENV, -e ENV environment to operate in; dev, demo or prod (default: dev) - --input INPUT, -i INPUT - UPP input file path; e.g. upp.bin or /dev/stdin (default: upp.bin) - --output OUTPUT, -o OUTPUT - response UPP output file path (ignored for key registration UPPs); e.g. response_upp.bin (default: response_upp.bin) ``` For this script the parameters are: - `--env/-e` The env to operate on. This parameter decides wether the UPP will be sent to `niomon.prod.ubirch.com`, `niomon.demo.ubirch.com` or `niomon.dev.ubirch.com`. The value can either be `prod`, `demo` or `dev`. It must match the stage, the UUID is registered on. @@ -247,17 +214,6 @@ To make sure, that the response UPP actually was sent by the uBirch backend, its $ python3 upp-verifier.py --help usage: upp-verifier.py [-h] [--verifying-key VK] [--verifying-key-uuid UUID] [--input INPUT] -Check if an UPP is valid/properly signed - -optional arguments: - -h, --help show this help message and exit - --verifying-key VK, -k VK - key to be used for verification; any verifying key in hex like "b12a906051f102881bbb487ee8264aa05d8d0fcc51218f2a47f562ceb9b0d068" - --verifying-key-uuid UUID, -u UUID - the UUID for the key supplied via -k (only needed when -k is specified); e.g.: 6eac4d0b-16e6-4508-8c46-22e7451ea5a1 - --input INPUT, -i INPUT - UPP input file path; e.g. upp.bin or /dev/stdin (default: upp.bin) - Note that, when trying to verify an UPP, sent by the uBirch backend (Niomon), a verifying key doesn't have to be provided via the -k option. Instead, this script will try to pick the correct stage key based on the UUID which is contained in the UPP, identifying the creator. If the UUID doesn't match any Niomon stage and no key was specified using -k, an error message will be printed. ``` - `--verifying-key/-k` If not trying to verify an UPP coming from uBirch Niomon but from another source, the verifying key for that source needs to be provided. This parameter expects the key as a hex-string like `b12a906051f102881bbb487ee8264aa05d8d0fcc51218f2a47f562ceb9b0d068`. @@ -274,7 +230,7 @@ $ python3 upp-verifier.py --input response_upp.bin ### Verifying an UPP chain When working with `chained UPPs` it can be useful to check whether the chain is in order and valid. For -this task, the [`upp-chain-checker.py`](upp-chain-checke.py) can be used. It reads in a list of UPPs, +this task, the [`upp-chain-checker.py`](upp-chain-checker.py) can be used. It reads in a list of UPPs, checks the signature of each UPP and compares the `prevsig` field with the `signature` of the last UPP. If at any point something doesn't match up, it will print an error message alonge with the number of the UPP at which the chain broke/something went wrong. The UPP list can either be read directly from a file @@ -283,23 +239,6 @@ a list of hex-encoded UPPs. ``` $ python3 upp-chain-checker.py -h usage: upp-chain-checker.py [-h] [--is-json ISJSON] [--is-hex ISHEX] INPUTFILE VK UUID - -Check if a sequence of chained UPPs is valid/properly signed and correctly chained - -positional arguments: - INPUTFILE Input file path; e.g. upp_list.bin, upp_list.json or /dev/stdin - VK key to be used for verification; any verifying key in hex like - "b12a906051f102881bbb487ee8264aa05d8d0fcc51218f2a47f562ceb9b0d068" - UUID the UUID for the verifying key; e.g.: 6eac4d0b-16e6-4508-8c46-22e7451ea5a1 - -optional arguments: - -h, --help show this help message and exit - --is-json ISJSON, -j ISJSON - If true, the script expects a JSON file for INPUTFILE (see below); e.g. true, false (default: false) - --is-hex ISHEX, -x ISHEX - If true, the script hex-encoded UPPs from the input file; e.g. true, false (default: false) - -The JSON file (when using --is-json true) es expected to contain a single field called "upps", which is a list of hex-encoded UPPs. Otherwise (--is-json false). If --is-hex is true, it expects a sequence of hex-encoded UPPs separated by newlines. The third (default) scenario is that the script expects a sequence of binary UPPs separated by newlines. If --is-json true is set, --is-hex will be ignored. ``` The `VK` and `UUID` arguments work like in the [UPP-Verifier](#verifying-an-upp), with the difference that they aren't optional and must be provided. Here is an example JSON file containing four UPPs @@ -356,22 +295,6 @@ uBirch Niomon accepting the UPP doesn't mean that it is anchored yet. This proce ``` $ python3 upp-anchoring-status.py -h usage: upp-anchoring-status.py [-h] [--ishash ISHASH] [--env ENV] [--ishex ISHEX] INPUT - -Requests the verification/anchoring of an UPP from the uBirch backend - -positional arguments: - INPUT input hash or upp path (depends on --ishash) - -optional arguments: - -h, --help show this help message and exit - --ishash ISHASH, -i ISHASH - sets if INPUT is being treated as a hash or upp path; true or false (default: False) - --env ENV, -e ENV the environment to operate in; dev, demo or prod (default: dev) - --ishex ISHEX, -x ISHEX - Sets whether the UPP input data is a hex string or binary; e.g. true, false (default: false) - -When --ishash/-i is set to true, the input argument is treated as a base64 payload hash. Otherwise, it is expected to be some kind of path to read a -UPP from. This can be a file path or also /dev/stdin if the UPP is piped to this program via standard input. ``` - `--ishash/-i` A boolean specifying whether the input data is a payload hash or an UPP. The payload hash is what is actually used to look up anchoring information about the UPP. This script can either extract it from a given UPP or just use the hash directly if provided. If directly provided, it must be base64 encoded (see the last example of this sub-section). `true` or `false`. - `--env/-e` The stage to check on. Should be the one the UPP was sent to. `prod`, `demo` or `dev`. @@ -420,29 +343,6 @@ In a real use case, not the UPP, but rather the original data itself has to be v ```txt $ python data-verifier.py --help usage: data-verifier.py [-h] [--ispath ISPATH] [--env ENV] [--isjson ISJSON] [--hash HASH] [--no-send NOSEND] [--ishl ISHASHLINK] INPUT - -Check if the hash of given input data is known to the uBirch backend (verify it) - -positional arguments: - INPUT input data or data file path (depends on --ispath) - -optional arguments: - -h, --help show this help message and exit - --ispath ISPATH, -i ISPATH - sets if INPUT is being treated as data or data file path; true or false (default: False) - --env ENV, -e ENV the environment to operate in; dev, demo or prod (default: dev) - --isjson ISJSON, -j ISJSON - tells the script to treat the input data as json and serealize it (see EXAMPLES.md for more information); true or false - (default: True) - --hash HASH, -a HASH sets the hash algorithm to use; sha256, sha512 or OFF to treat the input data as hash (default: sha256) - --no-send NOSEND, -n NOSEND - if set to true, the script will only generate the hash of the input data without sending it; true or false (default: False) - --ishl ISHASHLINK, -l ISHASHLINK - implied --isjson to be true; if set to true, the script will look for a hashlink list in the json object and use it to - decide which fields to hash; true or false (default: False) - -When --ispath/-i is set to true, the input data is treated as a file path to read the actual input data from. When setting --hash/-a to off, the -input argument is expected to be a valid base64 encoded hash. ``` - `--ispath/-i` Specifies wether the input is to be treated as a data-file path or direct input data. `true` or `false`. - `--env-e` The stage to check on. Should be the one the UPP corresponding to the data was sent to. `prod`, `demo` or `dev`. From 71a6a1c9601998490261ce3ce3b52c250bac8b10 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Tue, 16 Nov 2021 16:19:19 +0100 Subject: [PATCH 36/40] include changes from review --- examples/data-sender.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/data-sender.py b/examples/data-sender.py index 136e5df..5ca500d 100644 --- a/examples/data-sender.py +++ b/examples/data-sender.py @@ -46,7 +46,7 @@ def setup_argparse(self): help="UUID to work with; e.g.: 56bd9b85-6c6e-4a24-bf71-f2ac2de10183" ) self.argparser.add_argument("auth", metavar="AUTH", type=str, - help="uBirch device authentication token" + help="uBirch device authentication token, e.g.: 12345678-1234-1234-1234-123456789abc (this is NOT the UUID)" ) self.argparser.add_argument("--env", "-e", metavar="ENV", type=str, default=DEFAULT_ENV, help="environment to operate in; dev, demo or prod (default: %s)" % DEFAULT_ENV From 2c59f532aa7276077272754ca2a4652794d674c4 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Tue, 16 Nov 2021 16:23:37 +0100 Subject: [PATCH 37/40] [from review] replace old print statement with logger.x --- examples/upp-anchoring-status.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/upp-anchoring-status.py b/examples/upp-anchoring-status.py index df0f5dc..9da7d0e 100644 --- a/examples/upp-anchoring-status.py +++ b/examples/upp-anchoring-status.py @@ -154,15 +154,15 @@ def get_status(self) -> bool: jobj = json.loads(r.content) - print("Curr. UPP: \"%s\"" % jobj.get("upp", "-- no curr. upp information --")) - print("Prev. UPP: \"%s\"" % jobj.get("prev", "-- no prev. upp information --")) + logger.info("Curr. UPP: \"%s\"" % jobj.get("upp", "-- no curr. upp information --")) + logger.info("Prev. UPP: \"%s\"" % jobj.get("prev", "-- no prev. upp information --")) if jobj.get("anchors") in [None, []]: logger.info("The UPP has NOT been anchored into any blockchains yet! Please retry later") else: logger.info("The UPP has been fully anchored!") - print(jobj.get("anchors")) + logger.info(jobj.get("anchors")) elif r.status_code == 404: logger.info("The UPP is NOT known to the uBirch backend! (code: %d)" % r.status_code) except Exception as e: From 9c3027ec7342c321a1a005079e8614f34dea819a Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Tue, 16 Nov 2021 16:28:13 +0100 Subject: [PATCH 38/40] initialize ishex and ishex_str in __init__ before using it in setup_argparse --- examples/upp-chain-checker.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/upp-chain-checker.py b/examples/upp-chain-checker.py index b692124..b101feb 100644 --- a/examples/upp-chain-checker.py +++ b/examples/upp-chain-checker.py @@ -51,6 +51,9 @@ def __init__(self): self.isjson : bool = None self.isjson_str : str = None + self.ishex : bool = None + self.ishex_str : str = None + # initialize the argument parser self.setup_argparse() From f4cb6e1fee872a7f42692ed71d068664ae32863e Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Tue, 16 Nov 2021 16:29:18 +0100 Subject: [PATCH 39/40] adapt changes from review --- examples/upp-creator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/upp-creator.py b/examples/upp-creator.py index 2151972..2654f7e 100644 --- a/examples/upp-creator.py +++ b/examples/upp-creator.py @@ -76,10 +76,12 @@ def __init__(self): self.nostdout : bool = None self.payload : bytes = None self.ishl : bool = None + self.ishl_str : str = None self.ishash : bool = False self.hasher : object = None self.keystore : ubirch.KeyStore = None + self.keystore_pass : str = None self.proto : Proto = None self.payload : bytes = None self.upp : bytes = None @@ -252,7 +254,7 @@ def init_proto(self) -> bool: except Exception as e: logger.exception(e) - False + return False return True From 725ec4b2f85b90b79e1ef9080ad9873144f2d33c Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Tue, 16 Nov 2021 16:38:10 +0100 Subject: [PATCH 40/40] adapt changes from review --- ubirch/ubirch_protocol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ubirch/ubirch_protocol.py b/ubirch/ubirch_protocol.py index 6250f8f..1456886 100644 --- a/ubirch/ubirch_protocol.py +++ b/ubirch/ubirch_protocol.py @@ -289,8 +289,8 @@ def verfiy_signature(self, uuid: UUID, msgpackUPP: bytes) -> bool: """ Verify the integrity of the message and decode the contents Raises an value error when the message is too short + :param uuid: the uuid of the sender of the message :param msgpackUPP: the msgpack encoded message - :param unpackedUPP: (optional) if not provided, the function will unpack the upp itself :return: the decoded message """ # separate the message from the signature