From e02d9edc347700682f7476affe88d7f341bc5918 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 11 May 2023 17:24:34 +0200 Subject: [PATCH] remove nonce management, related repo methods not needed for borg2 repos (we derive a new session key for each borg invocation and start counting from 0). also not needed for borg 1.x repos because we only read them (borg transfer) and won't write new encrypted data to them. --- src/borg/crypto/key.py | 5 +- src/borg/crypto/nonces.py | 91 -------------- src/borg/remote.py | 12 -- src/borg/repository.py | 30 ----- src/borg/testsuite/key.py | 6 - src/borg/testsuite/nonces.py | 197 ------------------------------- src/borg/testsuite/repository.py | 42 ------- 7 files changed, 2 insertions(+), 381 deletions(-) delete mode 100644 src/borg/crypto/nonces.py delete mode 100644 src/borg/testsuite/nonces.py diff --git a/src/borg/crypto/key.py b/src/borg/crypto/key.py index 01b08579d9..c9caf7819c 100644 --- a/src/borg/crypto/key.py +++ b/src/borg/crypto/key.py @@ -25,7 +25,6 @@ from ..repoobj import RepoObj -from .nonces import NonceManager from .low_level import AES, bytes_to_int, num_cipher_blocks, hmac_sha256, blake2b_256, hkdf_hmac_sha512 from .low_level import AES256_CTR_HMAC_SHA256, AES256_CTR_BLAKE2b, AES256_OCB, CHACHA20_POLY1305 from . import low_level @@ -372,7 +371,8 @@ class AESKeyBase(KeyBase): logically_encrypted = True def encrypt(self, id, data): - next_iv = self.nonce_manager.ensure_reservation(self.cipher.next_iv(), self.cipher.block_count(len(data))) + # legacy, this is only used by the tests. + next_iv = self.cipher.next_iv() return self.cipher.encrypt(data, header=self.TYPE_STR, iv=next_iv) def decrypt(self, id, data): @@ -411,7 +411,6 @@ def init_ciphers(self, manifest_data=None): manifest_blocks = num_cipher_blocks(len(manifest_data)) nonce = self.cipher.extract_iv(manifest_data) + manifest_blocks self.cipher.set_iv(nonce) - self.nonce_manager = NonceManager(self.repository, nonce) class FlexiKey: diff --git a/src/borg/crypto/nonces.py b/src/borg/crypto/nonces.py deleted file mode 100644 index d1a9d9fbd2..0000000000 --- a/src/borg/crypto/nonces.py +++ /dev/null @@ -1,91 +0,0 @@ -import os -import sys -from binascii import unhexlify - -from ..helpers import get_security_dir -from ..helpers import bin_to_hex -from ..platform import SaveFile -from ..remote import InvalidRPCMethod - -from .low_level import bytes_to_long, long_to_bytes - -MAX_REPRESENTABLE_NONCE = 2**64 - 1 -NONCE_SPACE_RESERVATION = 2**28 # This in units of AES blocksize (16 bytes) - - -class NonceManager: - def __init__(self, repository, manifest_nonce): - self.repository = repository - self.end_of_nonce_reservation = None - self.manifest_nonce = manifest_nonce - self.nonce_file = os.path.join(get_security_dir(self.repository.id_str), "nonce") - - def get_local_free_nonce(self): - try: - with open(self.nonce_file) as fd: - return bytes_to_long(unhexlify(fd.read())) - except FileNotFoundError: - return None - - def commit_local_nonce_reservation(self, next_unreserved, start_nonce): - if self.get_local_free_nonce() != start_nonce: - raise Exception("nonce space reservation with mismatched previous state") - with SaveFile(self.nonce_file, binary=False) as fd: - fd.write(bin_to_hex(long_to_bytes(next_unreserved))) - - def get_repo_free_nonce(self): - try: - return self.repository.get_free_nonce() - except InvalidRPCMethod: - # old server version, suppress further calls - sys.stderr.write("Please upgrade to borg version 1.1+ on the server for safer AES-CTR nonce handling.\n") - self.get_repo_free_nonce = lambda: None - self.commit_repo_nonce_reservation = lambda next_unreserved, start_nonce: None - return None - - def commit_repo_nonce_reservation(self, next_unreserved, start_nonce): - self.repository.commit_nonce_reservation(next_unreserved, start_nonce) - - def ensure_reservation(self, nonce, nonce_space_needed): - """ - Call this before doing encryption, give current, yet unused, integer IV as - and the amount of subsequent (counter-like) IVs needed as . - Return value is the IV (counter) integer you shall use for encryption. - - Note: this method may return the you gave, if a reservation for it exists or - can be established, so make sure you give a unused nonce. - """ - # Nonces may never repeat, even if a transaction aborts or the system crashes. - # Therefore a part of the nonce space is reserved before any nonce is used for encryption. - # As these reservations are committed to permanent storage before any nonce is used, this protects - # against nonce reuse in crashes and transaction aborts. In that case the reservation still - # persists and the whole reserved space is never reused. - # - # Local storage on the client is used to protect against an attacker that is able to rollback the - # state of the server or can do arbitrary modifications to the repository. - # Storage on the server is used for the multi client use case where a transaction on client A is - # aborted and later client B writes to the repository. - # - # This scheme does not protect against attacker who is able to rollback the state of the server - # or can do arbitrary modifications to the repository in the multi client usecase. - - if self.end_of_nonce_reservation: - # we already got a reservation, if nonce_space_needed still fits everything is ok - next_nonce = nonce - assert next_nonce <= self.end_of_nonce_reservation - if next_nonce + nonce_space_needed <= self.end_of_nonce_reservation: - return next_nonce - - repo_free_nonce = self.get_repo_free_nonce() - local_free_nonce = self.get_local_free_nonce() - free_nonce_space = max( - x - for x in (repo_free_nonce, local_free_nonce, self.manifest_nonce, self.end_of_nonce_reservation) - if x is not None - ) - reservation_end = free_nonce_space + nonce_space_needed + NONCE_SPACE_RESERVATION - assert reservation_end < MAX_REPRESENTABLE_NONCE - self.commit_repo_nonce_reservation(reservation_end, repo_free_nonce) - self.commit_local_nonce_reservation(reservation_end, local_free_nonce) - self.end_of_nonce_reservation = reservation_end - return free_nonce_space diff --git a/src/borg/remote.py b/src/borg/remote.py index 9349d9bc6b..fab5d76a69 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -134,8 +134,6 @@ def __init__(self, data): "negotiate": ("client_data",), "open": ("path", "create", "lock_wait", "lock", "exclusive", "append_only"), "info": (), - "get_free_nonce": (), - "commit_nonce_reservation": ("next_unreserved", "start_nonce"), } @@ -159,8 +157,6 @@ class RepositoryServer: # pragma: no cover "save_key", "load_key", "break_lock", - "get_free_nonce", - "commit_nonce_reservation", "inject_exception", ) @@ -1024,14 +1020,6 @@ def save_key(self, keydata): def load_key(self): """actual remoting is done via self.call in the @api decorator""" - @api(since=parse_version("1.0.0")) - def get_free_nonce(self): - """actual remoting is done via self.call in the @api decorator""" - - @api(since=parse_version("1.0.0")) - def commit_nonce_reservation(self, next_unreserved, start_nonce): - """actual remoting is done via self.call in the @api decorator""" - @api(since=parse_version("1.0.0")) def break_lock(self): """actual remoting is done via self.call in the @api decorator""" diff --git a/src/borg/repository.py b/src/borg/repository.py index e10ae0dc5b..6eb87df074 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -369,36 +369,6 @@ def load_key(self): # note: if we return an empty string, it means there is no repo key return keydata.encode("utf-8") # remote repo: msgpack issue #99, returning bytes - def get_free_nonce(self): - if self.do_lock and not self.lock.got_exclusive_lock(): - raise AssertionError("bug in code, exclusive lock should exist here") - - nonce_path = os.path.join(self.path, "nonce") - try: - with open(nonce_path) as fd: - return int.from_bytes(unhexlify(fd.read()), byteorder="big") - except FileNotFoundError: - return None - - def commit_nonce_reservation(self, next_unreserved, start_nonce): - if self.do_lock and not self.lock.got_exclusive_lock(): - raise AssertionError("bug in code, exclusive lock should exist here") - - if self.get_free_nonce() != start_nonce: - raise Exception("nonce space reservation with mismatched previous state") - nonce_path = os.path.join(self.path, "nonce") - try: - with SaveFile(nonce_path, binary=False) as fd: - fd.write(bin_to_hex(next_unreserved.to_bytes(8, byteorder="big"))) - except PermissionError as e: - # error is only a problem if we even had a lock - if self.do_lock: - raise - logger.warning( - "%s: Failed writing to '%s'. This is expected when working on " - "read-only repositories." % (e.strerror, e.filename) - ) - def destroy(self): """Destroy the repository at `self.path`""" if self.append_only: diff --git a/src/borg/testsuite/key.py b/src/borg/testsuite/key.py index c0417990b5..5f1d0ed58a 100644 --- a/src/borg/testsuite/key.py +++ b/src/borg/testsuite/key.py @@ -116,12 +116,6 @@ def canonical_path(self): id = bytes(32) id_str = bin_to_hex(id) - def get_free_nonce(self): - return None - - def commit_nonce_reservation(self, next_unreserved, start_nonce): - pass - def save_key(self, data): self.key_data = data diff --git a/src/borg/testsuite/nonces.py b/src/borg/testsuite/nonces.py deleted file mode 100644 index d9548fa91c..0000000000 --- a/src/borg/testsuite/nonces.py +++ /dev/null @@ -1,197 +0,0 @@ -import os.path - -import pytest - -from ..crypto import nonces -from ..crypto.nonces import NonceManager -from ..crypto.key import bin_to_hex -from ..helpers import get_security_dir -from ..remote import InvalidRPCMethod - - -class TestNonceManager: - class MockRepository: - class _Location: - orig = "/some/place" - - _location = _Location() - id = bytes(32) - id_str = bin_to_hex(id) - - def get_free_nonce(self): - return self.next_free - - def commit_nonce_reservation(self, next_unreserved, start_nonce): - assert start_nonce == self.next_free - self.next_free = next_unreserved - - class MockOldRepository(MockRepository): - def get_free_nonce(self): - raise InvalidRPCMethod("") - - def commit_nonce_reservation(self, next_unreserved, start_nonce): - pytest.fail("commit_nonce_reservation should never be called on an old repository") - - def setUp(self): - self.repository = None - - def cache_nonce(self): - with open(os.path.join(get_security_dir(self.repository.id_str), "nonce")) as fd: - return fd.read() - - def set_cache_nonce(self, nonce): - with open(os.path.join(get_security_dir(self.repository.id_str), "nonce"), "w") as fd: - assert fd.write(nonce) - - def test_empty_cache_and_old_server(self, monkeypatch): - monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20) - - self.repository = self.MockOldRepository() - manager = NonceManager(self.repository, 0x2000) - next_nonce = manager.ensure_reservation(0x2000, 19) - assert next_nonce == 0x2000 - - assert self.cache_nonce() == "0000000000002033" - - def test_empty_cache(self, monkeypatch): - monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20) - - self.repository = self.MockRepository() - self.repository.next_free = 0x2000 - manager = NonceManager(self.repository, 0x2000) - next_nonce = manager.ensure_reservation(0x2000, 19) - assert next_nonce == 0x2000 - - assert self.cache_nonce() == "0000000000002033" - - def test_empty_nonce(self, monkeypatch): - monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20) - - self.repository = self.MockRepository() - self.repository.next_free = None - manager = NonceManager(self.repository, 0x2000) - next_nonce = manager.ensure_reservation(0x2000, 19) - assert next_nonce == 0x2000 - - assert self.cache_nonce() == "0000000000002033" - assert self.repository.next_free == 0x2033 - - # enough space in reservation - next_nonce = manager.ensure_reservation(0x2013, 13) - assert next_nonce == 0x2013 - assert self.cache_nonce() == "0000000000002033" - assert self.repository.next_free == 0x2033 - - # just barely enough space in reservation - next_nonce = manager.ensure_reservation(0x2020, 19) - assert next_nonce == 0x2020 - assert self.cache_nonce() == "0000000000002033" - assert self.repository.next_free == 0x2033 - - # no space in reservation - next_nonce = manager.ensure_reservation(0x2033, 16) - assert next_nonce == 0x2033 - assert self.cache_nonce() == "0000000000002063" - assert self.repository.next_free == 0x2063 - - # spans reservation boundary - next_nonce = manager.ensure_reservation(0x2043, 64) - assert next_nonce == 0x2063 - assert self.cache_nonce() == "00000000000020c3" - assert self.repository.next_free == 0x20C3 - - def test_sync_nonce(self, monkeypatch): - monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20) - - self.repository = self.MockRepository() - self.repository.next_free = 0x2000 - self.set_cache_nonce("0000000000002000") - - manager = NonceManager(self.repository, 0x2000) - next_nonce = manager.ensure_reservation(0x2000, 19) - assert next_nonce == 0x2000 - - assert self.cache_nonce() == "0000000000002033" - assert self.repository.next_free == 0x2033 - - def test_server_just_upgraded(self, monkeypatch): - monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20) - - self.repository = self.MockRepository() - self.repository.next_free = None - self.set_cache_nonce("0000000000002000") - - manager = NonceManager(self.repository, 0x2000) - next_nonce = manager.ensure_reservation(0x2000, 19) - assert next_nonce == 0x2000 - - assert self.cache_nonce() == "0000000000002033" - assert self.repository.next_free == 0x2033 - - def test_transaction_abort_no_cache(self, monkeypatch): - monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20) - - self.repository = self.MockRepository() - self.repository.next_free = 0x2000 - - manager = NonceManager(self.repository, 0x2000) - next_nonce = manager.ensure_reservation(0x1000, 19) - assert next_nonce == 0x2000 - - assert self.cache_nonce() == "0000000000002033" - assert self.repository.next_free == 0x2033 - - def test_transaction_abort_old_server(self, monkeypatch): - monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20) - - self.repository = self.MockOldRepository() - self.set_cache_nonce("0000000000002000") - - manager = NonceManager(self.repository, 0x2000) - next_nonce = manager.ensure_reservation(0x1000, 19) - assert next_nonce == 0x2000 - - assert self.cache_nonce() == "0000000000002033" - - def test_transaction_abort_on_other_client(self, monkeypatch): - monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20) - - self.repository = self.MockRepository() - self.repository.next_free = 0x2000 - self.set_cache_nonce("0000000000001000") - - manager = NonceManager(self.repository, 0x2000) - next_nonce = manager.ensure_reservation(0x1000, 19) - assert next_nonce == 0x2000 - - assert self.cache_nonce() == "0000000000002033" - assert self.repository.next_free == 0x2033 - - def test_interleaved(self, monkeypatch): - monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20) - - self.repository = self.MockRepository() - self.repository.next_free = 0x2000 - self.set_cache_nonce("0000000000002000") - - manager = NonceManager(self.repository, 0x2000) - next_nonce = manager.ensure_reservation(0x2000, 19) - assert next_nonce == 0x2000 - - assert self.cache_nonce() == "0000000000002033" - assert self.repository.next_free == 0x2033 - - # somehow the clients unlocks, another client reserves and this client relocks - self.repository.next_free = 0x4000 - - # enough space in reservation - next_nonce = manager.ensure_reservation(0x2013, 12) - assert next_nonce == 0x2013 - assert self.cache_nonce() == "0000000000002033" - assert self.repository.next_free == 0x4000 - - # spans reservation boundary - next_nonce = manager.ensure_reservation(0x201F, 21) - assert next_nonce == 0x4000 - assert self.cache_nonce() == "0000000000004035" - assert self.repository.next_free == 0x4035 diff --git a/src/borg/testsuite/repository.py b/src/borg/testsuite/repository.py index 51abc9b2d9..8c44c00f8f 100644 --- a/src/borg/testsuite/repository.py +++ b/src/borg/testsuite/repository.py @@ -613,48 +613,6 @@ def test_exceed_quota(self): assert self.repository.storage_quota_use == len(ch1) + 41 + 8 # now we have compacted. -class NonceReservation(RepositoryTestCaseBase): - def test_get_free_nonce_asserts(self): - self.reopen(exclusive=False) - with pytest.raises(AssertionError): - with self.repository: - self.repository.get_free_nonce() - - def test_get_free_nonce(self): - with self.repository: - assert self.repository.get_free_nonce() is None - - with open(os.path.join(self.repository.path, "nonce"), "w") as fd: - fd.write("0000000000000000") - assert self.repository.get_free_nonce() == 0 - - with open(os.path.join(self.repository.path, "nonce"), "w") as fd: - fd.write("5000000000000000") - assert self.repository.get_free_nonce() == 0x5000000000000000 - - def test_commit_nonce_reservation_asserts(self): - self.reopen(exclusive=False) - with pytest.raises(AssertionError): - with self.repository: - self.repository.commit_nonce_reservation(0x200, 0x100) - - def test_commit_nonce_reservation(self): - with self.repository: - with pytest.raises(Exception): - self.repository.commit_nonce_reservation(0x200, 15) - - self.repository.commit_nonce_reservation(0x200, None) - with open(os.path.join(self.repository.path, "nonce")) as fd: - assert fd.read() == "0000000000000200" - - with pytest.raises(Exception): - self.repository.commit_nonce_reservation(0x200, 15) - - self.repository.commit_nonce_reservation(0x400, 0x200) - with open(os.path.join(self.repository.path, "nonce")) as fd: - assert fd.read() == "0000000000000400" - - class RepositoryAuxiliaryCorruptionTestCase(RepositoryTestCaseBase): def setUp(self): super().setUp()