diff --git a/src/borg/crypto/key.py b/src/borg/crypto/key.py index 01b08579d9..c9caf7819c 100644 --- a/src/borg/crypto/key.py +++ b/src/borg/crypto/key.py @@ -25,7 +25,6 @@ from ..repoobj import RepoObj -from .nonces import NonceManager from .low_level import AES, bytes_to_int, num_cipher_blocks, hmac_sha256, blake2b_256, hkdf_hmac_sha512 from .low_level import AES256_CTR_HMAC_SHA256, AES256_CTR_BLAKE2b, AES256_OCB, CHACHA20_POLY1305 from . import low_level @@ -372,7 +371,8 @@ class AESKeyBase(KeyBase): logically_encrypted = True def encrypt(self, id, data): - next_iv = self.nonce_manager.ensure_reservation(self.cipher.next_iv(), self.cipher.block_count(len(data))) + # legacy, this is only used by the tests. + next_iv = self.cipher.next_iv() return self.cipher.encrypt(data, header=self.TYPE_STR, iv=next_iv) def decrypt(self, id, data): @@ -411,7 +411,6 @@ def init_ciphers(self, manifest_data=None): manifest_blocks = num_cipher_blocks(len(manifest_data)) nonce = self.cipher.extract_iv(manifest_data) + manifest_blocks self.cipher.set_iv(nonce) - self.nonce_manager = NonceManager(self.repository, nonce) class FlexiKey: diff --git a/src/borg/crypto/nonces.py b/src/borg/crypto/nonces.py deleted file mode 100644 index d1a9d9fbd2..0000000000 --- a/src/borg/crypto/nonces.py +++ /dev/null @@ -1,91 +0,0 @@ -import os -import sys -from binascii import unhexlify - -from ..helpers import get_security_dir -from ..helpers import bin_to_hex -from ..platform import SaveFile -from ..remote import InvalidRPCMethod - -from .low_level import bytes_to_long, long_to_bytes - -MAX_REPRESENTABLE_NONCE = 2**64 - 1 -NONCE_SPACE_RESERVATION = 2**28 # This in units of AES blocksize (16 bytes) - - -class NonceManager: - def __init__(self, repository, manifest_nonce): - self.repository = repository - self.end_of_nonce_reservation = None - self.manifest_nonce = manifest_nonce - self.nonce_file = os.path.join(get_security_dir(self.repository.id_str), "nonce") - - def get_local_free_nonce(self): - try: - with open(self.nonce_file) as fd: - return bytes_to_long(unhexlify(fd.read())) - except FileNotFoundError: - return None - - def commit_local_nonce_reservation(self, next_unreserved, start_nonce): - if self.get_local_free_nonce() != start_nonce: - raise Exception("nonce space reservation with mismatched previous state") - with SaveFile(self.nonce_file, binary=False) as fd: - fd.write(bin_to_hex(long_to_bytes(next_unreserved))) - - def get_repo_free_nonce(self): - try: - return self.repository.get_free_nonce() - except InvalidRPCMethod: - # old server version, suppress further calls - sys.stderr.write("Please upgrade to borg version 1.1+ on the server for safer AES-CTR nonce handling.\n") - self.get_repo_free_nonce = lambda: None - self.commit_repo_nonce_reservation = lambda next_unreserved, start_nonce: None - return None - - def commit_repo_nonce_reservation(self, next_unreserved, start_nonce): - self.repository.commit_nonce_reservation(next_unreserved, start_nonce) - - def ensure_reservation(self, nonce, nonce_space_needed): - """ - Call this before doing encryption, give current, yet unused, integer IV as - and the amount of subsequent (counter-like) IVs needed as . - Return value is the IV (counter) integer you shall use for encryption. - - Note: this method may return the you gave, if a reservation for it exists or - can be established, so make sure you give a unused nonce. - """ - # Nonces may never repeat, even if a transaction aborts or the system crashes. - # Therefore a part of the nonce space is reserved before any nonce is used for encryption. - # As these reservations are committed to permanent storage before any nonce is used, this protects - # against nonce reuse in crashes and transaction aborts. In that case the reservation still - # persists and the whole reserved space is never reused. - # - # Local storage on the client is used to protect against an attacker that is able to rollback the - # state of the server or can do arbitrary modifications to the repository. - # Storage on the server is used for the multi client use case where a transaction on client A is - # aborted and later client B writes to the repository. - # - # This scheme does not protect against attacker who is able to rollback the state of the server - # or can do arbitrary modifications to the repository in the multi client usecase. - - if self.end_of_nonce_reservation: - # we already got a reservation, if nonce_space_needed still fits everything is ok - next_nonce = nonce - assert next_nonce <= self.end_of_nonce_reservation - if next_nonce + nonce_space_needed <= self.end_of_nonce_reservation: - return next_nonce - - repo_free_nonce = self.get_repo_free_nonce() - local_free_nonce = self.get_local_free_nonce() - free_nonce_space = max( - x - for x in (repo_free_nonce, local_free_nonce, self.manifest_nonce, self.end_of_nonce_reservation) - if x is not None - ) - reservation_end = free_nonce_space + nonce_space_needed + NONCE_SPACE_RESERVATION - assert reservation_end < MAX_REPRESENTABLE_NONCE - self.commit_repo_nonce_reservation(reservation_end, repo_free_nonce) - self.commit_local_nonce_reservation(reservation_end, local_free_nonce) - self.end_of_nonce_reservation = reservation_end - return free_nonce_space diff --git a/src/borg/remote.py b/src/borg/remote.py index 9349d9bc6b..fab5d76a69 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -134,8 +134,6 @@ def __init__(self, data): "negotiate": ("client_data",), "open": ("path", "create", "lock_wait", "lock", "exclusive", "append_only"), "info": (), - "get_free_nonce": (), - "commit_nonce_reservation": ("next_unreserved", "start_nonce"), } @@ -159,8 +157,6 @@ class RepositoryServer: # pragma: no cover "save_key", "load_key", "break_lock", - "get_free_nonce", - "commit_nonce_reservation", "inject_exception", ) @@ -1024,14 +1020,6 @@ def save_key(self, keydata): def load_key(self): """actual remoting is done via self.call in the @api decorator""" - @api(since=parse_version("1.0.0")) - def get_free_nonce(self): - """actual remoting is done via self.call in the @api decorator""" - - @api(since=parse_version("1.0.0")) - def commit_nonce_reservation(self, next_unreserved, start_nonce): - """actual remoting is done via self.call in the @api decorator""" - @api(since=parse_version("1.0.0")) def break_lock(self): """actual remoting is done via self.call in the @api decorator""" diff --git a/src/borg/repository.py b/src/borg/repository.py index e10ae0dc5b..6eb87df074 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -369,36 +369,6 @@ def load_key(self): # note: if we return an empty string, it means there is no repo key return keydata.encode("utf-8") # remote repo: msgpack issue #99, returning bytes - def get_free_nonce(self): - if self.do_lock and not self.lock.got_exclusive_lock(): - raise AssertionError("bug in code, exclusive lock should exist here") - - nonce_path = os.path.join(self.path, "nonce") - try: - with open(nonce_path) as fd: - return int.from_bytes(unhexlify(fd.read()), byteorder="big") - except FileNotFoundError: - return None - - def commit_nonce_reservation(self, next_unreserved, start_nonce): - if self.do_lock and not self.lock.got_exclusive_lock(): - raise AssertionError("bug in code, exclusive lock should exist here") - - if self.get_free_nonce() != start_nonce: - raise Exception("nonce space reservation with mismatched previous state") - nonce_path = os.path.join(self.path, "nonce") - try: - with SaveFile(nonce_path, binary=False) as fd: - fd.write(bin_to_hex(next_unreserved.to_bytes(8, byteorder="big"))) - except PermissionError as e: - # error is only a problem if we even had a lock - if self.do_lock: - raise - logger.warning( - "%s: Failed writing to '%s'. This is expected when working on " - "read-only repositories." % (e.strerror, e.filename) - ) - def destroy(self): """Destroy the repository at `self.path`""" if self.append_only: diff --git a/src/borg/testsuite/key.py b/src/borg/testsuite/key.py index c0417990b5..5f1d0ed58a 100644 --- a/src/borg/testsuite/key.py +++ b/src/borg/testsuite/key.py @@ -116,12 +116,6 @@ def canonical_path(self): id = bytes(32) id_str = bin_to_hex(id) - def get_free_nonce(self): - return None - - def commit_nonce_reservation(self, next_unreserved, start_nonce): - pass - def save_key(self, data): self.key_data = data diff --git a/src/borg/testsuite/nonces.py b/src/borg/testsuite/nonces.py deleted file mode 100644 index d9548fa91c..0000000000 --- a/src/borg/testsuite/nonces.py +++ /dev/null @@ -1,197 +0,0 @@ -import os.path - -import pytest - -from ..crypto import nonces -from ..crypto.nonces import NonceManager -from ..crypto.key import bin_to_hex -from ..helpers import get_security_dir -from ..remote import InvalidRPCMethod - - -class TestNonceManager: - class MockRepository: - class _Location: - orig = "/some/place" - - _location = _Location() - id = bytes(32) - id_str = bin_to_hex(id) - - def get_free_nonce(self): - return self.next_free - - def commit_nonce_reservation(self, next_unreserved, start_nonce): - assert start_nonce == self.next_free - self.next_free = next_unreserved - - class MockOldRepository(MockRepository): - def get_free_nonce(self): - raise InvalidRPCMethod("") - - def commit_nonce_reservation(self, next_unreserved, start_nonce): - pytest.fail("commit_nonce_reservation should never be called on an old repository") - - def setUp(self): - self.repository = None - - def cache_nonce(self): - with open(os.path.join(get_security_dir(self.repository.id_str), "nonce")) as fd: - return fd.read() - - def set_cache_nonce(self, nonce): - with open(os.path.join(get_security_dir(self.repository.id_str), "nonce"), "w") as fd: - assert fd.write(nonce) - - def test_empty_cache_and_old_server(self, monkeypatch): - monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20) - - self.repository = self.MockOldRepository() - manager = NonceManager(self.repository, 0x2000) - next_nonce = manager.ensure_reservation(0x2000, 19) - assert next_nonce == 0x2000 - - assert self.cache_nonce() == "0000000000002033" - - def test_empty_cache(self, monkeypatch): - monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20) - - self.repository = self.MockRepository() - self.repository.next_free = 0x2000 - manager = NonceManager(self.repository, 0x2000) - next_nonce = manager.ensure_reservation(0x2000, 19) - assert next_nonce == 0x2000 - - assert self.cache_nonce() == "0000000000002033" - - def test_empty_nonce(self, monkeypatch): - monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20) - - self.repository = self.MockRepository() - self.repository.next_free = None - manager = NonceManager(self.repository, 0x2000) - next_nonce = manager.ensure_reservation(0x2000, 19) - assert next_nonce == 0x2000 - - assert self.cache_nonce() == "0000000000002033" - assert self.repository.next_free == 0x2033 - - # enough space in reservation - next_nonce = manager.ensure_reservation(0x2013, 13) - assert next_nonce == 0x2013 - assert self.cache_nonce() == "0000000000002033" - assert self.repository.next_free == 0x2033 - - # just barely enough space in reservation - next_nonce = manager.ensure_reservation(0x2020, 19) - assert next_nonce == 0x2020 - assert self.cache_nonce() == "0000000000002033" - assert self.repository.next_free == 0x2033 - - # no space in reservation - next_nonce = manager.ensure_reservation(0x2033, 16) - assert next_nonce == 0x2033 - assert self.cache_nonce() == "0000000000002063" - assert self.repository.next_free == 0x2063 - - # spans reservation boundary - next_nonce = manager.ensure_reservation(0x2043, 64) - assert next_nonce == 0x2063 - assert self.cache_nonce() == "00000000000020c3" - assert self.repository.next_free == 0x20C3 - - def test_sync_nonce(self, monkeypatch): - monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20) - - self.repository = self.MockRepository() - self.repository.next_free = 0x2000 - self.set_cache_nonce("0000000000002000") - - manager = NonceManager(self.repository, 0x2000) - next_nonce = manager.ensure_reservation(0x2000, 19) - assert next_nonce == 0x2000 - - assert self.cache_nonce() == "0000000000002033" - assert self.repository.next_free == 0x2033 - - def test_server_just_upgraded(self, monkeypatch): - monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20) - - self.repository = self.MockRepository() - self.repository.next_free = None - self.set_cache_nonce("0000000000002000") - - manager = NonceManager(self.repository, 0x2000) - next_nonce = manager.ensure_reservation(0x2000, 19) - assert next_nonce == 0x2000 - - assert self.cache_nonce() == "0000000000002033" - assert self.repository.next_free == 0x2033 - - def test_transaction_abort_no_cache(self, monkeypatch): - monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20) - - self.repository = self.MockRepository() - self.repository.next_free = 0x2000 - - manager = NonceManager(self.repository, 0x2000) - next_nonce = manager.ensure_reservation(0x1000, 19) - assert next_nonce == 0x2000 - - assert self.cache_nonce() == "0000000000002033" - assert self.repository.next_free == 0x2033 - - def test_transaction_abort_old_server(self, monkeypatch): - monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20) - - self.repository = self.MockOldRepository() - self.set_cache_nonce("0000000000002000") - - manager = NonceManager(self.repository, 0x2000) - next_nonce = manager.ensure_reservation(0x1000, 19) - assert next_nonce == 0x2000 - - assert self.cache_nonce() == "0000000000002033" - - def test_transaction_abort_on_other_client(self, monkeypatch): - monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20) - - self.repository = self.MockRepository() - self.repository.next_free = 0x2000 - self.set_cache_nonce("0000000000001000") - - manager = NonceManager(self.repository, 0x2000) - next_nonce = manager.ensure_reservation(0x1000, 19) - assert next_nonce == 0x2000 - - assert self.cache_nonce() == "0000000000002033" - assert self.repository.next_free == 0x2033 - - def test_interleaved(self, monkeypatch): - monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20) - - self.repository = self.MockRepository() - self.repository.next_free = 0x2000 - self.set_cache_nonce("0000000000002000") - - manager = NonceManager(self.repository, 0x2000) - next_nonce = manager.ensure_reservation(0x2000, 19) - assert next_nonce == 0x2000 - - assert self.cache_nonce() == "0000000000002033" - assert self.repository.next_free == 0x2033 - - # somehow the clients unlocks, another client reserves and this client relocks - self.repository.next_free = 0x4000 - - # enough space in reservation - next_nonce = manager.ensure_reservation(0x2013, 12) - assert next_nonce == 0x2013 - assert self.cache_nonce() == "0000000000002033" - assert self.repository.next_free == 0x4000 - - # spans reservation boundary - next_nonce = manager.ensure_reservation(0x201F, 21) - assert next_nonce == 0x4000 - assert self.cache_nonce() == "0000000000004035" - assert self.repository.next_free == 0x4035 diff --git a/src/borg/testsuite/repository.py b/src/borg/testsuite/repository.py index 51abc9b2d9..8c44c00f8f 100644 --- a/src/borg/testsuite/repository.py +++ b/src/borg/testsuite/repository.py @@ -613,48 +613,6 @@ def test_exceed_quota(self): assert self.repository.storage_quota_use == len(ch1) + 41 + 8 # now we have compacted. -class NonceReservation(RepositoryTestCaseBase): - def test_get_free_nonce_asserts(self): - self.reopen(exclusive=False) - with pytest.raises(AssertionError): - with self.repository: - self.repository.get_free_nonce() - - def test_get_free_nonce(self): - with self.repository: - assert self.repository.get_free_nonce() is None - - with open(os.path.join(self.repository.path, "nonce"), "w") as fd: - fd.write("0000000000000000") - assert self.repository.get_free_nonce() == 0 - - with open(os.path.join(self.repository.path, "nonce"), "w") as fd: - fd.write("5000000000000000") - assert self.repository.get_free_nonce() == 0x5000000000000000 - - def test_commit_nonce_reservation_asserts(self): - self.reopen(exclusive=False) - with pytest.raises(AssertionError): - with self.repository: - self.repository.commit_nonce_reservation(0x200, 0x100) - - def test_commit_nonce_reservation(self): - with self.repository: - with pytest.raises(Exception): - self.repository.commit_nonce_reservation(0x200, 15) - - self.repository.commit_nonce_reservation(0x200, None) - with open(os.path.join(self.repository.path, "nonce")) as fd: - assert fd.read() == "0000000000000200" - - with pytest.raises(Exception): - self.repository.commit_nonce_reservation(0x200, 15) - - self.repository.commit_nonce_reservation(0x400, 0x200) - with open(os.path.join(self.repository.path, "nonce")) as fd: - assert fd.read() == "0000000000000400" - - class RepositoryAuxiliaryCorruptionTestCase(RepositoryTestCaseBase): def setUp(self): super().setUp()