test_basic_functionality racy #6573
Description
Have you checked borgbackup docs, FAQ, and open Github issues?
- docs
- FAQ
- GitHub Issues
Is this a BUG / ISSUE report or a QUESTION?
Bug (test race condition)
System information. For client/server mode post info for both machines.
Your borg version (borg -V).
1.2.0
Operating system (distribution) and version.
NixOS Unstable
Hardware / network configuration, and filesystems used.
Multiple hosts (6C, 12C, 24C, 64C)
How much data is handled by borg?
n/a
Full borg commandline that lead to the problem (leave away excludes and passwords)
Running the tests using pytest --benchmark-skip --numprocesses=auto --pyargs borg.testsuite.
Describe the problem you're observing.
The single mentioned test is sometimes failing with high xdist parallelism.
Can you reproduce the problem? If so, describe how. If not, describe troubleshooting steps you took before opening the issue.
The more processes pytest can use, the more reliable it happens.
Include any warning/errors/backtraces from the system logs
________________ DiffArchiverTestCase.test_basic_functionality _________________
[gw0] linux -- Python 3.9.11 /nix/store/k1physzalj5vffsvl7ag6h6b6vaqip5x-python3-3.9.11/bin/python3.9
self = <borg.testsuite.archiver.DiffArchiverTestCase testMethod=test_basic_functionality>
def test_basic_functionality(self):
# Setup files for the first snapshot
self.create_regular_file('empty', size=0)
self.create_regular_file('file_unchanged', size=128)
self.create_regular_file('file_removed', size=256)
self.create_regular_file('file_removed2', size=512)
self.create_regular_file('file_replaced', size=1024)
os.mkdir('input/dir_replaced_with_file')
os.chmod('input/dir_replaced_with_file', stat.S_IFDIR | 0o755)
os.mkdir('input/dir_removed')
if are_symlinks_supported():
os.mkdir('input/dir_replaced_with_link')
os.symlink('input/dir_replaced_with_file', 'input/link_changed')
os.symlink('input/file_unchanged', 'input/link_removed')
os.symlink('input/file_removed2', 'input/link_target_removed')
os.symlink('input/empty', 'input/link_target_contents_changed')
os.symlink('input/empty', 'input/link_replaced_by_file')
if are_hardlinks_supported():
os.link('input/file_replaced', 'input/hardlink_target_replaced')
os.link('input/empty', 'input/hardlink_contents_changed')
os.link('input/file_removed', 'input/hardlink_removed')
os.link('input/file_removed2', 'input/hardlink_target_removed')
self.cmd('init', '--encryption=repokey', self.repository_location)
# Create the first snapshot
self.cmd('create', self.repository_location + '::test0', 'input')
# Setup files for the second snapshot
self.create_regular_file('file_added', size=2048)
self.create_regular_file('file_empty_added', size=0)
os.unlink('input/file_replaced')
self.create_regular_file('file_replaced', contents=b'0' * 4096)
os.unlink('input/file_removed')
os.unlink('input/file_removed2')
os.rmdir('input/dir_replaced_with_file')
self.create_regular_file('dir_replaced_with_file', size=8192)
os.chmod('input/dir_replaced_with_file', stat.S_IFREG | 0o755)
os.mkdir('input/dir_added')
os.rmdir('input/dir_removed')
if are_symlinks_supported():
os.rmdir('input/dir_replaced_with_link')
os.symlink('input/dir_added', 'input/dir_replaced_with_link')
os.unlink('input/link_changed')
os.symlink('input/dir_added', 'input/link_changed')
os.symlink('input/dir_added', 'input/link_added')
os.unlink('input/link_replaced_by_file')
self.create_regular_file('link_replaced_by_file', size=16384)
os.unlink('input/link_removed')
if are_hardlinks_supported():
os.unlink('input/hardlink_removed')
os.link('input/file_added', 'input/hardlink_added')
with open('input/empty', 'ab') as fd:
fd.write(b'appended_data')
# Create the second snapshot
self.cmd('create', self.repository_location + '::test1a', 'input')
self.cmd('create', '--chunker-params', '16,18,17,4095', self.repository_location + '::test1b', 'input')
def do_asserts(output, can_compare_ids):
# File contents changed (deleted and replaced with a new file)
change = 'B' if can_compare_ids else '{:<19}'.format('modified')
assert 'file_replaced' in output # added to debug #3494
assert '{} input/file_replaced'.format(change) in output
# File unchanged
assert 'input/file_unchanged' not in output
# Directory replaced with a regular file
if 'BORG_TESTS_IGNORE_MODES' not in os.environ:
assert '[drwxr-xr-x -> -rwxr-xr-x] input/dir_replaced_with_file' in output
# Basic directory cases
assert 'added directory input/dir_added' in output
assert 'removed directory input/dir_removed' in output
if are_symlinks_supported():
# Basic symlink cases
assert 'changed link input/link_changed' in output
assert 'added link input/link_added' in output
assert 'removed link input/link_removed' in output
# Symlink replacing or being replaced
assert '] input/dir_replaced_with_link' in output
assert '] input/link_replaced_by_file' in output
# Symlink target removed. Should not affect the symlink at all.
assert 'input/link_target_removed' not in output
# The inode has two links and the file contents changed. Borg
# should notice the changes in both links. However, the symlink
# pointing to the file is not changed.
change = '0 B' if can_compare_ids else '{:<19}'.format('modified')
assert '{} input/empty'.format(change) in output
if are_hardlinks_supported():
assert '{} input/hardlink_contents_changed'.format(change) in output
if are_symlinks_supported():
assert 'input/link_target_contents_changed' not in output
# Added a new file and a hard link to it. Both links to the same
# inode should appear as separate files.
assert 'added 2.05 kB input/file_added' in output
if are_hardlinks_supported():
assert 'added 2.05 kB input/hardlink_added' in output
# check if a diff between non-existent and empty new file is found
assert 'added 0 B input/file_empty_added' in output
# The inode has two links and both of them are deleted. They should
# appear as two deleted files.
assert 'removed 256 B input/file_removed' in output
if are_hardlinks_supported():
assert 'removed 256 B input/hardlink_removed' in output
# Another link (marked previously as the source in borg) to the
# same inode was removed. This should not change this link at all.
if are_hardlinks_supported():
assert 'input/hardlink_target_removed' not in output
# Another link (marked previously as the source in borg) to the
# same inode was replaced with a new regular file. This should not
# change this link at all.
if are_hardlinks_supported():
assert 'input/hardlink_target_replaced' not in output
def do_json_asserts(output, can_compare_ids):
def get_changes(filename, data):
chgsets = [j['changes'] for j in data if j['path'] == filename]
assert len(chgsets) < 2
# return a flattened list of changes for given filename
return [chg for chgset in chgsets for chg in chgset]
# convert output to list of dicts
joutput = [json.loads(line) for line in output.split('\n') if line]
# File contents changed (deleted and replaced with a new file)
expected = {'type': 'modified', 'added': 4096, 'removed': 1024} if can_compare_ids else {'type': 'modified'}
assert expected in get_changes('input/file_replaced', joutput)
# File unchanged
assert not any(get_changes('input/file_unchanged', joutput))
# Directory replaced with a regular file
if 'BORG_TESTS_IGNORE_MODES' not in os.environ:
assert {'type': 'mode', 'old_mode': 'drwxr-xr-x', 'new_mode': '-rwxr-xr-x'} in \
get_changes('input/dir_replaced_with_file', joutput)
# Basic directory cases
assert {'type': 'added directory'} in get_changes('input/dir_added', joutput)
assert {'type': 'removed directory'} in get_changes('input/dir_removed', joutput)
if are_symlinks_supported():
# Basic symlink cases
assert {'type': 'changed link'} in get_changes('input/link_changed', joutput)
assert {'type': 'added link'} in get_changes('input/link_added', joutput)
assert {'type': 'removed link'} in get_changes('input/link_removed', joutput)
# Symlink replacing or being replaced
assert any(chg['type'] == 'mode' and chg['new_mode'].startswith('l') for chg in
get_changes('input/dir_replaced_with_link', joutput))
assert any(chg['type'] == 'mode' and chg['old_mode'].startswith('l') for chg in
get_changes('input/link_replaced_by_file', joutput))
# Symlink target removed. Should not affect the symlink at all.
assert not any(get_changes('input/link_target_removed', joutput))
# The inode has two links and the file contents changed. Borg
# should notice the changes in both links. However, the symlink
# pointing to the file is not changed.
expected = {'type': 'modified', 'added': 13, 'removed': 0} if can_compare_ids else {'type': 'modified'}
assert expected in get_changes('input/empty', joutput)
if are_hardlinks_supported():
assert expected in get_changes('input/hardlink_contents_changed', joutput)
if are_symlinks_supported():
assert not any(get_changes('input/link_target_contents_changed', joutput))
# Added a new file and a hard link to it. Both links to the same
# inode should appear as separate files.
assert {'type': 'added', 'size': 2048} in get_changes('input/file_added', joutput)
if are_hardlinks_supported():
assert {'type': 'added', 'size': 2048} in get_changes('input/hardlink_added', joutput)
# check if a diff between non-existent and empty new file is found
assert {'type': 'added', 'size': 0} in get_changes('input/file_empty_added', joutput)
# The inode has two links and both of them are deleted. They should
# appear as two deleted files.
assert {'type': 'removed', 'size': 256} in get_changes('input/file_removed', joutput)
if are_hardlinks_supported():
assert {'type': 'removed', 'size': 256} in get_changes('input/hardlink_removed', joutput)
# Another link (marked previously as the source in borg) to the
# same inode was removed. This should not change this link at all.
if are_hardlinks_supported():
assert not any(get_changes('input/hardlink_target_removed', joutput))
# Another link (marked previously as the source in borg) to the
# same inode was replaced with a new regular file. This should not
# change this link at all.
if are_hardlinks_supported():
assert not any(get_changes('input/hardlink_target_replaced', joutput))
> do_asserts(self.cmd('diff', self.repository_location + '::test0', 'test1a'), True)
/nix/store/nah87xgdx09l55v5755b5zxaahn4kzp8-borgbackup-1.2.0/lib/python3.9/site-packages/borg/testsuite/archiver.py:4286:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
output = ' +13 B 0 B input/empty\n +13 B 0 B input/hardlink_contents_changed\n +4.1 kB -1.0 kB [-rw-r--r--...moved 512 B input/file_removed2\nremoved directory input/dir_removed\nremoved link input/link_removed\n'
can_compare_ids = True
def do_asserts(output, can_compare_ids):
# File contents changed (deleted and replaced with a new file)
change = 'B' if can_compare_ids else '{:<19}'.format('modified')
assert 'file_replaced' in output # added to debug #3494
> assert '{} input/file_replaced'.format(change) in output
E AssertionError: assert 'B input/file_replaced' in ' +13 B 0 B input/empty\n +13 B 0 B input/hardlink_contents_changed\n +4.1 kB -1.0 kB [-rw-r--r--...moved 512 B input/file_removed2\nremoved directory input/dir_removed\nremoved link input/link_removed\n'
E + where 'B input/file_replaced' = <built-in method format of str object at 0x7fffef644760>('B')
E + where <built-in method format of str object at 0x7fffef644760> = '{} input/file_replaced'.format
/nix/store/nah87xgdx09l55v5755b5zxaahn4kzp8-borgbackup-1.2.0/lib/python3.9/site-packages/borg/testsuite/archiver.py:4147: AssertionError