xattr: move to platform pkg, use cython, use bytes, use fd

src/borg/archive.py

@@ -1,4 +1,3 @@
-import errno
 import json
 import os
 import socket
@@ -723,33 +722,9 @@ def restore_attrs(self, path, item, symlink=False, fd=None):
         acl_set(path, item, self.numeric_owner)
         # chown removes Linux capabilities, so set the extended attributes at the end, after chown, since they include
         # the Linux capabilities in the "security.capability" attribute.
-        xattrs = item.get('xattrs', {})
-        for k, v in xattrs.items():
-            try:
-                xattr.setxattr(fd or path, k, v, follow_symlinks=False)
-            except OSError as e:
-                if e.errno == errno.E2BIG:
-                    # xattr is too big
-                    logger.warning('%s: Value or key of extended attribute %s is too big for this filesystem' %
-                                   (path, k.decode()))
-                    set_ec(EXIT_WARNING)
-                elif e.errno == errno.ENOTSUP:
-                    # xattrs not supported here
-                    logger.warning('%s: Extended attributes are not supported on this filesystem' % path)
-                    set_ec(EXIT_WARNING)
-                elif e.errno == errno.EACCES:
-                    # permission denied to set this specific xattr (this may happen related to security.* keys)
-                    logger.warning('%s: Permission denied when setting extended attribute %s' % (path, k.decode()))
-                    set_ec(EXIT_WARNING)
-                elif e.errno == errno.ENOSPC:
-                    # no space left on device while setting this specific xattr
-                    # ext4 reports ENOSPC when trying to set an xattr with >4kiB while ext4 can only support 4kiB xattrs
-                    # (in this case, this is NOT a "disk full" error, just a ext4 limitation).
-                    logger.warning('%s: No space left on device while setting extended attribute %s (len = %d)' % (
-                        path, k.decode(), len(v)))
-                    set_ec(EXIT_WARNING)
-                else:
-                    raise
+        warning = xattr.set_all(fd or path, item.get('xattrs', {}), follow_symlinks=False)
+        if warning:
+            set_ec(EXIT_WARNING)
         # bsdflags include the immutable flag and need to be set last:
         if not self.nobsdflags and 'bsdflags' in item:
             try:
@@ -972,11 +947,11 @@ def stat_simple_attrs(self, st):
             attrs['group'] = gid2group(st.st_gid)
         return attrs
 
-    def stat_ext_attrs(self, st, path):
+    def stat_ext_attrs(self, st, path, fd=None):
         attrs = {}
         bsdflags = 0
         with backup_io('extended stat'):
-            xattrs = xattr.get_all(path, follow_symlinks=False)
+            xattrs = xattr.get_all(fd or path, follow_symlinks=False)
             if not self.nobsdflags:
                 bsdflags = get_flags(path, st)
             acl_get(path, attrs, st, self.numeric_owner)
@@ -986,9 +961,9 @@ def stat_ext_attrs(self, st, path):
             attrs['bsdflags'] = bsdflags
         return attrs
 
-    def stat_attrs(self, st, path):
+    def stat_attrs(self, st, path, fd=None):
         attrs = self.stat_simple_attrs(st)
-        attrs.update(self.stat_ext_attrs(st, path))
+        attrs.update(self.stat_ext_attrs(st, path, fd=fd))
         return attrs
 
 
@@ -1144,6 +1119,7 @@ def process_stdin(self, path, cache):
 
     def process_file(self, path, st, cache):
         with self.create_helper(path, st, None) as (item, status, hardlinked, hardlink_master):  # no status yet
+            md = None
             is_special_file = is_special(st.st_mode)
             if not hardlinked or hardlink_master:
                 if not is_special_file:
@@ -1177,12 +1153,19 @@ def process_file(self, path, st, cache):
                         fh = Archive._open_rb(path)
                     with os.fdopen(fh, 'rb') as fd:
                         self.process_file_chunks(item, cache, self.stats, self.show_progress, backup_io_iter(self.chunker.chunkify(fd, fh)))
+                        md = self.metadata_collector.stat_attrs(st, path, fd=fh)
                     if not is_special_file:
                         # we must not memorize special files, because the contents of e.g. a
                         # block or char device will change without its mtime/size/inode changing.
                         cache.memorize_file(path_hash, st, [c.id for c in item.chunks])
                 self.stats.nfiles += 1
-            item.update(self.metadata_collector.stat_attrs(st, path))
+            if md is None:
+                fh = Archive._open_rb(path)
+                try:
+                    md = self.metadata_collector.stat_attrs(st, path, fd=fh)
+                finally:
+                    os.close(fh)
+            item.update(md)
             item.get_size(memorize=True)
             if is_special_file:
                 # we processed a special file like a regular file. reflect that in mode,

src/borg/platform/__init__.py

@@ -6,6 +6,7 @@
 Public APIs are documented in platform.base.
 """
 
+from .base import listxattr, getxattr, setxattr, ENOATTR
 from .base import acl_get, acl_set
 from .base import set_flags, get_flags
 from .base import SaveFile, SyncFile, sync_dir, fdatasync, safe_fadvise
@@ -21,12 +22,15 @@
 
 if sys.platform.startswith('linux'):  # pragma: linux only
     from .linux import API_VERSION as OS_API_VERSION
+    from .linux import listxattr, getxattr, setxattr
     from .linux import acl_get, acl_set
     from .linux import set_flags, get_flags
     from .linux import SyncFile
 elif sys.platform.startswith('freebsd'):  # pragma: freebsd only
     from .freebsd import API_VERSION as OS_API_VERSION
+    from .freebsd import listxattr, getxattr, setxattr
     from .freebsd import acl_get, acl_set
 elif sys.platform == 'darwin':  # pragma: darwin only
     from .darwin import API_VERSION as OS_API_VERSION
+    from .darwin import listxattr, getxattr, setxattr
     from .darwin import acl_get, acl_set

src/borg/platform/base.py

@@ -17,10 +17,49 @@
 are correctly composed into the base functionality.
 """
 
-API_VERSION = '1.1_03'
+API_VERSION = '1.2_01'
 
 fdatasync = getattr(os, 'fdatasync', os.fsync)
 
+from .xattr import ENOATTR
+
+
+def listxattr(path, *, follow_symlinks=True):
+    """
+    Return xattr names of a file (list of bytes objects).
+
+    *path* can either be a path (bytes) or an open file descriptor (int).
+    *follow_symlinks* indicates whether symlinks should be followed
+    and only applies when *path* is not an open file descriptor.
+    """
+    return []
+
+
+def getxattr(path, name, *, follow_symlinks=True):
+    """
+    Read xattr and return its value (as bytes).
+
+    *path* can either be a path (bytes) or an open file descriptor (int).
+    *name* is the name of the xattr to read (bytes).
+    *follow_symlinks* indicates whether symlinks should be followed
+    and only applies when *path* is not an open file descriptor.
+    """
+    # as this base dummy implementation returns [] from listxattr,
+    # it must raise here for any given name:
+    raise OSError(ENOATTR, os.strerror(ENOATTR), path)
+
+
+def setxattr(path, name, value, *, follow_symlinks=True):
+    """
+    Write xattr on *path*.
+
+    *path* can either be a path (bytes) or an open file descriptor (int).
+    *name* is the name of the xattr to read (bytes).
+    *value* is the value to write (bytes).
+    *follow_symlinks* indicates whether symlinks should be followed
+    and only applies when *path* is not an open file descriptor.
+    """
+
 
 def acl_get(path, item, st, numeric_owner=False):
     """

src/borg/platform/darwin.pyx

@@ -1,9 +1,26 @@
 import os
 
+from libc.stdint cimport uint32_t
+
 from ..helpers import user2uid, group2gid
 from ..helpers import safe_decode, safe_encode
+from .xattr import _listxattr_inner, _getxattr_inner, _setxattr_inner, split_string0
+
+API_VERSION = '1.2_01'
+
+cdef extern from "sys/xattr.h":
+    ssize_t c_listxattr "listxattr" (const char *path, char *list, size_t size, int flags)
+    ssize_t c_flistxattr "flistxattr" (int filedes, char *list, size_t size, int flags)
+
+    ssize_t c_getxattr "getxattr" (const char *path, const char *name, void *value, size_t size, uint32_t pos, int flags)
+    ssize_t c_fgetxattr "fgetxattr" (int filedes, const char *name, void *value, size_t size, uint32_t pos, int flags)
+
+    int c_setxattr "setxattr" (const char *path, const char *name, const void *value, size_t size, uint32_t pos, int flags)
+    int c_fsetxattr "fsetxattr" (int filedes, const char *name, const void *value, size_t size, uint32_t pos, int flags)
+
+    int XATTR_NOFOLLOW
 
-API_VERSION = '1.1_03'
+cdef int XATTR_NOFLAGS = 0x0000
 
 cdef extern from "sys/acl.h":
     ctypedef struct _acl_t:
@@ -18,6 +35,47 @@ cdef extern from "sys/acl.h":
     int ACL_TYPE_EXTENDED
 
 
+def listxattr(path, *, follow_symlinks=True):
+    def func(path, buf, size):
+        if isinstance(path, int):
+            return c_flistxattr(path, <char *> buf, size, XATTR_NOFLAGS)
+        else:
+            if follow_symlinks:
+                return c_listxattr(path, <char *> buf, size, XATTR_NOFLAGS)
+            else:
+                return c_listxattr(path, <char *> buf, size, XATTR_NOFOLLOW)
+
+    n, buf = _listxattr_inner(func, path)
+    return [name for name in split_string0(buf[:n]) if name]
+
+
+def getxattr(path, name, *, follow_symlinks=True):
+    def func(path, name, buf, size):
+        if isinstance(path, int):
+            return c_fgetxattr(path, name, <char *> buf, size, 0, XATTR_NOFLAGS)
+        else:
+            if follow_symlinks:
+                return c_getxattr(path, name, <char *> buf, size, 0, XATTR_NOFLAGS)
+            else:
+                return c_getxattr(path, name, <char *> buf, size, 0, XATTR_NOFOLLOW)
+
+    n, buf = _getxattr_inner(func, path, name)
+    return bytes(buf[:n])
+
+
+def setxattr(path, name, value, *, follow_symlinks=True):
+    def func(path, name, value, size):
+        if isinstance(path, int):
+            return c_fsetxattr(path, name, <char *> value, size, 0, XATTR_NOFLAGS)
+        else:
+            if follow_symlinks:
+                return c_setxattr(path, name, <char *> value, size, 0, XATTR_NOFLAGS)
+            else:
+                return c_setxattr(path, name, <char *> value, size, 0, XATTR_NOFOLLOW)
+
+    _setxattr_inner(func, path, name, value)
+
+
 def _remove_numeric_id_if_possible(acl):
     """Replace the user/group field with the local uid/gid if possible
     """

src/borg/platform/freebsd.pyx

@@ -2,13 +2,29 @@ import os
 
 from ..helpers import posix_acl_use_stored_uid_gid
 from ..helpers import safe_encode, safe_decode
+from .xattr import _listxattr_inner, _getxattr_inner, _setxattr_inner, split_lstring
 
-API_VERSION = '1.1_03'
+API_VERSION = '1.2_01'
 
 cdef extern from "errno.h":
     int errno
     int EINVAL
 
+cdef extern from "sys/extattr.h":
+    ssize_t c_extattr_list_file "extattr_list_file" (const char *path, int attrnamespace, void *data, size_t nbytes)
+    ssize_t c_extattr_list_link "extattr_list_link" (const char *path, int attrnamespace, void *data, size_t nbytes)
+    ssize_t c_extattr_list_fd "extattr_list_fd" (int fd, int attrnamespace, void *data, size_t nbytes)
+
+    ssize_t c_extattr_get_file "extattr_get_file" (const char *path, int attrnamespace, const char *attrname, void *data, size_t nbytes)
+    ssize_t c_extattr_get_link "extattr_get_link" (const char *path, int attrnamespace, const char *attrname, void *data, size_t nbytes)
+    ssize_t c_extattr_get_fd "extattr_get_fd" (int fd, int attrnamespace, const char *attrname, void *data, size_t nbytes)
+
+    int c_extattr_set_file "extattr_set_file" (const char *path, int attrnamespace, const char *attrname, const void *data, size_t nbytes)
+    int c_extattr_set_link "extattr_set_link" (const char *path, int attrnamespace, const char *attrname, const void *data, size_t nbytes)
+    int c_extattr_set_fd "extattr_set_fd" (int fd, int attrnamespace, const char *attrname, const void *data, size_t nbytes)
+
+    int EXTATTR_NAMESPACE_USER
+
 cdef extern from "sys/types.h":
     int ACL_TYPE_ACCESS
     int ACL_TYPE_DEFAULT
@@ -32,6 +48,47 @@ cdef extern from "unistd.h":
     int _PC_ACL_NFS4
 
 
+def listxattr(path, *, follow_symlinks=True):
+    def func(path, buf, size):
+        if isinstance(path, int):
+            return c_extattr_list_fd(path, EXTATTR_NAMESPACE_USER, <char *> buf, size)
+        else:
+            if follow_symlinks:
+                return c_extattr_list_file(path, EXTATTR_NAMESPACE_USER, <char *> buf, size)
+            else:
+                return c_extattr_list_link(path, EXTATTR_NAMESPACE_USER, <char *> buf, size)
+
+    n, buf = _listxattr_inner(func, path)
+    return [name for name in split_lstring(buf[:n]) if name]
+
+
+def getxattr(path, name, *, follow_symlinks=True):
+    def func(path, name, buf, size):
+        if isinstance(path, int):
+            return c_extattr_get_fd(path, EXTATTR_NAMESPACE_USER, name, <char *> buf, size)
+        else:
+            if follow_symlinks:
+                return c_extattr_get_file(path, EXTATTR_NAMESPACE_USER, name, <char *> buf, size)
+            else:
+                return c_extattr_get_link(path, EXTATTR_NAMESPACE_USER, name, <char *> buf, size)
+
+    n, buf = _getxattr_inner(func, path, name)
+    return bytes(buf[:n])
+
+
+def setxattr(path, name, value, *, follow_symlinks=True):
+    def func(path, name, value, size):
+        if isinstance(path, int):
+            return c_extattr_set_fd(path, EXTATTR_NAMESPACE_USER, name, <char *> value, size)
+        else:
+            if follow_symlinks:
+                return c_extattr_set_file(path, EXTATTR_NAMESPACE_USER, name, <char *> value, size)
+            else:
+                return c_extattr_set_link(path, EXTATTR_NAMESPACE_USER, name, <char *> value, size)
+
+    _setxattr_inner(func, path, name, value)
+
+
 cdef _get_acl(p, type, item, attribute, int flags):
     cdef acl_t acl
     cdef char *text