ctypes: use the correct ABI for variadic functions

On arm64 the calling convention for variardic functions is different
than the convention for fixed-arg functions of the same arg types.

ctypes needs to use ffi_prep_cif_var to tell libffi which calling
convention to use.

This patch adds a new attribute "f.variadic" for ctypes function
pointers, so users can specify which functions are variadic.

It will also auto-detect varargs when a function is called with more
arguments than f.argtypes specifies.   Since this is a new option
and it only matters on arm64 on apple platforms, lots of existing code
will not set f.variadic.   This will do the right thing in most
situations.

Author: lawrence-danna-apple

Doc/library/ctypes.rst

@@ -1579,6 +1579,13 @@ They are instances of a private class:
       value usable as argument (integer, string, ctypes instance).  This allows
       defining adapters that can adapt custom objects as function parameters.
 
+   .. attribute:: variadic
+
+      Assign a boolean to specify that the function takes a variable nubmer of
+      arguments.   This does not matter on most platforms, but for Apple arm64
+      platforms variadic functions have a different calling convention than
+      normal functions.
+
    .. attribute:: errcheck
 
       Assign a Python function or another callable to this attribute. The

Lib/test/test_bytes.py

@@ -1034,6 +1034,8 @@ def test_from_format(self):
             c_char_p)
 
         PyBytes_FromFormat = pythonapi.PyBytes_FromFormat
+        PyBytes_FromFormat.variadic = True
+        PyBytes_FromFormat.argtypes = (c_char_p,)
         PyBytes_FromFormat.restype = py_object
 
         # basic tests

Lib/test/test_unicode.py

@@ -2509,11 +2509,14 @@ class CAPITest(unittest.TestCase):
     def test_from_format(self):
         support.import_module('ctypes')
         from ctypes import (
+            c_char_p,
             pythonapi, py_object, sizeof,
             c_int, c_long, c_longlong, c_ssize_t,
             c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p)
         name = "PyUnicode_FromFormat"
         _PyUnicode_FromFormat = getattr(pythonapi, name)
+        _PyUnicode_FromFormat.argtypes = (c_char_p,)
+        _PyUnicode_FromFormat.variadic = True
         _PyUnicode_FromFormat.restype = py_object
 
         def PyUnicode_FromFormat(format, *args):

Misc/NEWS.d/next/Core and Builtins/2020-06-30-18-26-26.bpo-41100.rcZuB5.rst

@@ -0,0 +1 @@
+ctypes: Mac OS 11: use correct ABI for variadic functions on arm64

Modules/_ctypes/_ctypes.c

@@ -3320,6 +3320,35 @@ PyCFuncPtr_get_restype(PyCFuncPtrObject *self, void *Py_UNUSED(ignored))
     }
 }
 
+static int
+PyCFuncPtr_set_variadic(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ignored))
+{
+    StgDictObject *dict = PyObject_stgdict((PyObject *)self);
+    assert(dict);
+    int r = PyObject_IsTrue(ob);
+    if (r == 1) {
+        dict->flags |= FUNCFLAG_VARIADIC;
+        return 0;
+    } else if (r == 0) {
+        dict->flags &= ~FUNCFLAG_VARIADIC;
+        return 0;
+    } else {
+        return -1;
+    }
+}
+
+static PyObject *
+PyCFuncPtr_get_variadic(PyCFuncPtrObject *self, void *Py_UNUSED(ignored))
+{
+    StgDictObject *dict = PyObject_stgdict((PyObject *)self);
+    assert(dict); /* Cannot be NULL for PyCFuncPtrObject instances */
+    if (dict->flags & FUNCFLAG_VARIADIC)
+        Py_RETURN_TRUE;
+    else
+        Py_RETURN_FALSE;
+}
+
+
 static int
 PyCFuncPtr_set_argtypes(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ignored))
 {
@@ -3365,6 +3394,8 @@ static PyGetSetDef PyCFuncPtr_getsets[] = {
     { "argtypes", (getter)PyCFuncPtr_get_argtypes,
       (setter)PyCFuncPtr_set_argtypes,
       "specify the argument types", NULL },
+    { "variadic", (getter)PyCFuncPtr_get_variadic, (setter)PyCFuncPtr_set_variadic,
+      "specify if function takes variable number of arguments", NULL },
     { NULL, NULL }
 };
 
@@ -5839,6 +5870,7 @@ PyInit__ctypes(void)
     PyModule_AddObject(m, "FUNCFLAG_USE_ERRNO", PyLong_FromLong(FUNCFLAG_USE_ERRNO));
     PyModule_AddObject(m, "FUNCFLAG_USE_LASTERROR", PyLong_FromLong(FUNCFLAG_USE_LASTERROR));
     PyModule_AddObject(m, "FUNCFLAG_PYTHONAPI", PyLong_FromLong(FUNCFLAG_PYTHONAPI));
+    PyModule_AddObject(m, "FUNCFLAG_VARIADIC", PyLong_FromLong(FUNCFLAG_VARIADIC));
     PyModule_AddStringConstant(m, "__version__", "1.1.0");
 
     PyModule_AddObject(m, "_memmove_addr", PyLong_FromVoidPtr(memmove));

Modules/_ctypes/callproc.c

@@ -86,6 +86,10 @@
 #define DONT_USE_SEH
 #endif
 
+#if defined(__APPLE__) && __arm64__
+#define HAVE_FFI_PREP_CIF_VAR 1
+#endif
+
 #define CTYPES_CAPSULE_NAME_PYMEM "_ctypes pymem"
 
 static void pymem_destructor(PyObject *ptr)
@@ -812,7 +816,8 @@ static int _call_function_pointer(int flags,
                                   ffi_type **atypes,
                                   ffi_type *restype,
                                   void *resmem,
-                                  int argcount)
+                                  int argcount,
+                                  int argtypecount)
 {
     PyThreadState *_save = NULL; /* For Py_BLOCK_THREADS and Py_UNBLOCK_THREADS */
     PyObject *error_object = NULL;
@@ -835,15 +840,39 @@ static int _call_function_pointer(int flags,
     if ((flags & FUNCFLAG_CDECL) == 0)
         cc = FFI_STDCALL;
 #endif
-    if (FFI_OK != ffi_prep_cif(&cif,
-                               cc,
-                               argcount,
-                               restype,
-                               atypes)) {
-        PyErr_SetString(PyExc_RuntimeError,
-                        "ffi_prep_cif failed");
-        return -1;
+
+#if HAVE_FFI_PREP_CIF_VAR
+    /* Everyone SHOULD set f.variadic=True on variadic function pointers, but
+     * lots of existing code will not.  If there's at least one arg and more
+     * args are passed than are defined in the prototype, then it must be a
+     * variadic function. */
+    if ((flags & FUNCFLAG_VARIADIC) ||
+        (argtypecount != 0 && argcount > argtypecount))
+    {
+        if (FFI_OK != ffi_prep_cif_var(&cif,
+                                       cc,
+                                       argtypecount,
+                                       argcount,
+                                       restype,
+                                       atypes)) {
+            PyErr_SetString(PyExc_RuntimeError,
+                            "ffi_prep_cif_var failed");
+            return -1;
+        }
+    } else {
+#endif
+        if (FFI_OK != ffi_prep_cif(&cif,
+                                   cc,
+                                   argcount,
+                                   restype,
+                                   atypes)) {
+            PyErr_SetString(PyExc_RuntimeError,
+                            "ffi_prep_cif failed");
+            return -1;
+        }
+#if HAVE_FFI_PREP_CIF_VAR
     }
+#endif
 
     if (flags & (FUNCFLAG_USE_ERRNO | FUNCFLAG_USE_LASTERROR)) {
         error_object = _ctypes_get_errobj(&space);
@@ -1212,9 +1241,8 @@ PyObject *_ctypes_callproc(PPROC pProc,
 
     if (-1 == _call_function_pointer(flags, pProc, avalues, atypes,
                                      rtype, resbuf,
-                                     Py_SAFE_DOWNCAST(argcount,
-                                                      Py_ssize_t,
-                                                      int)))
+                                     Py_SAFE_DOWNCAST(argcount, Py_ssize_t, int),
+                                     Py_SAFE_DOWNCAST(argtype_count, Py_ssize_t, int)))
         goto cleanup;
 
 #ifdef WORDS_BIGENDIAN

Modules/_ctypes/ctypes.h

@@ -285,6 +285,7 @@ PyObject *_ctypes_callproc(PPROC pProc,
 #define FUNCFLAG_PYTHONAPI 0x4
 #define FUNCFLAG_USE_ERRNO 0x8
 #define FUNCFLAG_USE_LASTERROR 0x10
+#define FUNCFLAG_VARIADIC 0x20
 
 #define TYPEFLAG_ISPOINTER 0x100
 #define TYPEFLAG_HASPOINTER 0x200