ARM OSX Migrator

.ci_support/osx_arm64_target_platformosx-arm64.yaml

@@ -0,0 +1,58 @@
+MACOSX_DEPLOYMENT_TARGET:
+- '11.0'
+bzip2:
+- '1'
+c_compiler:
+- clang
+c_compiler_version:
+- '11'
+channel_sources:
+- conda-forge/label/llvm_rc,conda-forge,https://conda-web.anaconda.org/conda-forge
+channel_targets:
+- conda-forge main
+cxx_compiler:
+- clangxx
+cxx_compiler_version:
+- '11'
+libffi:
+- '3.2'
+macos_machine:
+- arm64-apple-darwin20.0.0
+macos_min_version:
+- '11.0'
+openssl:
+- 1.1.1
+pin_run_as_build:
+  bzip2:
+    max_pin: x
+  libffi:
+    max_pin: x.x
+  openssl:
+    max_pin: x.x.x
+  python:
+    min_pin: x.x
+    max_pin: x.x
+  readline:
+    max_pin: x
+  sqlite:
+    max_pin: x
+  tk:
+    max_pin: x.x
+  xz:
+    max_pin: x.x
+  zlib:
+    max_pin: x.x
+python:
+- '3.8'
+readline:
+- '8.0'
+sqlite:
+- '3'
+target_platform:
+- osx-arm64
+tk:
+- '8.6'
+xz:
+- '5.2'
+zlib:
+- '1.2'

conda-forge.yml

@@ -1,5 +1,7 @@
+build_platform: {osx_arm64: osx_64}
 compiler_stack: comp7
+conda_forge_output_validation: true
 max_py_ver: '38'
 max_r_ver: '35'
 provider: {linux_aarch64: native, linux_ppc64le: native, osx: azure, win: azure}
-conda_forge_output_validation: true
+test_on_native_only: true

recipe/build_base.sh

@@ -1,6 +1,9 @@
 #!/bin/bash
 set -ex
 
+# Get an updated config.sub and config.guess
+cp $BUILD_PREFIX/share/libtool/build-aux/config.* .
+
 # The LTO/PGO information was sourced from @pitrou and the Debian rules file in:
 # http://http.debian.net/debian/pool/main/p/python3.6/python3.6_3.6.2-2.debian.tar.xz
 # https://packages.debian.org/source/sid/python3.6
@@ -76,7 +79,7 @@ find "${PREFIX}/lib" -name "libbz2*${SHLIB_EXT}*" | xargs rm -fv {}
 AR=$(basename "${AR}")
 
 # CC must contain the string 'gcc' or else distutils thinks it is on macOS and uses '-R' to set rpaths.
-if [[ ${target_platform} == osx-64 ]]; then
+if [[ ${target_platform} == osx-* ]]; then
   CC=$(basename "${CC}")
 else
   CC=$(basename "${GCC}")
@@ -127,30 +130,30 @@ fi
 
 export CPPFLAGS CFLAGS CXXFLAGS LDFLAGS
 
-if [[ ${target_platform} == osx-64 ]]; then
+if [[ ${target_platform} == osx-* ]]; then
   sed -i -e "s/@OSX_ARCH@/$ARCH/g" Lib/distutils/unixccompiler.py
 fi
 
-if [[ "${BUILD}" != "${HOST}" ]] && [[ -n "${BUILD}" ]] && [[ -n "${HOST}" ]]; then
+if [[ "${CONDA_BUILD_CROSS_COMPILATION}" == "1" ]]; then
   # Build the exact same Python for the build machine. It would be nice (and might be
   # possible already?) to be able to make this just an 'exact' pinned build dependency
   # of a split-package?
   BUILD_PYTHON_PREFIX=${PWD}/build-python-install
   mkdir build-python-build
   pushd build-python-build
     (unset CPPFLAGS LDFLAGS;
-     export CC=/usr/bin/gcc \
-            CXX=/usr/bin/g++ \
-            CPP=/usr/bin/cpp \
+     export CC=${CC_FOR_BUILD} \
+            CXX=${CXX_FOR_BUILD} \
+            CPP="${CC_FOR_BUILD} -E" \
             CFLAGS="-O2" \
-            AR=/usr/bin/ar \
-            RANLIB=/usr/bin/ranlib \
-            LD=/usr/bin/ld && \
+            AR="$(${CC_FOR_BUILD} --print-prog-name=ar)" \
+            RANLIB="$(${CC_FOR_BUILD} --print-prog-name=ranlib)" \
+            LD="$(${CC_FOR_BUILD} --print-prog-name=ld)" && \
       ${SRC_DIR}/configure --build=${BUILD} \
                            --host=${BUILD} \
                            --prefix=${BUILD_PYTHON_PREFIX} \
                            --with-ensurepip=no && \
-      make && \
+      make -j${CPU_COUNT} && \
       make install)
     export PATH=${BUILD_PYTHON_PREFIX}/bin:${PATH}
     ln -s ${BUILD_PYTHON_PREFIX}/bin/python${VER} ${BUILD_PYTHON_PREFIX}/bin/python
@@ -177,6 +180,30 @@ if [[ -n ${HOST} ]]; then
   fi
 fi
 
+if [[ ${target_platform} == osx-64 ]]; then
+  export MACHDEP=darwin
+  export ac_sys_system=Darwin
+  export ac_sys_release=13.4.0
+  export MACOSX_DEFAULT_ARCH=x86_64
+  # TODO: check with LLVM 12 if the following hack is needed.
+  # https://reviews.llvm.org/D76461 may have fixed the need for the following hack.
+  echo '#!/bin/bash' > $BUILD_PREFIX/bin/$HOST-llvm-ar
+  echo "$BUILD_PREFIX/bin/llvm-ar --format=darwin" '"$@"' >> $BUILD_PREFIX/bin/$HOST-llvm-ar
+  chmod +x $BUILD_PREFIX/bin/$HOST-llvm-ar
+elif [[ ${target_platform} == osx-arm64 ]]; then
+  export MACHDEP=darwin
+  export ac_sys_system=Darwin
+  export ac_sys_release=20.0.0
+  export MACOSX_DEFAULT_ARCH=arm64
+  echo '#!/bin/bash' > $BUILD_PREFIX/bin/$HOST-llvm-ar
+  echo "$BUILD_PREFIX/bin/llvm-ar --format=darwin" '"$@"' >> $BUILD_PREFIX/bin/$HOST-llvm-ar
+  chmod +x $BUILD_PREFIX/bin/$HOST-llvm-ar
+elif [[ ${target_platform} == linux-* ]]; then
+  export MACHDEP=linux
+  export ac_sys_system=Linux
+  export ac_sys_release=
+fi
+
 # Not used at present but we should run 'make test' and finish up TESTOPTS (see debians rules).
 declare -a TEST_EXCLUDES
 TEST_EXCLUDES+=(test_ensurepip test_venv)
@@ -210,16 +237,18 @@ _common_configure_args+=("--with-tcltk-libs=-L${PREFIX}/lib -ltcl8.6 -ltk8.6")
 # Add more optimization flags for the static Python interpreter:
 declare -a PROFILE_TASK=()
 if [[ ${_OPTIMIZED} == yes ]]; then
-  _common_configure_args+=(--enable-optimizations)
   _common_configure_args+=(--with-lto)
-  _MAKE_TARGET=profile-opt
-  # To speed up build times during testing (1):
-  if [[ ${QUICK_BUILD} == yes ]]; then
-    # TODO :: It seems this is just profiling everything, on Windows, only 40 odd tests are
-    #         run while on Unix, all 400+ are run, making this slower and less well curated
-    _PROFILE_TASK+=(PROFILE_TASK="-m test --pgo")
-  else
-    _PROFILE_TASK+=(PROFILE_TASK="-m test --pgo-extended")
+  if [[ "$CONDA_BUILD_CROSS_COMPILATION" != "1" ]]; then
+    _common_configure_args+=(--enable-optimizations)
+    _MAKE_TARGET=profile-opt
+    # To speed up build times during testing (1):
+    if [[ ${QUICK_BUILD} == yes ]]; then
+      # TODO :: It seems this is just profiling everything, on Windows, only 40 odd tests are
+      #         run while on Unix, all 400+ are run, making this slower and less well curated
+      _PROFILE_TASK+=(PROFILE_TASK="-m test --pgo")
+    else
+      _PROFILE_TASK+=(PROFILE_TASK="-m test --pgo-extended")
+    fi
   fi
   if [[ ${CC} =~ .*gcc.* ]]; then
     LTO_CFLAGS+=(-fuse-linker-plugin)
@@ -412,4 +441,5 @@ fi
 # There are some strange distutils files around. Delete them
 rm -rf ${PREFIX}/lib/python${VER}/distutils/command/*.exe
 
+python -c "import compileall,os;compileall.compile_dir(os.environ['PREFIX'])"
 rm ${PREFIX}/lib/libpython${VER}.a

recipe/meta.yaml

@@ -44,6 +44,11 @@ source:
       # Python 3.8 is pretty broken without this.
       - patches/0022-Add-CondaEcosystemModifyDllSearchPath.patch
       - patches/0026-Revert-part-of-https-bugs.python.org-issue33895-http.patch
+      - patches/0027-cross-compile-darwin.patch
+      - patches/gh21114.patch
+      - patches/gh21241.patch
+      - patches/gh21228.patch
+      - patches/gh21564.patch  # [osx and arm64]
 
   # TODO :: Depend on our own packages for these:
   - url: https://github.com/python/cpython-source-deps/archive/xz-5.2.2.zip          # [win]
@@ -76,12 +81,11 @@ source:
     folder: externals/libffi                                                         # [win]
     sha256: 4872e72e188a5aa1124db0c3b163a4163e84ead359a514d86dd7c6fa2d2ff02a         # [win]
 
-
 build:
-  number: 3
+  number: 4
   script_env:
-   - PY_INTERP_LINKAGE_NATURE
-   - PY_INTERP_DEBUG
+    - PY_INTERP_LINKAGE_NATURE
+    - PY_INTERP_DEBUG
 
 outputs:
   - name: python
@@ -106,6 +110,9 @@ outputs:
       #   - lib/python3.8/lib-dynload/_hashlib.cpython-38-x86_64-linux-gnu.so  # [linux]
       #   - lib/libpython3.dylib  # [osx]
       # match python.org compiler standard
+      skip: True            # [win and int(float(vc)) < 14]
+      skip_compile_pyc:
+        - '*.py'            # [osx and arm64]
       # Would like to be able to append to the calculated build string here, ping @msarahan, is this possible?
       string: h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}{{ linkage_nature }}{{ debug }}_cpython
       run_exports:
@@ -137,6 +144,7 @@ outputs:
         - posix  # [win]
         - ld_impl_{{ target_platform }} >=2.34  # [linux]
         - binutils_impl_{{ target_platform }} >=2.34  # [linux]
+        - libtool  # [unix]
       host:
         - bzip2  # [not win]
         - sqlite
@@ -209,6 +217,7 @@ outputs:
         -   popd
         - popd
         - test ! -f default.profraw   # [osx]
+
   - name: libpython-static
     build:
       activate_in_script: True

recipe/patches/0027-cross-compile-darwin.patch

@@ -0,0 +1,45 @@
+--- a/configure	2020-08-14 02:59:05.841967550 +0000
++++ b/configure	2020-08-14 03:02:01.839782466 +0000
+@@ -3312,6 +3312,8 @@
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: \"$MACHDEP\"" >&5
+ $as_echo "\"$MACHDEP\"" >&6; }
+
++if test -z "${_PYTHON_HOST_PLATFORM}"
++then
+ if test "$cross_compiling" = yes; then
+ 	case "$host" in
+ 	*-*-linux*)
+@@ -3336,6 +3338,7 @@
+ 	esac
+ 	_PYTHON_HOST_PLATFORM="$MACHDEP${_host_cpu:+-$_host_cpu}"
+ fi
++fi
+
+ # Some systems cannot stand _XOPEN_SOURCE being defined at all; they
+ # disable features if it is defined, without any means to access these
+@@ -6172,7 +6175,7 @@
+ if test "$cross_compiling" = yes; then
+     case "$READELF" in
+ 	readelf|:)
+-	as_fn_error $? "readelf for the host is required for cross builds" "$LINENO" 5
++	#as_fn_error $? "readelf for the host is required for cross builds" "$LINENO" 5
+ 	;;
+     esac
+ fi
+@@ -9244,7 +9247,7 @@
+   conftest.$ac_objext conftest.beam conftest.$ac_ext
+ fi
+
+-
++    if test -z "${MACOSX_DEFAULT_ARCH}"; then
+     if test "${ac_osx_32bit}" = "yes"; then
+     	case `/usr/bin/arch` in
+     	i386)
+@@ -9271,6 +9274,7 @@
+     	esac
+
+     fi
++    fi
+
+     LIBTOOL_CRUFT=$LIBTOOL_CRUFT" -lSystem -lSystemStubs -arch_only ${MACOSX_DEFAULT_ARCH}"
+     LIBTOOL_CRUFT=$LIBTOOL_CRUFT' -install_name $(PYTHONFRAMEWORKINSTALLDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK)'

recipe/patches/gh21114.patch

@@ -0,0 +1,40 @@
+From 2add6fb11b0eddf459c2e79ec86376a8adb00cfc Mon Sep 17 00:00:00 2001
+From: Ronald Oussoren <ronaldoussoren@mac.com>
+Date: Wed, 24 Jun 2020 14:22:16 +0200
+Subject: [PATCH] BPO-41101: Support "arm64" in Mac/Tools/pythonw
+
+---
+ Mac/Tools/pythonw.c                                         | 6 ++++++
+ .../next/macOS/2020-06-24-14-21-17.bpo-41101.z9hCsP.rst     | 2 ++
+ 2 files changed, 8 insertions(+)
+ create mode 100644 Misc/NEWS.d/next/macOS/2020-06-24-14-21-17.bpo-41101.z9hCsP.rst
+
+diff --git a/Mac/Tools/pythonw.c b/Mac/Tools/pythonw.c
+index 1d2db383f943c..21e79665442b6 100644
+--- a/Mac/Tools/pythonw.c
++++ b/Mac/Tools/pythonw.c
+@@ -119,10 +119,16 @@ setup_spawnattr(posix_spawnattr_t* spawnattr)
+
+ #elif defined(__ppc__)
+     cpu_types[0] = CPU_TYPE_POWERPC;
++
+ #elif defined(__i386__)
+     cpu_types[0] = CPU_TYPE_X86;
++
++#elif defined(__arm64__)
++    cpu_types[0] = CPU_TYPE_ARM64;
++
+ #else
+ #       error "Unknown CPU"
++
+ #endif
+
+     if (posix_spawnattr_setbinpref_np(spawnattr, count,
+diff --git a/Misc/NEWS.d/next/macOS/2020-06-24-14-21-17.bpo-41101.z9hCsP.rst b/Misc/NEWS.d/next/macOS/2020-06-24-14-21-17.bpo-41101.z9hCsP.rst
+new file mode 100644
+index 0000000000000..f66863db00c06
+--- /dev/null
++++ b/Misc/NEWS.d/next/macOS/2020-06-24-14-21-17.bpo-41101.z9hCsP.rst
+@@ -0,0 +1,2 @@
++Support the new "arm64" architecture for macOS in the pythonw executable in
++framework builds.