Optimize Multiplatform cache builds

.github/workflows/ci.yml

@@ -1583,8 +1583,13 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
         run: ./scripts/ci/images/ci_build_ci_image_on_ci.sh
         env:
           PREPARE_BUILDX_CACHE: "true"
+      - name: "Start ARM instance"
+        run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
       - name: "Build CI image cache and push ${{env.PYTHON_MAJOR_MINOR_VERSION}}"
         run: ./scripts/ci/images/ci_build_prod_image_on_ci.sh
         env:
           VERSION_SUFFIX_FOR_PYPI: ".dev0"
           PREPARE_BUILDX_CACHE: "true"
+      - name: "Stop ARM instance"
+        run: ./scripts/ci/images/ci_stop_arm_instance.sh
+        if: always()

scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+
+SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)"
+# This is an AMI that is based on Basic Amazon Linux AMI with installed and configured docker service
+WORKING_DIR="/tmp/armdocker"
+INSTANCE_INFO="${WORKING_DIR}/instance_info.json"
+ARM_AMI="ami-002fa24639ab2520a"
+INSTANCE_TYPE="c6gd.medium"
+MARKET_OPTIONS="MarketType=spot,SpotOptions={MaxPrice=0.1,SpotInstanceType=one-time}"
+REGION="us-east-2"
+EC2_USER="ec2-user"
+USER_DATA_FILE="${SCRIPTS_DIR}/self_terminate.sh"
+
+function start_arm_instance() {
+    set -x
+    mkdir -p "${WORKING_DIR}"
+    cd "${WORKING_DIR}"
+    aws ec2 run-instances \
+        --region "${REGION}" \
+        --image-id "${ARM_AMI}" \
+        --count 1 \
+        --instance-type "${INSTANCE_TYPE}" \
+        --user-data "file://${USER_DATA_FILE}" \
+        --instance-market-options "${MARKET_OPTIONS}" \
+        --instance-initiated-shutdown-behavior terminate \
+        --output json \
+        > "${INSTANCE_INFO}"
+
+    INSTANCE_ID=$(jq < "${INSTANCE_INFO}" ".Instances[0].InstanceId" -r)
+    AVAILABILITY_ZONE=$(jq < "${INSTANCE_INFO}" ".Instances[0].Placement.AvailabilityZone" -r)
+    SECURITY_GROUP=$(jq < "${INSTANCE_INFO}" ".Instances[0].NetworkInterfaces[0].Groups[0].GroupId" -r)
+
+    aws ec2 wait instance-status-ok --instance-ids "${INSTANCE_ID}"
+    INSTANCE_PUBLIC_DNS_NAME=$(aws ec2 describe-instances \
+        --filters "Name=instance-state-name,Values=running" "Name=instance-id,Values=${INSTANCE_ID}" \
+        --query 'Reservations[*].Instances[*].PublicDnsName' --output text)
+    rm -f my_key
+    ssh-keygen -t rsa -f my_key -N ""
+    aws ec2-instance-connect send-ssh-public-key --instance-id "${INSTANCE_ID}" \
+        --availability-zone "${AVAILABILITY_ZONE}" \
+        --instance-os-user "${EC2_USER}" \
+        --ssh-public-key file://my_key.pub
+    aws ec2 authorize-security-group-ingress --region "${REGION}" --group-id "${SECURITY_GROUP}" \
+        --protocol tcp --port 22 --cidr "0.0.0.0/0" || true
+    autossh -f -L12357:/var/run/docker.sock \
+        -o "IdentitiesOnly=yes" -o "StrictHostKeyChecking=no" \
+        -i my_key \
+        "${EC2_USER}@${INSTANCE_PUBLIC_DNS_NAME}"
+    docker buildx create --name airflow_cache --append localhost:12357
+    docker buildx ls
+}
+
+start_arm_instance

scripts/ci/images/ci_stop_arm_instance.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+
+# This is an AMI that is based on Basic Amazon Linux AMI with installed and configured docker service
+WORKING_DIR="/tmp/armdocker"
+INSTANCE_INFO="${WORKING_DIR}/instance_info.json"
+
+function stop_arm_instance() {
+    INSTANCE_ID=$(jq < "${INSTANCE_INFO}" ".Instances[0].InstanceId" -r)
+    aws ec2 stop-instances --instance-ids "${INSTANCE_ID}"
+}
+
+stop_arm_instance

scripts/ci/images/self_terminate.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This instance will run for maximum 50 minutes and
+# It will terminate itself after that (it can also
+# be terminated immediately when the job finishes)
+echo "sudo shutdown -h now" | at now +50 min