From 8f48ad9b5d59d9a5f2e265f114e2696bbd934a96 Mon Sep 17 00:00:00 2001 From: Honza Pokorny Date: Mon, 18 Nov 2019 15:49:48 -0400 Subject: [PATCH 1/2] Add option to mirror openshift images We add a new `MIRROR_IMAGES` option. When non-empty, it will use `oc adm` to mirror openshift images to our local registry. It will also modify `install-config.yaml` to add the new mirror. --- 04_setup_ironic.sh | 8 ++++++++ common.sh | 1 + config_example.sh | 3 +++ ocp_install_env.sh | 1 + utils.sh | 16 ++++++++++++++++ 5 files changed, 29 insertions(+) diff --git a/04_setup_ironic.sh b/04_setup_ironic.sh index 190466482..45cc01a7f 100755 --- a/04_setup_ironic.sh +++ b/04_setup_ironic.sh @@ -49,6 +49,14 @@ for IMAGE_VAR in $(env | grep "_LOCAL_IMAGE=" | grep -o "^[^=]*") ; do echo "RUN sed -i 's%$OLDIMAGE%${!IMAGE_VAR}%g' /release-manifests/*" >> $DOCKERFILE done +if [ ! -z "${MIRROR_IMAGES}" ]; then + oc adm release mirror \ + --insecure=true \ + -a $REGISTRY_AUTH_FILE \ + --from $OPENSHIFT_RELEASE_IMAGE \ + --to $LOCAL_REGISTRY_ADDRESS/localimages/local-release-image +fi + if [ -f assets/templates/99_local-registry.yaml ] ; then build_installer sudo podman image build -t $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE -f $DOCKERFILE diff --git a/common.sh b/common.sh index 956b4a2d2..4f0de2037 100644 --- a/common.sh +++ b/common.sh @@ -37,6 +37,7 @@ fi source $CONFIG export LOCAL_REGISTRY_ADDRESS=${LOCAL_REGISTRY_ADDRESS:-"192.168.111.1:5000"} +export MIRROR_IMAGES=${MIRROR_IMAGES:-} # # See https://openshift-release.svc.ci.openshift.org for release details diff --git a/config_example.sh b/config_example.sh index 1517e7268..1e8228427 100644 --- a/config_example.sh +++ b/config_example.sh @@ -11,6 +11,9 @@ set -x #export IRONIC_LOCAL_IMAGE=quay.io/username/ironic #export MACHINE_CONFIG_OPERATOR_LOCAL_IMAGE=https://github.com/openshift/machine-config-operator +# Mirror latest ci images to local registry +#export MIRROR_IMAGES=true + # Switch to upstream metal3-io ironic images instead of openshift ones. #export UPSTREAM_IRONIC=true diff --git a/ocp_install_env.sh b/ocp_install_env.sh index 8dcde5232..89b7022b8 100644 --- a/ocp_install_env.sh +++ b/ocp_install_env.sh @@ -102,6 +102,7 @@ platform: dnsVIP: ${DNS_VIP} hosts: $(master_node_map_to_install_config $NUM_MASTERS) +$(image_mirror_config) pullSecret: | $(echo $PULL_SECRET | jq -c .) sshKey: | diff --git a/utils.sh b/utils.sh index b0a7cc000..139f72693 100644 --- a/utils.sh +++ b/utils.sh @@ -195,3 +195,19 @@ function bmo_config_map { cp ocp/deploy/metal3-config.yaml assets/generated/99_metal3-config.yaml } + +function image_mirror_config { + if [ ! -z "${MIRROR_IMAGES}" ]; then + TAGGED=$(echo $OPENSHIFT_RELEASE_IMAGE | sed -e 's/release://') + RELEASE=$(echo $OPENSHIFT_RELEASE_IMAGE | grep -o 'registry.svc.ci.openshift.org[^":]\+') + cat << EOF +imageContentSources: +- mirrors: + - ${LOCAL_REGISTRY_ADDRESS}/localimages/local-release-image + source: ${RELEASE} +- mirrors: + - ${LOCAL_REGISTRY_ADDRESS}/localimages/local-release-image + source: ${TAGGED} +EOF + fi +} From 842b8d57b630208baaff0f8a10038c283137edc8 Mon Sep 17 00:00:00 2001 From: Yurii Prokulevych Date: Mon, 25 Nov 2019 13:55:13 +0100 Subject: [PATCH 2/2] Add SSL support for local mirror. There are issues while using local mirror that doesn't support SSL. Current change addresses this limitation. - [x] create self-signed ssl cert for local mirror - [x] create pull-secret for local mirror - [x] combine locall pull secret with global - [x] add custom ssl-cert to install-config.yml - [x] extract the installation program from the mirrored content - [x] use local pull secret for deployment --- 01_install_requirements.sh | 9 ++- 02_configure_host.sh | 9 ++- 04_setup_ironic.sh | 55 +++++++++++++----- common.sh | 16 ++++-- config_example.sh | 15 ++++- ocp_install_env.sh | 6 ++ utils.sh | 112 ++++++++++++++++++++++++++++++++++++- 7 files changed, 194 insertions(+), 28 deletions(-) diff --git a/01_install_requirements.sh b/01_install_requirements.sh index e95655abe..9271e621c 100755 --- a/01_install_requirements.sh +++ b/01_install_requirements.sh @@ -30,11 +30,6 @@ popd if sudo systemctl is-active docker-distribution.service; then sudo systemctl disable --now docker-distribution.service fi -reg_state=$(sudo podman inspect registry --format "{{.State.Status}}" || echo "error") -if [[ "$reg_state" != "running" ]]; then - sudo podman rm registry -f || true - sudo podman run -d -p 5000:5000 --name registry docker.io/registry:latest -fi # Install oc client oc_version=4.4 @@ -66,3 +61,7 @@ if ! which dep 2>&1 >/dev/null ; then mkdir -p $GOPATH/bin curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh fi + +if [[ ! -z "${MIRROR_IMAGES}" || $(env | grep "_LOCAL_IMAGE=") ]]; then + setup_local_registry +fi diff --git a/02_configure_host.sh b/02_configure_host.sh index 5ac756483..a82f02372 100755 --- a/02_configure_host.sh +++ b/02_configure_host.sh @@ -124,7 +124,7 @@ if [ "$MANAGE_BR_BRIDGE" == "y" ] ; then fi # Add firewall rules to ensure the image caches can be reached on the host -for PORT in 80 5000 ; do +for PORT in 80 ${LOCAL_REGISTRY_PORT} ; do if [ "${RHEL8}" = "True" ] ; then sudo firewall-cmd --zone=libvirt --add-port=$PORT/tcp sudo firewall-cmd --zone=libvirt --add-port=$PORT/tcp --permanent @@ -168,3 +168,10 @@ if [ "$MANAGE_BR_BRIDGE" == "y" ] ; then sudo systemctl restart NetworkManager fi fi + +if [[ ! -z "${MIRROR_IMAGES}" || $(env | grep "_LOCAL_IMAGE=") ]]; then + # create authfile for local registry + sudo podman login --authfile ${REGISTRY_CREDS} \ + -u ${REGISTRY_USER} -p ${REGISTRY_PASS} \ + ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT} +fi diff --git a/04_setup_ironic.sh b/04_setup_ironic.sh index 45cc01a7f..ef7ddca22 100755 --- a/04_setup_ironic.sh +++ b/04_setup_ironic.sh @@ -19,12 +19,16 @@ rm -f assets/templates/99_local-registry.yaml $OPENSHIFT_INSTALL_PATH/data/data/ export REGISTRY_AUTH_FILE=$(mktemp "pullsecret--XXXXXXXXXX") { echo "${PULL_SECRET}" ; } 2> /dev/null > $REGISTRY_AUTH_FILE +# Combine pull-secret with registry's password +COMBINED_AUTH_FILE=$(mktemp "combined-pullsecret--XXXXXXXXXX") +jq -s '.[0] * .[1]' ${REGISTRY_AUTH_FILE} ${REGISTRY_CREDS} | tee ${COMBINED_AUTH_FILE} + DOCKERFILE=$(mktemp "release-update--XXXXXXXXXX") echo "FROM $OPENSHIFT_RELEASE_IMAGE" > $DOCKERFILE for IMAGE_VAR in $(env | grep "_LOCAL_IMAGE=" | grep -o "^[^=]*") ; do IMAGE=${!IMAGE_VAR} - sudo -E podman pull $OPENSHIFT_RELEASE_IMAGE + sudo -E podman pull --authfile $COMBINED_AUTH_FILE $OPENSHIFT_RELEASE_IMAGE # Is it a git repo? if [[ "$IMAGE" =~ "://" ]] ; then @@ -33,15 +37,15 @@ for IMAGE_VAR in $(env | grep "_LOCAL_IMAGE=" | grep -o "^[^=]*") ; do [ -e "$REPOPATH" ] || git clone $IMAGE $REPOPATH cd $REPOPATH export $IMAGE_VAR=${IMAGE##*/}:latest - export $IMAGE_VAR=$LOCAL_REGISTRY_ADDRESS/localimages/${!IMAGE_VAR} - sudo podman build -t ${!IMAGE_VAR} . + export $IMAGE_VAR=$LOCAL_REGISTRY_ADDRESS:$LOCAL_REGISTRY_PORT/localimages/${!IMAGE_VAR} + sudo podman build --authfile $COMBINED_AUTH_FILE -t ${!IMAGE_VAR} . cd - - sudo podman push --tls-verify=false ${!IMAGE_VAR} ${!IMAGE_VAR} + sudo podman push --tls-verify=false --authfile $COMBINED_AUTH_FILE ${!IMAGE_VAR} ${!IMAGE_VAR} fi # Update the bootstrap and master nodes to treat LOCAL_REGISTRY_ADDRESS as insecure mkdir -p $OPENSHIFT_INSTALL_PATH/data/data/bootstrap/baremetal/files/etc/containers - echo -e "[registries.insecure]\nregistries = ['${LOCAL_REGISTRY_ADDRESS}']" > $OPENSHIFT_INSTALL_PATH/data/data/bootstrap/baremetal/files/etc/containers/registries.conf + echo -e "[registries.insecure]\nregistries = ['${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}']" > $OPENSHIFT_INSTALL_PATH/data/data/bootstrap/baremetal/files/etc/containers/registries.conf cp assets/templates/99_local-registry.yaml.optional assets/templates/99_local-registry.yaml IMAGE_NAME=$(echo ${IMAGE_VAR/_LOCAL_IMAGE} | tr '[:upper:]_' '[:lower:]-') @@ -50,17 +54,39 @@ for IMAGE_VAR in $(env | grep "_LOCAL_IMAGE=" | grep -o "^[^=]*") ; do done if [ ! -z "${MIRROR_IMAGES}" ]; then + + # combine global and local secrets + # pull from one registry and push to local one + # hence credentials are different + + EXTRACT_DIR=$(mktemp -d "mirror-installer--XXXXXXXXXX") + + TAG=$( echo $OPENSHIFT_RELEASE_IMAGE | sed -e 's/[[:alnum:]/.]*release://' ) + MIRROR_LOG_FILE=/tmp/tmp_image_mirror-${TAG}.log + oc adm release mirror \ --insecure=true \ - -a $REGISTRY_AUTH_FILE \ - --from $OPENSHIFT_RELEASE_IMAGE \ - --to $LOCAL_REGISTRY_ADDRESS/localimages/local-release-image + -a ${COMBINED_AUTH_FILE} \ + --from ${OPENSHIFT_RELEASE_IMAGE} \ + --to-release-image ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:${TAG} \ + --to ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image 2>&1 | tee ${MIRROR_LOG_FILE} + + #To ensure that you use the correct images for the version of OpenShift Container Platform that you selected, + #you must extract the installation program from the mirrored content: + + oc adm release extract --registry-config "${COMBINED_AUTH_FILE}" \ + --command=openshift-baremetal-install --to "${EXTRACT_DIR}" \ + "${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:${TAG}" + + mv -f "${EXTRACT_DIR}/openshift-baremetal-install" ocp/ + + rm -rf "${EXTRACT_DIR}" fi if [ -f assets/templates/99_local-registry.yaml ] ; then build_installer - sudo podman image build -t $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE -f $DOCKERFILE - sudo podman push --tls-verify=false $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE + sudo podman image build --authfile $COMBINED_AUTH_FILE -t $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE -f $DOCKERFILE + sudo podman push --tls-verify=false --authfile $COMBINED_AUTH_FILE $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE $OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE fi rm -f $DOCKERFILE @@ -70,12 +96,12 @@ for name in ironic ironic-api ironic-conductor ironic-inspector dnsmasq httpd ma done # Remove existing pod -if sudo podman pod exists ironic-pod ; then +if sudo podman pod exists ironic-pod ; then sudo podman pod rm ironic-pod -f fi # Create pod -sudo podman pod create -n ironic-pod +sudo podman pod create -n ironic-pod IRONIC_IMAGE=${IRONIC_LOCAL_IMAGE:-$IRONIC_IMAGE} IRONIC_IPA_DOWNLOADER_IMAGE=${IRONIC_IPA_DOWNLOADER_LOCAL_IMAGE:-$IRONIC_IPA_DOWNLOADER_IMAGE} @@ -84,7 +110,8 @@ for IMAGE in ${IRONIC_IMAGE} ${IRONIC_IPA_DOWNLOADER_IMAGE} ${VBMC_IMAGE} ${SUSH sudo -E podman pull $([[ $IMAGE =~ $LOCAL_REGISTRY_ADDRESS.* ]] && echo "--tls-verify=false" ) $IMAGE done -rm -rf $REGISTRY_AUTH_FILE +rm -rf ${REGISTRY_AUTH_FILE} +rm -rf ${COMBINED_AUTH_FILE} CACHED_MACHINE_OS_IMAGE="${IRONIC_DATA_DIR}/html/images/${MACHINE_OS_IMAGE_NAME}" if [ ! -f "${CACHED_MACHINE_OS_IMAGE}" ]; then @@ -110,7 +137,7 @@ if [ "$NODES_PLATFORM" = "libvirt" ]; then sudo podman run -d --net host --privileged --name vbmc --pod ironic-pod \ -v "$WORKING_DIR/virtualbmc/vbmc":/root/.vbmc -v "/root/.ssh":/root/ssh \ "${VBMC_IMAGE}" - + sudo podman run -d --net host --privileged --name sushy-tools --pod ironic-pod \ -v "$WORKING_DIR/virtualbmc/sushy-tools":/root/sushy -v "/root/.ssh":/root/ssh \ "${SUSHY_TOOLS_IMAGE}" diff --git a/common.sh b/common.sh index 4f0de2037..a3133a8d5 100644 --- a/common.sh +++ b/common.sh @@ -36,9 +36,6 @@ if [ -z "${CONFIG:-}" ]; then fi source $CONFIG -export LOCAL_REGISTRY_ADDRESS=${LOCAL_REGISTRY_ADDRESS:-"192.168.111.1:5000"} -export MIRROR_IMAGES=${MIRROR_IMAGES:-} - # # See https://openshift-release.svc.ci.openshift.org for release details # @@ -81,7 +78,7 @@ fi if env | grep -q "_LOCAL_IMAGE=" ; then # We're going to be using a locally modified release image - export OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE="${LOCAL_REGISTRY_ADDRESS}/localimages/local-release-image:latest" + export OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE="${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image:latest" fi # Set variables @@ -216,3 +213,14 @@ fi # Defaults the variable to enable testing a custom machine-api-operator image export TEST_CUSTOM_MAO=${TEST_CUSTOM_MAO:-false} + +# mirror images for installation in restricted network +export MIRROR_IMAGES=${MIRROR_IMAGES:-} + +# variables for local registry configuration +export LOCAL_REGISTRY_ADDRESS=${LOCAL_REGISTRY_ADDRESS:-"192.168.111.1"} +export LOCAL_REGISTRY_PORT=${LOCAL_REGISTRY_PORT:-"5000"} +export REGISTRY_USER=${REGISTRY_USER:-ocp-user} +export REGISTRY_PASS=${REGISTRY_PASS:-ocp-pass} +export REGISTRY_DIR=${REGISTRY_DIR:-$WORKING_DIR/registry} +export REGISTRY_CREDS=${REGISTRY_CREDS:-$HOME/private-mirror.json} diff --git a/config_example.sh b/config_example.sh index 1e8228427..5fcf22b54 100644 --- a/config_example.sh +++ b/config_example.sh @@ -76,4 +76,17 @@ set -x # Name of branch in the above repo which contains the custom MAO changes #export MAO_BRANCH="mao-fix" -#export LOCAL_REGISTRY_ADDRESS="192.168.111.1:5000" +#export LOCAL_REGISTRY_ADDRESS="192.168.111.1" +#export LOCAL_REGISTRY_PORT="5000" + +# configure username for registry +#export REGISTRY_USER=some-user + +# congiugre password for registry user +#export REGISTRY_PASS=some-pass + +# configure base directory for registry +#export REGISTRY_DIR=/opt/registry + +# configure location of mirror's creds +#export REGISTRY_CREDS=${REGISTRY_CREDS:-$USER/private-mirror.json} diff --git a/ocp_install_env.sh b/ocp_install_env.sh index 89b7022b8..517cfef5a 100644 --- a/ocp_install_env.sh +++ b/ocp_install_env.sh @@ -78,6 +78,12 @@ function generate_ocp_install_config() { # TODO - Change worker replicas to ${NUM_WORKERS} once the machine-api-operator # deploys the baremetal-operator + # when using local mirror set pull secret to this mirror + # also this should ensure we don't accidentally pull from upstream + if [ ! -z "${MIRROR_IMAGES}" ]; then + export PULL_SECRET=$(cat ${REGISTRY_CREDS}) + fi + mkdir -p "${outdir}" cat > "${outdir}/install-config.yaml" << EOF apiVersion: v1 diff --git a/utils.sh b/utils.sh index 139f72693..76f77de70 100644 --- a/utils.sh +++ b/utils.sh @@ -198,16 +198,122 @@ function bmo_config_map { function image_mirror_config { if [ ! -z "${MIRROR_IMAGES}" ]; then + TAG=$( echo $OPENSHIFT_RELEASE_IMAGE | sed -e 's/[[:alnum:]/.]*release://' ) TAGGED=$(echo $OPENSHIFT_RELEASE_IMAGE | sed -e 's/release://') RELEASE=$(echo $OPENSHIFT_RELEASE_IMAGE | grep -o 'registry.svc.ci.openshift.org[^":]\+') - cat << EOF + INDENTED_CERT=$( cat $REGISTRY_DIR/certs/registry.crt | awk '{ print " ", $0 }' ) + MIRROR_LOG_FILE=/tmp/tmp_image_mirror-${TAG}.log + if [ ! -s ${MIRROR_LOG_FILE} ]; then + cat << EOF imageContentSources: - mirrors: - - ${LOCAL_REGISTRY_ADDRESS}/localimages/local-release-image + - ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image source: ${RELEASE} - mirrors: - - ${LOCAL_REGISTRY_ADDRESS}/localimages/local-release-image + - ${LOCAL_REGISTRY_ADDRESS}:${LOCAL_REGISTRY_PORT}/localimages/local-release-image source: ${TAGGED} +additionalTrustBundle: | +${INDENTED_CERT} EOF + else + cat ${MIRROR_LOG_FILE} | sed -n '/To use the new mirrored repository to install/,/To use the new mirrored repository for upgrades/p' |\ + sed -e '/^$/d' -e '/To use the new mirrored repository/d' + cat << EOF +additionalTrustBundle: | +${INDENTED_CERT} +EOF + fi + fi +} + +function setup_local_registry() { + + # httpd-tools provides htpasswd utility + sudo yum install -y httpd-tools + + sudo mkdir -pv ${REGISTRY_DIR}/{auth,certs,data} + sudo chown -R $USER:$USER ${REGISTRY_DIR} + + pushd $REGISTRY_DIR/certs + SSL_HOST_NAME="${LOCAL_REGISTRY_ADDRESS}" + + if [[ $( echo $SSL_HOST_NAME | grep -Eo '^[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}\.[[:digit:]]{1,3}$') ]];then + SSL_EXT_8="subjectAltName = IP:${SSL_HOST_NAME}" + SSL_EXT_7="subjectAltName = IP:${SSL_HOST_NAME}" + else + SSL_EXT_8="subjectAltName = otherName:${SSL_HOST_NAME}" + SSL_EXT_7="subjectAltName = DNS:${SSL_HOST_NAME}" + fi + + # + # registry key and cert are generated if they don't exist + # + if [[ ! -s ${REGISTRY_DIR}/certs/registry.key ]]; then + openssl genrsa -out ${REGISTRY_DIR}/certs/registry.key 2048 fi + + if [[ ! -s ${REGISTRY_DIR}/certs/registry.crt ]]; then + + if [ "${RHEL8}" = "True" ] ; then + openssl req -x509 \ + -key ${REGISTRY_DIR}/certs/registry.key \ + -out ${REGISTRY_DIR}/certs/registry.crt \ + -days 365 \ + -addext "${SSL_EXT_8}" \ + -subj "/C=US/ST=NC/L=Raleigh/O=Test Company/OU=Testing/CN=${SSL_HOST_NAME}" + else + SSL_TMP_CONF=$(mktemp 'my-ssl-conf.XXXXXX') + cat > ${SSL_TMP_CONF} <