diff --git a/Makefile b/Makefile index 4c855623..6649c47f 100644 --- a/Makefile +++ b/Makefile @@ -117,23 +117,23 @@ run: .PHONY: run-ready run-ready: @echo "Waiting 5m for the MicroShift service to be ready" - @for _ in $$(seq 300); do \ + @for _ in $$(seq 60); do \ if sudo podman exec -i "${USHIFT_IMAGE}" systemctl -q is-active microshift.service ; then \ printf "\nOK\n" && exit 0; \ fi ; \ - echo -n "." && sleep 1 ; \ + echo -n "." && sleep 5 ; \ done ; \ printf "\nFAILED\n" && exit 1 .PHONY: run-healthy run-healthy: - @echo "Waiting 10m for the MicroShift service to be healthy" + @echo "Waiting 15m for the MicroShift service to be healthy" @for _ in $$(seq 60); do \ state=$$(sudo podman exec -i "${USHIFT_IMAGE}" systemctl show --property=SubState --value greenboot-healthcheck) ; \ if [ "$${state}" = "exited" ] ; then \ printf "\nOK\n" && exit 0; \ fi ; \ - echo -n "." && sleep 10 ; \ + echo -n "." && sleep 15 ; \ done ; \ printf "\nFAILED\n" && exit 1 diff --git a/docs/run.md b/docs/run.md index 7bc0ae81..3fa5487a 100644 --- a/docs/run.md +++ b/docs/run.md @@ -71,9 +71,9 @@ The following options can be specified in the make command line using the `NAME= This step includes: * Loading the `openvswitch` module required when OVN-K CNI driver is used - when compiled with the non-default `WITH_KINDNET=0` build option. + when compiled with the non-default `WITH_KINDNET=0` image build option. * Preparing a 1GB TopoLVM CSI backend on the host to be used by MicroShift when - compiled with the default `WITH_TOPOLVM=1` build option. + compiled with the default `WITH_TOPOLVM=1` image build option. ```bash make run diff --git a/packaging/microshift-builder.Containerfile b/packaging/microshift-builder.Containerfile index ce0c0d9d..2350e7fa 100644 --- a/packaging/microshift-builder.Containerfile +++ b/packaging/microshift-builder.Containerfile @@ -42,15 +42,23 @@ RUN "${USHIFT_PREBUILD_SCRIPT}" --replace "${OKD_REPO}" "${OKD_VERSION_TAG}" # Building all MicroShift downstream RPMs and SRPMs # hadolint ignore=DL3059 -RUN WITH_KINDNET=1 MICROSHIFT_VARIANT="community" \ - make -C "${HOME}/microshift" rpm srpm +RUN MICROSHIFT_VARIANT="community" make -C "${HOME}/microshift" rpm srpm + +# Building Kindnet upstream RPM +COPY --chown=${USER}:${USER} ./src/kindnet/kindnet.spec "${HOME}/microshift/packaging/rpm/microshift.spec" +COPY --chown=${USER}:${USER} ./src/kindnet/assets/ "${HOME}/microshift/assets/optional/" +COPY --chown=${USER}:${USER} ./src/kindnet/dropins/ "${HOME}/microshift/packaging/kindnet/" +COPY --chown=${USER}:${USER} ./src/kindnet/crio.conf.d/ "${HOME}/microshift/packaging/crio.conf.d/" +# Prepare and build Kindnet upstream RPM +RUN "${USHIFT_PREBUILD_SCRIPT}" --replace-kindnet "${OKD_REPO}" "${OKD_VERSION_TAG}" && \ + MICROSHIFT_VARIANT="community" make -C "${HOME}/microshift" rpm # Building TopoLVM upstream RPM -COPY --chmod=644 ./src/topolvm/topolvm.spec "${HOME}/microshift/packaging/rpm/microshift.spec" -COPY ./src/topolvm/assets/ "${HOME}/microshift/assets/optional/topolvm/" -COPY ./src/topolvm/dropins/ "${HOME}/microshift/packaging/microshift/dropins/" -COPY ./src/topolvm/greenboot/ "${HOME}/microshift/packaging/greenboot/" -COPY ./src/topolvm/release/ "${HOME}/microshift/assets/optional/topolvm/" +COPY --chown=${USER}:${USER} ./src/topolvm/topolvm.spec "${HOME}/microshift/packaging/rpm/microshift.spec" +COPY --chown=${USER}:${USER} ./src/topolvm/assets/ "${HOME}/microshift/assets/optional/topolvm/" +COPY --chown=${USER}:${USER} ./src/topolvm/dropins/ "${HOME}/microshift/packaging/microshift/dropins/" +COPY --chown=${USER}:${USER} ./src/topolvm/greenboot/ "${HOME}/microshift/packaging/greenboot/" +COPY --chown=${USER}:${USER} ./src/topolvm/release/ "${HOME}/microshift/assets/optional/topolvm/" RUN MICROSHIFT_VARIANT="community" make -C "${HOME}/microshift" rpm # Post-build MicroShift configuration diff --git a/src/image/prebuild.sh b/src/image/prebuild.sh index 7f39a191..6aa25a55 100755 --- a/src/image/prebuild.sh +++ b/src/image/prebuild.sh @@ -4,7 +4,7 @@ set -euo pipefail MICROSHIFT_ROOT="/home/microshift/microshift" declare -A UNAME_TO_GOARCH_MAP=( ["x86_64"]="amd64" ["aarch64"]="arm64" ) -verify(){ +verify() { local -r okd_url=$1 local -r okd_releaseTag=$2 @@ -14,11 +14,11 @@ verify(){ fi } -replace_assets(){ +replace_base_assets() { local -r okd_url=$1 local -r okd_releaseTag=$2 local -r arch=$(uname -m) - local -r temp_release_json=$(mktemp "/tmp/release-${arch}.XXXXX.json") + local -r temp_json=$(mktemp "/tmp/release-${arch}.XXXXX.json") # replace Microshift images with upstream (from OKD release) for op in $(jq -e -r '.images | keys []' "${MICROSHIFT_ROOT}/assets/release/release-${arch}.json") @@ -27,40 +27,33 @@ replace_assets(){ image=$(oc adm release info --image-for="${op}" "${okd_url}:${okd_releaseTag}" || true) if [ -n "${image}" ] ; then echo "${op} ${image}" - jq --arg a "${op}" --arg b "${image}" '.images[$a] = $b' "${MICROSHIFT_ROOT}/assets/release/release-${arch}.json" >"${temp_release_json}" - mv "${temp_release_json}" "${MICROSHIFT_ROOT}/assets/release/release-${arch}.json" + jq --arg a "${op}" --arg b "${image}" '.images[$a] = $b' "${MICROSHIFT_ROOT}/assets/release/release-${arch}.json" >"${temp_json}" + mv "${temp_json}" "${MICROSHIFT_ROOT}/assets/release/release-${arch}.json" fi done pod_image=$(oc adm release info --image-for=pod "${okd_url}:${okd_releaseTag}" || true) # update the infra pods for crio sed -i 's,pause_image .*,pause_image = '"\"${pod_image}\""',' "${MICROSHIFT_ROOT}/packaging/crio.conf.d/10-microshift_${UNAME_TO_GOARCH_MAP[${arch}]}.conf" +} - # kube proxy is required for kindnet - kube_proxy_okd_image_with_hash=$(oc adm release info --image-for="kube-proxy" "${okd_url}:${okd_releaseTag}") - echo "kube-proxy ${kube_proxy_okd_image_with_hash}" - # The OKD image we retrieve is in the format quay.io/okd/scos-content@sha256:, - # where the image name and digest (hash) are combined in a single string. - # However, in the kustomization.${arch}.yaml file, we need the image name (newName) and - # the digest in separate fields. To achieve this, we first extract the image name and digest - # using parameter expansion, then use the yq command to insert these values into the - # appropriate places within the YAML file. - kube_proxy_okd_image_name="${kube_proxy_okd_image_with_hash%%@*}" - kube_proxy_okd_image_hash="${kube_proxy_okd_image_with_hash##*@}" - # install yq tool to update the image and hash +replace_olm_assets() { + local -r okd_url=$1 + local -r okd_releaseTag=$2 + local -r arch=$(uname -m) + local -r temp_json=$(mktemp "/tmp/release-olm-${arch}.XXXXX.json") + + # Install the yq tool "${MICROSHIFT_ROOT}"/scripts/fetch_tools.sh yq - "${MICROSHIFT_ROOT}"/_output/bin/yq eval ".images[] |= select(.name == \"kube-proxy\") |= (.newName = \"${kube_proxy_okd_image_name}\" | .digest = \"${kube_proxy_okd_image_hash}\")" -i "${MICROSHIFT_ROOT}/assets/optional/kube-proxy/kustomization.${arch}.yaml" - jq --arg img "$kube_proxy_okd_image_with_hash" '.images["kube-proxy"] = $img' "${MICROSHIFT_ROOT}/assets/optional/kube-proxy/release-kube-proxy-${arch}.json" >"${temp_release_json}" - mv "${temp_release_json}" "${MICROSHIFT_ROOT}/assets/optional/kube-proxy/release-kube-proxy-${arch}.json" - # replace olm images with upstream (from OKD release) + # Replace olm images with upstream (from OKD release) # This is extracted from openshift/microshift/scripts/auto-rebase/rebase.sh and modified to work with OKD release local olm_image_refs_file="${MICROSHIFT_ROOT}/assets/optional/operator-lifecycle-manager/image-references" local kustomization_arch_file="${MICROSHIFT_ROOT}/assets/optional/operator-lifecycle-manager/kustomization.${arch}.yaml" local olm_release_json="${MICROSHIFT_ROOT}/assets/optional/operator-lifecycle-manager/release-olm-${arch}.json" # Create the OLM release-${arch}.json file with base structure - jq -n '{"release": {"base": "unknown"}, "images": {}}' > "${olm_release_json}" + jq -n '{"release": {"base": "upstream"}, "images": {}}' > "${olm_release_json}" # Create extra kustomization for each arch in separate file cat < "${kustomization_arch_file}" @@ -92,8 +85,8 @@ EOF EOF # Update JSON file - jq --arg container "${container}" --arg img "${new_image}" '.images[$container] = $img' "${olm_release_json}" >"${temp_release_json}" - mv "${temp_release_json}" "${olm_release_json}" + jq --arg container "${container}" --arg img "${new_image}" '.images[$container] = $img' "${olm_release_json}" >"${temp_json}" + mv "${temp_json}" "${olm_release_json}" fi done @@ -126,32 +119,71 @@ EOF fi } -fix_rpm_spec(){ +replace_kindnet_assets() { + local -r okd_url=$1 + local -r okd_releaseTag=$2 + local -r arch=$(uname -m) + local -r temp_json=$(mktemp "/tmp/release-kindnet-${arch}.XXXXX.json") + + # Install the yq tool + "${MICROSHIFT_ROOT}"/scripts/fetch_tools.sh yq + + # kube proxy is required for kindnet + local -r image_with_hash=$(oc adm release info --image-for="kube-proxy" "${okd_url}:${okd_releaseTag}") + echo "kube-proxy ${image_with_hash}" + # The OKD image we retrieve is in the format quay.io/okd/scos-content@sha256:, + # where the image name and digest (hash) are combined in a single string. + # However, in the kustomization.${arch}.yaml file, we need the image name (newName) and + # the digest in separate fields. To achieve this, we first extract the image name and digest + # using parameter expansion, then use the yq command to insert these values into the + # appropriate places within the YAML file. + local -r image_name="${image_with_hash%%@*}" + local -r image_hash="${image_with_hash##*@}" + + # Update the image and hash + "${MICROSHIFT_ROOT}"/_output/bin/yq eval \ + ".images[] |= select(.name == \"kube-proxy\") |= (.newName = \"${image_name}\" | .digest = \"${image_hash}\")" \ + -i "${MICROSHIFT_ROOT}/assets/optional/kube-proxy/kustomization.${arch}.yaml" + jq --arg img "$image_with_hash" '.images["kube-proxy"] = $img' \ + "${MICROSHIFT_ROOT}/assets/optional/kube-proxy/release-kube-proxy-${arch}.json" >"${temp_json}" + mv "${temp_json}" "${MICROSHIFT_ROOT}/assets/optional/kube-proxy/release-kube-proxy-${arch}.json" +} + +fix_rpm_spec() { # Fix the RPM spec by removing the microshift-networking package hard dependency sed -i 's/Requires: microshift-networking/Recommends: microshift-networking/' "${MICROSHIFT_ROOT}/packaging/rpm/microshift.spec" } usage() { echo "Usage:" - echo "$(basename "$0") --verify OKD_URL RELEASE_TAG verify upstream release" - echo "$(basename "$0") --replace OKD_URL RELEASE_TAG replace microshift assets with upstream images" + echo "$(basename "$0") --verify OKD_URL RELEASE_TAG verify OKD upstream release" + echo "$(basename "$0") --replace OKD_URL RELEASE_TAG replace MicroShift assets with OKD upstream images" + echo "$(basename "$0") --replace-kindnet OKD_URL RELEASE_TAG replace Kindnet assets with OKD upstream images" exit 1 } -if [ $# -eq 3 ] ; then - case "$1" in - --replace) - verify "$2" "$3" - replace_assets "$2" "$3" - fix_rpm_spec - ;; - --verify) - verify "$2" "$3" - ;; - *) - usage - ;; - esac -else +# +# Main +# +if [ $# -ne 3 ] ; then usage fi + +case "$1" in +--replace) + verify "$2" "$3" + replace_base_assets "$2" "$3" + replace_olm_assets "$2" "$3" + fix_rpm_spec + ;; +--replace-kindnet) + verify "$2" "$3" + replace_kindnet_assets "$2" "$3" + ;; +--verify) + verify "$2" "$3" + ;; +*) + usage + ;; +esac diff --git a/src/kindnet/assets/kindnet/00-namespace.yaml b/src/kindnet/assets/kindnet/00-namespace.yaml new file mode 100644 index 00000000..97fff125 --- /dev/null +++ b/src/kindnet/assets/kindnet/00-namespace.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: kube-kindnet + labels: + name: kube-kindnet + openshift.io/run-level: "0" + openshift.io/cluster-monitoring: "true" + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/warn: privileged + annotations: + openshift.io/node-selector: "" + openshift.io/description: "kindnet Kubernetes components" + workload.openshift.io/allowed: "management" diff --git a/src/kindnet/assets/kindnet/01-service-account.yaml b/src/kindnet/assets/kindnet/01-service-account.yaml new file mode 100644 index 00000000..27d49f3a --- /dev/null +++ b/src/kindnet/assets/kindnet/01-service-account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kindnet + namespace: kube-kindnet \ No newline at end of file diff --git a/src/kindnet/assets/kindnet/02-cluster-role.yaml b/src/kindnet/assets/kindnet/02-cluster-role.yaml new file mode 100644 index 00000000..05fe118b --- /dev/null +++ b/src/kindnet/assets/kindnet/02-cluster-role.yaml @@ -0,0 +1,88 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kindnet +rules: + - apiGroups: [""] + resources: + - namespaces + - nodes + - pods + verbs: + - get + - list + - patch + - watch + - update + - apiGroups: [""] + resources: + - pods + verbs: + - get + - list + - patch + - watch + - delete + - apiGroups: [""] + resources: + - configmaps + verbs: + - get + - create + - update + - patch + - apiGroups: [""] + resources: + - services + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + - apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: + - get + - list + - watch + - apiGroups: ["", "events.k8s.io"] + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: ["security.openshift.io"] + resources: + - securitycontextconstraints + verbs: + - use + resourceNames: + - privileged + - apiGroups: [""] + resources: + - "nodes/status" + verbs: + - patch + - update + - apiGroups: ["apiextensions.k8s.io"] + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch + - apiGroups: ['authentication.k8s.io'] + resources: ['tokenreviews'] + verbs: ['create'] + - apiGroups: ['authorization.k8s.io'] + resources: ['subjectaccessreviews'] + verbs: ['create'] \ No newline at end of file diff --git a/src/kindnet/assets/kindnet/03-cluster-role-binding.yaml b/src/kindnet/assets/kindnet/03-cluster-role-binding.yaml new file mode 100644 index 00000000..e19e3c17 --- /dev/null +++ b/src/kindnet/assets/kindnet/03-cluster-role-binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kindnet +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kindnet +subjects: + - kind: ServiceAccount + name: kindnet + namespace: kube-kindnet \ No newline at end of file diff --git a/src/kindnet/assets/kindnet/04-daemonset.yaml b/src/kindnet/assets/kindnet/04-daemonset.yaml new file mode 100644 index 00000000..2d79ca8d --- /dev/null +++ b/src/kindnet/assets/kindnet/04-daemonset.yaml @@ -0,0 +1,86 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app: kindnet + k8s-app: kindnet + tier: node + name: kube-kindnet-ds + namespace: kube-kindnet +spec: + selector: + matchLabels: + app: kindnet + k8s-app: kindnet + template: + metadata: + labels: + app: kindnet + k8s-app: kindnet + tier: node + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + containers: + - name: kube-kindnet + image: kindnet + imagePullPolicy: IfNotPresent + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_SUBNET + value: 10.244.0.0/16 + resources: + requests: + cpu: 100m + memory: 50Mi + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + privileged: false + volumeMounts: + - name: cni + mountPath: /etc/cni/net.d + - name: xtables-lock + mountPath: /run/xtables.lock + readOnly: false + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: nri-plugin + mountPath: /var/run/nri + hostNetwork: true + priorityClassName: system-node-critical + serviceAccountName: kindnet + tolerations: + - effect: NoSchedule + operator: Exists + volumes: + - hostPath: + path: /etc/cni/net.d + name: cni + - hostPath: + path: /run/xtables.lock + type: FileOrCreate + name: xtables-lock + - name: lib-modules + hostPath: + path: /lib/modules + - name: nri-plugin + hostPath: + path: /var/run/nri \ No newline at end of file diff --git a/src/kindnet/assets/kindnet/kustomization.aarch64.yaml b/src/kindnet/assets/kindnet/kustomization.aarch64.yaml new file mode 100644 index 00000000..9d3190e0 --- /dev/null +++ b/src/kindnet/assets/kindnet/kustomization.aarch64.yaml @@ -0,0 +1,5 @@ + +images: + - name: kindnet + newName: docker.io/kindest/kindnetd + digest: sha256:2bdc3188f2ddc8e54841f69ef900a8dde1280057c97500f966a7ef31364021f1 \ No newline at end of file diff --git a/src/kindnet/assets/kindnet/kustomization.x86_64.yaml b/src/kindnet/assets/kindnet/kustomization.x86_64.yaml new file mode 100644 index 00000000..b52b3393 --- /dev/null +++ b/src/kindnet/assets/kindnet/kustomization.x86_64.yaml @@ -0,0 +1,5 @@ + +images: + - name: kindnet + newName: docker.io/kindest/kindnetd + digest: sha256:7a9c9fa59dd517cdc2c82eef1e51392524dd285e9cf7cb5a851c49f294d6cd11 \ No newline at end of file diff --git a/src/kindnet/assets/kindnet/kustomization.yaml b/src/kindnet/assets/kindnet/kustomization.yaml new file mode 100644 index 00000000..2f4dab61 --- /dev/null +++ b/src/kindnet/assets/kindnet/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - 00-namespace.yaml + - 01-service-account.yaml + - 02-cluster-role.yaml + - 03-cluster-role-binding.yaml + - 04-daemonset.yaml diff --git a/src/kindnet/assets/kindnet/release-kindnet-aarch64.json b/src/kindnet/assets/kindnet/release-kindnet-aarch64.json new file mode 100644 index 00000000..3bf67014 --- /dev/null +++ b/src/kindnet/assets/kindnet/release-kindnet-aarch64.json @@ -0,0 +1,5 @@ +{ + "images": { + "kindnet": "docker.io/kindest/kindnetd@sha256:2bdc3188f2ddc8e54841f69ef900a8dde1280057c97500f966a7ef31364021f1" + } +} diff --git a/src/kindnet/assets/kindnet/release-kindnet-x86_64.json b/src/kindnet/assets/kindnet/release-kindnet-x86_64.json new file mode 100644 index 00000000..706a268a --- /dev/null +++ b/src/kindnet/assets/kindnet/release-kindnet-x86_64.json @@ -0,0 +1,5 @@ +{ + "images": { + "kindnet": "docker.io/kindest/kindnetd@sha256:7a9c9fa59dd517cdc2c82eef1e51392524dd285e9cf7cb5a851c49f294d6cd11" + } +} diff --git a/src/kindnet/assets/kube-proxy/00-namespace.yaml b/src/kindnet/assets/kube-proxy/00-namespace.yaml new file mode 100644 index 00000000..3b8d6cea --- /dev/null +++ b/src/kindnet/assets/kube-proxy/00-namespace.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: kube-proxy + labels: + name: kindnet + openshift.io/run-level: "0" + openshift.io/cluster-monitoring: "true" + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/warn: privileged + annotations: + openshift.io/node-selector: "" + openshift.io/description: "kube-proxy Kubernetes components" + workload.openshift.io/allowed: "management" \ No newline at end of file diff --git a/src/kindnet/assets/kube-proxy/01-service-account.yaml b/src/kindnet/assets/kube-proxy/01-service-account.yaml new file mode 100644 index 00000000..9b118afa --- /dev/null +++ b/src/kindnet/assets/kube-proxy/01-service-account.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-app: kube-proxy + name: kube-proxy + namespace: kube-proxy \ No newline at end of file diff --git a/src/kindnet/assets/kube-proxy/02-cluster-role.yaml b/src/kindnet/assets/kube-proxy/02-cluster-role.yaml new file mode 100644 index 00000000..ae9c2bb2 --- /dev/null +++ b/src/kindnet/assets/kube-proxy/02-cluster-role.yaml @@ -0,0 +1,23 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:kube-proxy +rules: + - apiGroups: + - "" + resources: + - services + - endpoints + - nodes + - configmaps + verbs: + - list + - watch + - get + - apiGroups: ["discovery.k8s.io"] + resources: + - endpointslices + verbs: + - list + - watch + - get \ No newline at end of file diff --git a/src/kindnet/assets/kube-proxy/03-cluster-role-binding.yaml b/src/kindnet/assets/kube-proxy/03-cluster-role-binding.yaml new file mode 100644 index 00000000..2edc2f19 --- /dev/null +++ b/src/kindnet/assets/kube-proxy/03-cluster-role-binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:kube-proxy +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-proxy +subjects: + - kind: ServiceAccount + name: kube-proxy + namespace: kube-proxy \ No newline at end of file diff --git a/src/kindnet/assets/kube-proxy/04-configmap.yaml b/src/kindnet/assets/kube-proxy/04-configmap.yaml new file mode 100644 index 00000000..0ef2a1b6 --- /dev/null +++ b/src/kindnet/assets/kube-proxy/04-configmap.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +data: + config.conf: | + apiVersion: kubeproxy.config.k8s.io/v1alpha1 + kind: KubeProxyConfiguration + clusterCIDR: 10.42.0.0/16 + mode: iptables + clientConnection: + kubeconfig: /var/lib/kubeconfig + iptables: + masqueradeAll: true + conntrack: + maxPerCore: 0 + featureGates: + AllAlpha: false +kind: ConfigMap +metadata: + labels: + app: kube-proxy + k8s-app: kube-proxy + name: kube-proxy + namespace: kube-proxy \ No newline at end of file diff --git a/src/kindnet/assets/kube-proxy/05-daemonset.yaml b/src/kindnet/assets/kube-proxy/05-daemonset.yaml new file mode 100644 index 00000000..cf8bdaa0 --- /dev/null +++ b/src/kindnet/assets/kube-proxy/05-daemonset.yaml @@ -0,0 +1,52 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kube-proxy + namespace: kube-proxy +spec: + selector: + matchLabels: + k8s-app: kube-proxy + template: + metadata: + labels: + k8s-app: kube-proxy + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + serviceAccountName: kube-proxy # Reference the Service Account here + containers: + - name: kube-proxy + image: kube-proxy + command: + - /usr/bin/kube-proxy + - --config=/var/lib/kube-proxy/config.conf + volumeMounts: + - name: config + mountPath: /var/lib/kube-proxy/ + readOnly: true + - name: kubeconfig + mountPath: /var/lib/kubeconfig + readOnly: true + securityContext: + privileged: true + hostNetwork: true # Allows the pod to use the host network + dnsPolicy: ClusterFirstWithHostNet + tolerations: + - effect: NoSchedule + operator: Exists + volumes: + - name: config + configMap: + name: kube-proxy + - hostPath: + path: /var/lib/microshift/resources/kubeadmin/kubeconfig + type: FileOrCreate + name: kubeconfig \ No newline at end of file diff --git a/src/kindnet/assets/kube-proxy/kustomization.aarch64.yaml b/src/kindnet/assets/kube-proxy/kustomization.aarch64.yaml new file mode 100644 index 00000000..e62971cb --- /dev/null +++ b/src/kindnet/assets/kube-proxy/kustomization.aarch64.yaml @@ -0,0 +1,5 @@ + +images: + - name: kube-proxy + newName: quay.io/openshift-release-dev/ocp-v4.0-art-dev + digest: sha256:9590a1bb77bdf6ff6f75efd48c1e32bb5a97a00145a5a96b25e8cf03e55f2b98 diff --git a/src/kindnet/assets/kube-proxy/kustomization.x86_64.yaml b/src/kindnet/assets/kube-proxy/kustomization.x86_64.yaml new file mode 100644 index 00000000..32d138db --- /dev/null +++ b/src/kindnet/assets/kube-proxy/kustomization.x86_64.yaml @@ -0,0 +1,5 @@ + +images: + - name: kube-proxy + newName: quay.io/openshift-release-dev/ocp-v4.0-art-dev + digest: sha256:6aefc3c86fc2beeb3a4b5f9ce5debba753955e5289a3f1c1cf5cd79280fe46ef diff --git a/src/kindnet/assets/kube-proxy/kustomization.yaml b/src/kindnet/assets/kube-proxy/kustomization.yaml new file mode 100644 index 00000000..67a36b80 --- /dev/null +++ b/src/kindnet/assets/kube-proxy/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - 00-namespace.yaml + - 01-service-account.yaml + - 02-cluster-role.yaml + - 03-cluster-role-binding.yaml + - 04-configmap.yaml + - 05-daemonset.yaml diff --git a/src/kindnet/assets/kube-proxy/release-kube-proxy-aarch64.json b/src/kindnet/assets/kube-proxy/release-kube-proxy-aarch64.json new file mode 100644 index 00000000..960dc8eb --- /dev/null +++ b/src/kindnet/assets/kube-proxy/release-kube-proxy-aarch64.json @@ -0,0 +1,5 @@ +{ + "images": { + "kube-proxy": "quay.io/okd/scos-content@sha256:585cdfeab0f6b006bb9cca22dd0f543424c424959a0ebe355c233f0d41ee724f" + } +} diff --git a/src/kindnet/assets/kube-proxy/release-kube-proxy-x86_64.json b/src/kindnet/assets/kube-proxy/release-kube-proxy-x86_64.json new file mode 100644 index 00000000..960dc8eb --- /dev/null +++ b/src/kindnet/assets/kube-proxy/release-kube-proxy-x86_64.json @@ -0,0 +1,5 @@ +{ + "images": { + "kube-proxy": "quay.io/okd/scos-content@sha256:585cdfeab0f6b006bb9cca22dd0f543424c424959a0ebe355c233f0d41ee724f" + } +} diff --git a/src/kindnet/crio.conf.d/13-microshift-kindnet.conf b/src/kindnet/crio.conf.d/13-microshift-kindnet.conf new file mode 100644 index 00000000..729156bd --- /dev/null +++ b/src/kindnet/crio.conf.d/13-microshift-kindnet.conf @@ -0,0 +1,11 @@ +[crio.network] +# The kindnet network name is configured in /etc/cni/net.d/10-kindnet.conflist. +# By declaring this, CRI-O will wait until that network is up and running. +cni_default_network = "kindnet" + +# Kindnet uses utilities from the containernetworking-plugins package, which +# must be installed from the AppStream repository on CentOS 9, or from the GitHub +# release page on CentOS 10 (https://github.com/containernetworking/plugins/releases) +plugin_dirs = [ + "/usr/libexec/cni", +] diff --git a/src/kindnet/dropins/00-disableDefaultCNI.yaml b/src/kindnet/dropins/00-disableDefaultCNI.yaml new file mode 100644 index 00000000..e368f3a8 --- /dev/null +++ b/src/kindnet/dropins/00-disableDefaultCNI.yaml @@ -0,0 +1,7 @@ +network: + # CNIPlugin is a user defined string value matching one of the above CNI values. MicroShift uses this + # value to decide whether to deploy the OVN-K as default CNI. An unset field defaults to "" during yaml parsing, and thus + # could mean that the cluster has been upgraded. In order to support the existing out-of-box behavior, MicroShift + # assumes an empty string to mean the OVN-K should be deployed. + # Allowed values are: unset or one of ["", "ovnk", "none"] + cniPlugin: "none" diff --git a/src/kindnet/dropins/microshift-kindnet.service b/src/kindnet/dropins/microshift-kindnet.service new file mode 100644 index 00000000..a33b5a3a --- /dev/null +++ b/src/kindnet/dropins/microshift-kindnet.service @@ -0,0 +1,28 @@ +[Unit] +Description=MicroShift +Wants=network-online.target crio.service +After=network-online.target crio.service + +# Control shutdown order by declaring this service to start Before the kubepods.slice +# transient systemd unit; this makes system shutdown delay MicroShift shutdown until +# all the pod containers are down. This is important because some services need to talk +# to the MicroShift API during shutdown (i.e. releasing leader election locks or cleaning +# up other resources) MicroShift restart or manual stop will not stop the kubepods. +Before=kubepods.slice + +[Service] +WorkingDirectory=/usr/bin/ +ExecStart=microshift run +Restart=always +User=root +Type=notify +Delegate=yes +CPUAccounting=yes +BlockIOAccounting=yes +MemoryAccounting=yes +LimitNOFILE=1048576 +TimeoutStartSec=4m + +[Install] +WantedBy=multi-user.target +Also=microshift-cleanup-kubelet.service diff --git a/src/kindnet/kindnet.spec b/src/kindnet/kindnet.spec new file mode 100644 index 00000000..ccda8646 --- /dev/null +++ b/src/kindnet/kindnet.spec @@ -0,0 +1,99 @@ +# +# Beginning of the header copied from microshift/packaging/rpm/microshift.spec +# +%global shortcommit %(c=%{commit}; echo ${c:0:7}) +# Debug info not supported with Go +%global debug_package %{nil} + +Name: microshift +Version: %{version} +Release: %{release}%{dist} +Summary: MicroShift service +License: ASL 2.0 +URL: https://github.com/openshift/microshift +Source0: https://github.com/openshift/microshift/archive/%{commit}/microshift-%{shortcommit}.tar.gz + +ExclusiveArch: x86_64 aarch64 + +%description +The microshift package provides an OpenShift Kubernetes distribution optimized for small form factor and edge computing. + +%prep +%setup -n microshift-%{commit} +# +# End of the header copied from microshift/packaging/rpm/microshift.spec +# + +%package kindnet +Summary: kindnet CNI for MicroShift +ExclusiveArch: x86_64 aarch64 +Requires: microshift = %{version} + +%description kindnet +The microshift-kindnet package provides the required manifests for the kindnet CNI and the dependent +kube-proxy to be installed on MicroShift. + +%package kindnet-release-info +Summary: Release information for kindnet CNI for MicroShift +BuildArch: noarch +Requires: microshift-release-info = %{version} + +%description kindnet-release-info +The microshift-kindnet-release-info package provides release information files for this +release. These files contain the list of container image references used by the kindnet CNI +with the dependent kube-proxy for MicroShift. + +%install +install -d -m755 %{buildroot}/%{_sysconfdir}/microshift/config.d +install -d -m755 %{buildroot}/%{_sysconfdir}/microshift/manifests.d +install -d -m755 %{buildroot}%{_sysconfdir}/crio/crio.conf.d + +# kindnet +install -d -m755 %{buildroot}/%{_prefix}/lib/microshift/manifests.d/000-microshift-kindnet +install -d -m755 %{buildroot}%{_sysconfdir}/systemd/system +# Copy all the manifests except the arch specific ones +install -p -m644 assets/optional/kindnet/0* %{buildroot}/%{_prefix}/lib/microshift/manifests.d/000-microshift-kindnet +install -p -m644 assets/optional/kindnet/kustomization.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/000-microshift-kindnet +install -p -m644 packaging/kindnet/00-disableDefaultCNI.yaml %{buildroot}%{_sysconfdir}/microshift/config.d/00-disableDefaultCNI.yaml +install -p -m644 packaging/kindnet/microshift-kindnet.service %{buildroot}%{_sysconfdir}/systemd/system/microshift.service +install -p -m644 packaging/crio.conf.d/13-microshift-kindnet.conf %{buildroot}%{_sysconfdir}/crio/crio.conf.d/13-microshift-kindnet.conf + +%ifarch x86_64 +cat assets/optional/kindnet/kustomization.x86_64.yaml >> %{buildroot}/%{_prefix}/lib/microshift/manifests.d/000-microshift-kindnet/kustomization.yaml +%endif + +%ifarch %{arm} aarch64 +cat assets/optional/kindnet/kustomization.aarch64.yaml >> %{buildroot}/%{_prefix}/lib/microshift/manifests.d/000-microshift-kindnet/kustomization.yaml +%endif + +# kube-proxy +install -d -m755 %{buildroot}/%{_prefix}/lib/microshift/manifests.d/000-microshift-kube-proxy +# Copy all the manifests except the arch specific ones +install -p -m644 assets/optional/kube-proxy/0* %{buildroot}/%{_prefix}/lib/microshift/manifests.d/000-microshift-kube-proxy +install -p -m644 assets/optional/kube-proxy/kustomization.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/000-microshift-kube-proxy + +%ifarch x86_64 +cat assets/optional/kube-proxy/kustomization.x86_64.yaml >> %{buildroot}/%{_prefix}/lib/microshift/manifests.d/000-microshift-kube-proxy/kustomization.yaml +%endif + +%ifarch %{arm} aarch64 +cat assets/optional/kube-proxy/kustomization.aarch64.yaml >> %{buildroot}/%{_prefix}/lib/microshift/manifests.d/000-microshift-kube-proxy/kustomization.yaml +%endif + +# kindnet-release-info +mkdir -p -m755 %{buildroot}%{_datadir}/microshift/release +install -p -m644 assets/optional/kindnet/release-kindnet-{x86_64,aarch64}.json %{buildroot}%{_datadir}/microshift/release/ +install -p -m644 assets/optional/kube-proxy/release-kube-proxy-{x86_64,aarch64}.json %{buildroot}%{_datadir}/microshift/release/ + +%files kindnet +%dir %{_prefix}/lib/microshift/manifests.d/000-microshift-kindnet +%dir %{_prefix}/lib/microshift/manifests.d/000-microshift-kube-proxy +%{_prefix}/lib/microshift/manifests.d/000-microshift-kindnet/* +%{_prefix}/lib/microshift/manifests.d/000-microshift-kube-proxy/* +%config(noreplace) %{_sysconfdir}/microshift/config.d/00-disableDefaultCNI.yaml +%{_sysconfdir}/systemd/system/microshift.service +%{_sysconfdir}/crio/crio.conf.d/13-microshift-kindnet.conf + +%files kindnet-release-info +%{_datadir}/microshift/release/release-kindnet-{x86_64,aarch64}.json +%{_datadir}/microshift/release/release-kube-proxy-{x86_64,aarch64}.json diff --git a/src/quickstart.sh b/src/quickstart.sh index f8f70925..808bd141 100755 --- a/src/quickstart.sh +++ b/src/quickstart.sh @@ -37,10 +37,10 @@ function run_bootc_image() { local -r image_ref="$1" # Prerequisites for running the MicroShift container: - # - If the OVN-K CNI driver is used (`WITH_KINDNET=0` non-default build option), - # the `openvswitch` module must be loaded on the host. - # - If the TopoLVM CSI driver is used (`WITH_TOPOLVM=1` default build option), - # the /dev/dm-* device must be shared with the container. + # - If the OVN-K CNI driver is used (`WITH_KINDNET=0` non-default image build + # option), the `openvswitch` module must be loaded on the host. + # - If the TopoLVM CSI driver is used (`WITH_TOPOLVM=1` default image build + # option), the /dev/dm-* device must be shared with the container. echo "Running '${image_ref}'" modprobe openvswitch || true