test/test-framework: remove, and update/remove references and dependent tests

.gitignore

@@ -103,7 +103,6 @@ images/scorecard-test-kuttl/scorecard-test-kuttl
 # Test artifacts
 pkg/ansible/runner/testdata/valid.yaml
 /bin
-test/test-framework/go.sum
 internal/scorecard/alpha/examples/custom-scorecard-tests/go.sum
 
 # Website

.travis.yml

@@ -129,18 +129,19 @@ jobs:
 
     # Test subcommands
     - <<: *test
-      name: Subcommands on Kubernetes
-      script: make test-subcommand
+      name: Subcommands and Integration on Kubernetes
+      script:
+      - make test-subcommand
+      - make test-integration
 
-    # Build and test go 
+    # Build and test go
     - <<: *test
       name: Go on Kubernetes
       before_script:
       - (cd / && go get github.com/mattn/goveralls)
       script:
       - make test-unit
       - make test-e2e-go
-      - make test-integration
       after_success:
       - $GOPATH/bin/goveralls -service=travis-ci -coverprofile=coverage.out -repotoken=$COVERALLS_TOKEN
 
@@ -216,7 +217,7 @@ jobs:
         - make image-build-helm
         - make image-push-helm
 
-    # Build and deploy arm64 scorecard-test docker image 
+    # Build and deploy arm64 scorecard-test docker image
     - <<: *deploy
       name: Docker image for scorecard-test (arm64)
       arch: arm64

Makefile

@@ -96,16 +96,13 @@ setup-k8s:
 
 ##@ Generate
 
-.PHONY: generate gen-cli-doc gen-test-framework gen-changelog
+.PHONY: generate gen-cli-doc gen-changelog
 
-generate: gen-cli-doc gen-test-framework  ## Run all non-release generate targets
+generate: gen-cli-doc  ## Run all non-release generate targets
 
 gen-cli-doc: ## Generate CLI documentation
 	./hack/generate/cli-doc/gen-cli-doc.sh
 
-gen-test-framework: build/operator-sdk ## Run generate commands to update test/test-framework
-	./hack/generate/test-framework/gen-test-framework.sh
-
 gen-changelog: ## Generate CHANGELOG.md and migration guide updates
 	./hack/generate/changelog/gen-changelog.sh
 
@@ -269,8 +266,6 @@ test-ci: test-sanity test-unit install test-subcommand test-e2e ## Run the CI te
 .PHONY: test-subcommand test-subcommand-olm-install
 
 test-subcommand: test-subcommand-olm-install
-	./hack/tests/subcommand-bundle.sh
-	./hack/tests/subcommand-generate-csv.sh
 
 test-subcommand-olm-install:
 	./hack/tests/subcommand-olm-install.sh

hack/ci/setup-k8s.sh

@@ -1,31 +1,27 @@
 #!/usr/bin/env bash
 
-set -eux
+set -eu
 
-# This image tag corresponds to a Kubernetes version that kind installs using
-# images at:
+source hack/lib/common.sh
+
+# This image tag corresponds to a Kubernetes version that kind installs using node images:
 # https://hub.docker.com/r/kindest/node/tags
 K8S_VERSION=$1
-KIND_VERSION="v0.8.1"
 KIND_IMAGE="docker.io/kindest/node:${K8S_VERSION}"
 
-# Download the latest version of kind, which supports all versions of
-# Kubernetes v1.11+.
-curl -Lo kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
-chmod +x kind
-sudo mv kind /usr/local/bin/
+# Download the latest version of kind and kubectl, which are needed for project unit and e2e tests.
+prepare_staging_dir $tmp_sdk_root
+install_kind $tmp_sdk_root
+install_kubectl $tmp_sdk_root
+fetch_tools $tmp_sdk_root
+setup_envs $tmp_sdk_root
 
 # Create a cluster of version $K8S_VERSION.
 kind create cluster --image="$KIND_IMAGE"
 
 # Run this command externally after installation:
 kind export kubeconfig
 
-# kubectl is needed for the single namespace local test and the ansible tests.
-curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl
-chmod +x kubectl
-sudo mv kubectl /usr/local/bin/
-
 # Use the "default" namespace, and ensure kind is running.
 kubectl config set-context --current --namespace=default
 kubectl cluster-info

hack/lib/common.sh

@@ -26,6 +26,14 @@ else
   error_color=''
   reset_color=''
 fi
+# Operating system and architecture name of the local machine.
+ARCH_NAME="$(arch)"
+case "$ARCH_NAME" in
+  x86_64) ARCH_NAME="amd64" ;;
+  aarch64) ARCH_NAME="arm64" ;;
+  *);;
+esac
+OS_NAME="$(uname | awk '{ print tolower($0) }')"
 
 # Roots used by tests.
 tmp_root=/tmp
@@ -67,14 +75,13 @@ function prepare_staging_dir {
     rm -f "$1/bin/operator-sdk"
   fi
 
-  mkdir -p "$1"
+  mkdir -p "${1}/bin"
 }
 
 # Fetch k8s API gen tools and make it available under $1/bin.
 function fetch_tools {
   if [[ -z "$SKIP_FETCH_TOOLS" ]]; then
     fetch_envtest_tools $@
-    install_kind $@
   fi
 }
 
@@ -89,7 +96,7 @@ function fetch_envtest_tools {
   # curl -fL --retry 3 --keepalive-time 2 "${url}" -o "${tmp_sdk_root}/${server_tar}"
   # tar -zxvf "${tmp_sdk_root}/${server_tar}"
 
-  local tools_archive_name="kubebuilder-tools-${ENVTEST_TOOLS_VERSION}-$(go env GOOS)-$(go env GOARCH).tar.gz"
+  local tools_archive_name="kubebuilder-tools-${ENVTEST_TOOLS_VERSION}-${OS_NAME}-${ARCH_NAME}.tar.gz"
   local tools_download_url="https://storage.googleapis.com/kubebuilder-tools/$tools_archive_name"
 
   local tools_archive_path="$1/$tools_archive_name"
@@ -122,12 +129,21 @@ function build_sdk {
 
 # Install the 'kind' binary at version $KIND_VERSION.
 function install_kind {
-
   local kind_path="${1}/bin/kind"
 
   header_text "installing kind $KIND_VERSION"
-  local kind_binary="kind-$(go env GOOS)-$(go env GOARCH)"
+  local kind_binary="kind-${OS_NAME}-${ARCH_NAME}"
   local kind_url="https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/$kind_binary"
   curl -sSLo "$kind_path" $kind_url
   chmod +x "$kind_path"
 }
+
+# Install the 'kubectl' binary at version $K8S_VERSION.
+function install_kubectl() {
+  local kubectl_path="${1}/bin/kubectl"
+
+  header_text "installing kubectl $K8S_VERSION"
+  local kubectl_url="https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/${OS_NAME}/${ARCH_NAME}/kubectl"
+  curl -sSLo "$kubectl_path" $kubectl_url
+  chmod +x "$kubectl_path"
+}

hack/lib/test_lib.sh

@@ -42,34 +42,6 @@ function trap_add() {
     done
 }
 
-# add_go_mod_replace adds a "replace" directive from $1 to $2 with an
-# optional version version $3 to the current working directory's go.mod file.
-function add_go_mod_replace() {
-	local from_path="${1:?first path in replace statement is required}"
-	local to_path="${2:?second path in replace statement is required}"
-	local version="${3:-}"
-
-	if [[ ! -d "$to_path" && -z "$version" ]]; then
-		echo "second replace path $to_path requires a version be set because it is not a directory"
-		exit 1
-	fi
-	if [[ ! -e go.mod ]]; then
-		echo "go.mod file not found in $(pwd)"
-		exit 1
-	fi
-
-	# Check if a replace line already exists. If it does, remove. If not, append.
-	if grep -q "${from_path} =>" go.mod; then
-		sed -E -i 's|^.+'"${from_path} =>"'.+$||g' go.mod
-	fi
-	# Do not use "go mod edit" so formatting stays the same.
-	local replace="replace ${from_path} => ${to_path}"
-	if [[ -n "$version" ]]; then
-		replace="$replace $version"
-	fi
-	echo "$replace" >> go.mod
-}
-
 # check_dir accepts 3 args:
 # 1: test case string
 # 2: directory to test for existence

hack/tests/e2e-ansible-molecule.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 
+source hack/lib/common.sh
 source hack/lib/test_lib.sh
 source hack/lib/image_lib.sh
 
@@ -16,6 +17,8 @@ pip3 install --user ansible-lint yamllint
 pip3 install --user docker openshift jmespath
 ansible-galaxy collection install community.kubernetes
 
+setup_envs $tmp_sdk_root
+
 remove_prereqs() {
     header_text "Deleting resources"
     kubectl delete --wait=true --ignore-not-found=true --timeout=60s -f "$OPERATORDIR/deploy/crds/ansible.example.com_memcacheds_crd.yaml"

hack/tests/e2e-ansible.sh

@@ -14,6 +14,8 @@ ROOTDIR="$(pwd)"
 TMPDIR="$(mktemp -d)"
 trap_add 'rm -rf $TMPDIR' EXIT
 
+setup_envs $tmp_sdk_root
+
 deploy_operator() {
     header_text "Running deploy operator"
     kubectl create -f "$OPERATORDIR/deploy/service_account.yaml"

hack/tests/e2e-go.sh

@@ -13,14 +13,7 @@ tests=$test_dir/e2e
 export TRACE=1
 export GO111MODULE=on
 
-prepare_staging_dir $tmp_sdk_root
-fetch_tools $tmp_sdk_root
-# These envtest environment variables are required for the default unit tests
-# scaffolded in the test operator project. No e2e tests currently use envtest.
 setup_envs $tmp_sdk_root
-build_sdk $tmp_sdk_root
-
-kubectl cluster-info
 
 docker pull gcr.io/kubebuilder/kube-rbac-proxy:v0.5.0
 load_image_if_kind gcr.io/kubebuilder/kube-rbac-proxy:v0.5.0

hack/tests/e2e-helm.sh

@@ -10,6 +10,8 @@ ROOTDIR="$(pwd)"
 TMPDIR="$(mktemp -d)"
 trap_add 'rm -rf $TMPDIR' EXIT
 
+setup_envs $tmp_sdk_root
+
 test_namespace="test-e2e-helm"
 
 deploy_operator() {

hack/tests/integration.sh

@@ -2,15 +2,29 @@
 
 set -eu
 
+source hack/lib/common.sh
+source hack/lib/test_lib.sh
 source hack/lib/image_lib.sh
 
-export OSDK_INTEGRATION_IMAGE="quay.io/example/memcached-operator:latest"
+TMPDIR="$(mktemp -d -p /tmp memcached-operator-XXXX)"
+trap_add 'rm -rf $TMPDIR' EXIT
+pushd "$TMPDIR"
 
-# Build the operator image.
-pushd test/test-framework
-operator-sdk build "$OSDK_INTEGRATION_IMAGE"
-# If using a kind cluster, load the image into all nodes.
+setup_envs $tmp_sdk_root
+
+header_text "Initializing test project"
+
+# Initialize a basic memcached-operator project
+operator-sdk init --repo github.com/example/memcached-operator --domain example.com --fetch-deps=false
+operator-sdk create api --group cache --version v1alpha1 --kind Memcached --controller --resource
+sed -i 's@Foo string `json:"foo,omitempty"`@// +optional\
+        Count int `json:"count,omitempty"`@' api/v1alpha1/memcached_types.go
+
+# Build the operator's image.
+export OSDK_INTEGRATION_IMAGE="quay.io/example/memcached-operator:integration"
+make docker-build IMG="$OSDK_INTEGRATION_IMAGE"
 load_image_if_kind "$OSDK_INTEGRATION_IMAGE"
+
 popd
 
 # Install OLM on the cluster if not installed.
@@ -20,13 +34,17 @@ if ! operator-sdk olm status > /dev/null 2>&1; then
   olm_latest_exists=1
 fi
 
-# Integration tests will use default loading rules for the kubeconfig if
-# KUBECONFIG is not set.
+docker pull gcr.io/kubebuilder/kube-rbac-proxy:v0.5.0
+load_image_if_kind gcr.io/kubebuilder/kube-rbac-proxy:v0.5.0
+
+header_text "Running integration tests"
+
+# Integration tests will use default loading rules for the kubeconfig if KUBECONFIG is not set.
 go test -v ./test/integration
 
+header_text "Integration tests succeeded"
+
 # Uninstall OLM if it was installed for test purposes.
 if eval "(( $olm_latest_exists ))"; then
   operator-sdk olm uninstall
 fi
-
-echo -e "\n=== Integration tests succeeded ===\n"

hack/tests/sanity-check.sh

@@ -8,7 +8,6 @@ go fmt ./...
 ./hack/check-license.sh
 ./hack/check-error-log-msg-format.sh
 ./hack/generate/cli-doc/gen-cli-doc.sh
-./hack/generate/test-framework/gen-test-framework.sh
 go run ./hack/generate/changelog/gen-changelog.go -validate-only
 
 # Make sure repo is still in a clean state.