diff --git a/test/performance/broker-imc/continuous/100-broker-imc-setup.yaml b/test/performance/broker-imc/continuous/100-broker-imc-setup.yaml new file mode 100644 index 00000000000..7ed5fe24e3b --- /dev/null +++ b/test/performance/broker-imc/continuous/100-broker-imc-setup.yaml @@ -0,0 +1,182 @@ +# Copyright 2019 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: eventing.knative.dev/v1alpha1 +kind: Broker +metadata: + name: imc + namespace: default +spec: + channelTemplateSpec: + apiVersion: messaging.knative.dev/v1alpha1 + kind: InMemoryChannel + +--- + +apiVersion: eventing.knative.dev/v1alpha1 +kind: Trigger +metadata: + name: broker-imc + namespace: default +spec: + broker: imc + subscriber: + ref: + apiVersion: v1 + kind: Service + name: broker-imc-receiver + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: eventing-broker-ingress + namespace: default + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: eventing-broker-filter + namespace: default + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: perf-eventing + namespace: default + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: perf-eventing +rules: + - apiGroups: [""] + resources: ["nodes", "pods"] + verbs: ["list"] + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: perf-eventing +subjects: + - kind: ServiceAccount + name: perf-eventing + namespace: default +roleRef: + kind: ClusterRole + name: perf-eventing + apiGroup: rbac.authorization.k8s.io + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: eventing-broker-ingress + namespace: default +subjects: + - kind: ServiceAccount + name: eventing-broker-ingress + namespace: default +roleRef: + kind: ClusterRole + name: eventing-broker-ingress + apiGroup: rbac.authorization.k8s.io + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: eventing-broker-filter + namespace: default +subjects: + - kind: ServiceAccount + name: eventing-broker-filter + namespace: default +roleRef: + kind: ClusterRole + name: eventing-broker-filter + apiGroup: rbac.authorization.k8s.io + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: eventing-config-reader-default-eventing-broker-ingress + namespace: knative-eventing +subjects: + - kind: ServiceAccount + name: eventing-broker-ingress + namespace: default +roleRef: + kind: ClusterRole + name: eventing-config-reader + apiGroup: rbac.authorization.k8s.io + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: eventing-config-reader-default-eventing-broker-filter + namespace: knative-eventing +subjects: + - kind: ServiceAccount + name: eventing-broker-filter + namespace: default +roleRef: + kind: ClusterRole + name: eventing-config-reader + apiGroup: rbac.authorization.k8s.io + +--- + +apiVersion: v1 +kind: Service +metadata: + name: broker-imc-receiver +spec: + selector: + role: broker-imc-receiver + ports: + - name: http + port: 80 + targetPort: cloudevents + protocol: TCP + +--- + +apiVersion: v1 +kind: Service +metadata: + name: broker-imc-aggregator +spec: + selector: + role: broker-imc-aggregator + ports: + - name: grpc + port: 10000 + targetPort: grpc + protocol: TCP diff --git a/test/performance/broker-imc/continuous/200-broker-imc.yaml b/test/performance/broker-imc/continuous/200-broker-imc.yaml new file mode 100644 index 00000000000..aaaf9a4b5ff --- /dev/null +++ b/test/performance/broker-imc/continuous/200-broker-imc.yaml @@ -0,0 +1,133 @@ +# Copyright 2019 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: broker-imc-receiver + namespace: default + labels: + role: broker-imc-receiver +spec: + schedule: "0/15 * * * *" + # History must be zero to ensure no failed pods stick around and block the next job + successfulJobsHistoryLimit: 0 + failedJobsHistoryLimit: 0 + jobTemplate: + spec: + completions: 1 + parallelism: 1 + backoffLimit: 0 + # Allow up to 14 minutes, then clean up to make room for the next attempt + activeDeadlineSeconds: 840 + template: + metadata: + labels: + role: broker-imc-receiver + spec: + serviceAccountName: perf-eventing + restartPolicy: Never + containers: + - name: sender-receiver + image: knative.dev/eventing/test/test_images/performance + args: + - "--role=sender-receiver" + - "--sink=http://imc-broker" + - "--aggregator=broker-imc-aggregator:10000" + - "--pace=100:10,400:20,800:30,1000:60,2000:60,3000:60" + - "--verbose" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + requests: + cpu: 1000m + memory: 2Gi + ports: + - name: cloudevents + containerPort: 8080 + +--- + +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: broker-imc-aggregator + namespace: default + labels: + role: broker-imc-aggregator +spec: + schedule: "0/15 * * * *" + # History must be zero to ensure no failed pods stick around and block the next job + successfulJobsHistoryLimit: 0 + failedJobsHistoryLimit: 0 + jobTemplate: + spec: + completions: 1 + parallelism: 1 + backoffLimit: 0 + # Allow up to 14 minutes, then clean up to make room for the next attempt + activeDeadlineSeconds: 840 + template: + metadata: + labels: + role: broker-imc-aggregator + spec: + serviceAccountName: perf-eventing + restartPolicy: Never + containers: + - name: aggregator + image: knative.dev/eventing/test/test_images/performance + args: + - "--role=aggregator" + # set to the number of sender-receiver + - "--expect-records=1" + - "--mako-tags=channel=imc" + - "--verbose" + ports: + - name: grpc + containerPort: 10000 + resources: + requests: + cpu: 1000m + memory: 2Gi + volumeMounts: + - name: config-mako + mountPath: /etc/config-mako + terminationMessagePolicy: FallbackToLogsOnError + - name: mako + image: gcr.io/knative-performance/mako-microservice:latest + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /var/secret/robot.json + volumeMounts: + - name: service-account + mountPath: /var/secret + ports: + - name: quickstore + containerPort: 9813 + terminationMessagePolicy: FallbackToLogsOnError + volumes: + - name: config-mako + configMap: + name: config-mako + - name: service-account + secret: + secretName: service-account diff --git a/test/performance/tools/common.sh b/test/performance/tools/common.sh index 51315ee0815..541b0a257e3 100644 --- a/test/performance/tools/common.sh +++ b/test/performance/tools/common.sh @@ -19,6 +19,8 @@ export PROJECT_NAME="knative-eventing-performance" export USER_NAME="mako-job@knative-eventing-performance.iam.gserviceaccount.com" export TEST_ROOT_PATH="$GOPATH/src/knative.dev/eventing/test/performance" export KO_DOCKER_REPO="gcr.io/knative-eventing-performance" +export TEST_CONFIG_VARIANT="continuous" +export TEST_NAMESPACE="default" function header() { echo "***** $1 *****" @@ -74,16 +76,6 @@ function get_gke_credentials() { gcloud container clusters get-credentials ${name} --zone=${zone} --project=${PROJECT_NAME} || abort "Failed to get cluster creds" } -# Delete all the benchmark resources -# $1 -> cluster_name -function delete_benchmark_resources() { - name=$1 - - echo ">> Delete all existing jobs and test resources" - kubectl delete job --all - ko delete -f "${TEST_ROOT_PATH}/$name" -} - # Install the eventing resources from the repo function install_eventing_resources() { pushd . @@ -110,7 +102,7 @@ function install_eventing_resources() { # the kubectl context's default namespace. function install_mako_resources() { echo ">> Setting up config-mako ConfigMap" - cat < cluster_name, $2 -> test config variant +function delete_benchmark_resources() { + name=$1 + variant=$2 + + echo ">> Delete all existing jobs and test resources" + kubectl delete job --all + ko delete -n ${TEST_NAMESPACE} -f "${TEST_ROOT_PATH}/${name}/${variant}/" || abort "Failed to delete ${name}/${variant} resources" +} + +# Apply all the benchmark resources +# $1 -> cluster_name, $2 -> test config variant function apply_benchmark_resources() { name=$1 + variant=$2 echo ">> Applying $name benchmark yamls" # install the service and cronjob to run the benchmark # NOTE: this assumes we have a benchmark with the same name as the cluster # If service creation takes long time, we will have some initially unreachable errors in the test echo "Using ko version $(ko version)" - ko apply -f "$TEST_ROOT_PATH/$name" || abort "Failed to apply $name benchmark yamls" + ko apply -n ${TEST_NAMESPACE} -f "$TEST_ROOT_PATH/$name/${variant}/" || abort "Failed to apply ${name}/${variant} benchmark yamls" } # Update resources installed on the cluster with the up-to-date code. This # assumes the benchmark is running in the kubectl context's default namespace. -# $1 -> cluster_name, $2 -> cluster_zone +# $1 -> cluster_name function update_cluster() { name=$1 install_eventing_resources install_mako_resources - delete_benchmark_resources $name - apply_benchmark_resources $name + delete_benchmark_resources $name $TEST_CONFIG_VARIANT + apply_benchmark_resources $name $TEST_CONFIG_VARIANT } # Create a new cluster and install serving components and apply benchmark yamls. diff --git a/test/test_images/performance/dev.config b/test/test_images/performance/dev.config index 7116534a492..9a20441dba5 100644 --- a/test/test_images/performance/dev.config +++ b/test/test_images/performance/dev.config @@ -4,8 +4,8 @@ # mako update_benchmark test/performance/broker-latency/dev.config project_name: "Knative" -benchmark_name: "Development - Eventing Latency & Throughput" -description: "Measure latency and throughput of eventing components." +benchmark_name: "Development - Broker Latency & Throughput" +description: "Measure latency and throughput of the broker using various channels." benchmark_key: '5903682180743168' # Human owners for manual benchmark adjustments. @@ -13,7 +13,7 @@ owner_list: "grantrodgers@google.com" owner_list: "chizhg@google.com" # Anyone can add their IAM robot here to publish to this benchmark. -owner_list: "mako-job@knative-performance.iam.gserviceaccount.com" +owner_list: "mako-job@knative-eventing-performance.iam.gserviceaccount.com" # This is grantrodgers' robot: owner_list: "mako-upload@grantrodgers-crd.iam.gserviceaccount.com" @@ -52,4 +52,4 @@ metric_info_list: { metric_info_list: { value_key: "ft" label: "failure-throughput" -} \ No newline at end of file +} diff --git a/test/test_images/performance/kodata/prod.config b/test/test_images/performance/kodata/prod.config new file mode 120000 index 00000000000..6115d5c8651 --- /dev/null +++ b/test/test_images/performance/kodata/prod.config @@ -0,0 +1 @@ +../prod.config \ No newline at end of file diff --git a/test/test_images/performance/prod.config b/test/test_images/performance/prod.config new file mode 100644 index 00000000000..12b9b31db6b --- /dev/null +++ b/test/test_images/performance/prod.config @@ -0,0 +1,54 @@ +### Creating this benchmark: +# mako create_benchmark test/performance/broker-latency/prod.config +### Updating this benchmark: +# mako update_benchmark test/performance/broker-latency/prod.config + +project_name: "Knative" +benchmark_name: "Broker Latency & Throughput" +description: "Measure latency and throughput of the broker using various channels." +benchmark_key: '5036156928393216' + +# Human owners for manual benchmark adjustments. +owner_list: "grantrodgers@google.com" +owner_list: "chizhg@google.com" +owner_list: "srinivashegde@google.com" + +# Only this robot should publish data to Mako for this key! +owner_list: "mako-job@knative-eventing-performance.iam.gserviceaccount.com" + +# Define the name and type for x-axis of run charts +input_value_info: { + value_key: "t" + label: "time" + type: TIMESTAMP +} + +# Note: value_key is stored repeatedly and should be very short (ideally one or two characters). +metric_info_list: { + value_key: "pl" + label: "publish-latency" +} +metric_info_list: { + value_key: "pe" + label: "publish-errors" +} +metric_info_list: { + value_key: "st" + label: "send-throughput" +} +metric_info_list: { + value_key: "dl" + label: "deliver-latency" +} +metric_info_list: { + value_key: "de" + label: "deliver-errors" +} +metric_info_list: { + value_key: "dt" + label: "deliver-throughput" +} +metric_info_list: { + value_key: "ft" + label: "failure-throughput" +}