diff --git a/test/performance/broker-latency/continuous/broker-latency-setup.yaml b/test/performance/broker-latency/continuous/broker-latency-setup.yaml new file mode 100644 index 00000000000..543096f4fc5 --- /dev/null +++ b/test/performance/broker-latency/continuous/broker-latency-setup.yaml @@ -0,0 +1,182 @@ +# Copyright 2019 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: eventing.knative.dev/v1alpha1 +kind: Broker +metadata: + name: in-memory + namespace: default +spec: + channelTemplateSpec: + apiVersion: messaging.knative.dev/v1alpha1 + kind: InMemoryChannel + +--- + +apiVersion: eventing.knative.dev/v1alpha1 +kind: Trigger +metadata: + name: broker-latency + namespace: default +spec: + broker: in-memory + subscriber: + ref: + apiVersion: v1 + kind: Service + name: broker-latency-in-memory-receiver + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: eventing-broker-ingress + namespace: default + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: eventing-broker-filter + namespace: default + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: perf-eventing + namespace: default + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: perf-eventing +rules: + - apiGroups: [""] + resources: ["nodes", "pods"] + verbs: ["list"] + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: perf-eventing +subjects: + - kind: ServiceAccount + name: perf-eventing + namespace: default +roleRef: + kind: ClusterRole + name: perf-eventing + apiGroup: rbac.authorization.k8s.io + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: eventing-broker-ingress + namespace: default +subjects: + - kind: ServiceAccount + name: eventing-broker-ingress + namespace: default +roleRef: + kind: ClusterRole + name: eventing-broker-ingress + apiGroup: rbac.authorization.k8s.io + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: eventing-broker-filter + namespace: default +subjects: + - kind: ServiceAccount + name: eventing-broker-filter + namespace: default +roleRef: + kind: ClusterRole + name: eventing-broker-filter + apiGroup: rbac.authorization.k8s.io + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: eventing-config-reader-default-eventing-broker-ingress + namespace: knative-eventing +subjects: + - kind: ServiceAccount + name: eventing-broker-ingress + namespace: default +roleRef: + kind: ClusterRole + name: eventing-config-reader + apiGroup: rbac.authorization.k8s.io + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: eventing-config-reader-default-eventing-broker-filter + namespace: knative-eventing +subjects: + - kind: ServiceAccount + name: eventing-broker-filter + namespace: default +roleRef: + kind: ClusterRole + name: eventing-config-reader + apiGroup: rbac.authorization.k8s.io + +--- + +apiVersion: v1 +kind: Service +metadata: + name: broker-latency-in-memory-receiver +spec: + selector: + role: broker-latency-in-memory-receiver + ports: + - name: http + port: 80 + targetPort: cloudevents + protocol: TCP + +--- + +apiVersion: v1 +kind: Service +metadata: + name: broker-latency-in-memory-aggregator +spec: + selector: + role: broker-latency-in-memory-aggregator + ports: + - name: grpc + port: 10000 + targetPort: grpc + protocol: TCP diff --git a/test/performance/broker-latency/continuous/broker-latency.yaml b/test/performance/broker-latency/continuous/broker-latency.yaml new file mode 100644 index 00000000000..e742e9b8fb0 --- /dev/null +++ b/test/performance/broker-latency/continuous/broker-latency.yaml @@ -0,0 +1,122 @@ +# Copyright 2019 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: broker-latency-in-memory-receiver + namespace: default + labels: + role: broker-latency-in-memory-receiver +spec: + schedule: "0,30 * * * *" + jobTemplate: + spec: + completions: 1 + parallelism: 1 + backoffLimit: 0 + template: + metadata: + labels: + role: broker-latency-in-memory-receiver + spec: + serviceAccountName: perf-eventing + restartPolicy: Never + containers: + - name: sender-receiver + image: knative.dev/eventing/test/test_images/latencymako + args: + - "--role=sender-receiver" + - "--sink=http://in-memory-broker" + - "--aggregator=broker-latency-in-memory-aggregator:10000" + - "--pace=100:10,200:20,400:60" + - "--verbose" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + requests: + cpu: 1000m + memory: 2Gi + ports: + - name: cloudevents + containerPort: 8080 + +--- + +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: broker-latency-in-memory-aggregator + namespace: default + labels: + role: broker-latency-in-memory-aggregator +spec: + schedule: "0,30 * * * *" + jobTemplate: + spec: + completions: 1 + parallelism: 1 + backoffLimit: 0 + template: + metadata: + labels: + role: broker-latency-in-memory-aggregator + spec: + serviceAccountName: perf-eventing + restartPolicy: Never + containers: + - name: aggregator + image: knative.dev/eventing/test/test_images/latencymako + args: + - "--role=aggregator" + # set to the number of sender-receiver + - "--expect-records=1" + - "--verbose" + ports: + - name: grpc + containerPort: 10000 + resources: + requests: + cpu: 1000m + memory: 2Gi + volumeMounts: + - name: config-mako + mountPath: /etc/config-mako + terminationMessagePolicy: FallbackToLogsOnError + - name: mako + image: gcr.io/knative-performance/mako-microservice:latest + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /var/secret/robot.json + volumeMounts: + - name: service-account + mountPath: /var/secret + ports: + - name: quickstore + containerPort: 9813 + terminationMessagePolicy: FallbackToLogsOnError + volumes: + - name: config-mako + configMap: + name: config-mako + - name: service-account + secret: + secretName: service-account \ No newline at end of file diff --git a/test/performance/broker-latency/broker-latency-setup.yaml b/test/performance/broker-latency/single/broker-latency-setup.yaml similarity index 87% rename from test/performance/broker-latency/broker-latency-setup.yaml rename to test/performance/broker-latency/single/broker-latency-setup.yaml index 8c5ea325e9b..06c7e73c9ec 100644 --- a/test/performance/broker-latency/broker-latency-setup.yaml +++ b/test/performance/broker-latency/single/broker-latency-setup.yaml @@ -12,18 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -apiVersion: v1 -kind: Namespace -metadata: - name: perf-eventing - ---- - apiVersion: eventing.knative.dev/v1alpha1 kind: Broker metadata: name: in-memory-test-broker - namespace: perf-eventing spec: channelTemplateSpec: apiVersion: messaging.knative.dev/v1alpha1 @@ -35,7 +27,6 @@ apiVersion: eventing.knative.dev/v1alpha1 kind: Trigger metadata: name: broker-latency - namespace: perf-eventing spec: broker: in-memory-test-broker subscriber: @@ -50,7 +41,6 @@ apiVersion: v1 kind: ServiceAccount metadata: name: eventing-broker-ingress - namespace: perf-eventing --- @@ -58,7 +48,6 @@ apiVersion: v1 kind: ServiceAccount metadata: name: eventing-broker-filter - namespace: perf-eventing --- @@ -66,7 +55,6 @@ apiVersion: v1 kind: ServiceAccount metadata: name: perf-eventing - namespace: perf-eventing --- @@ -88,7 +76,6 @@ metadata: subjects: - kind: ServiceAccount name: perf-eventing - namespace: perf-eventing roleRef: kind: ClusterRole name: perf-eventing @@ -100,11 +87,9 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: eventing-broker-ingress - namespace: perf-eventing subjects: - kind: ServiceAccount name: eventing-broker-ingress - namespace: perf-eventing roleRef: kind: ClusterRole name: eventing-broker-ingress @@ -116,11 +101,9 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: eventing-broker-filter - namespace: perf-eventing subjects: - kind: ServiceAccount name: eventing-broker-filter - namespace: perf-eventing roleRef: kind: ClusterRole name: eventing-broker-filter @@ -136,7 +119,6 @@ metadata: subjects: - kind: ServiceAccount name: eventing-broker-ingress - namespace: perf-eventing roleRef: kind: ClusterRole name: eventing-config-reader @@ -148,11 +130,9 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: eventing-config-reader-default-eventing-broker-filter - namespace: knative-eventing subjects: - kind: ServiceAccount name: eventing-broker-filter - namespace: perf-eventing roleRef: kind: ClusterRole name: eventing-config-reader @@ -164,7 +144,6 @@ apiVersion: v1 kind: Service metadata: name: broker-latency-consumer - namespace: perf-eventing spec: selector: role: broker-latency-consumer @@ -180,7 +159,6 @@ apiVersion: v1 kind: Service metadata: name: broker-latency-aggregator - namespace: perf-eventing spec: selector: role: broker-latency-aggregator diff --git a/test/performance/broker-latency/broker-latency.yaml b/test/performance/broker-latency/single/broker-latency.yaml similarity index 97% rename from test/performance/broker-latency/broker-latency.yaml rename to test/performance/broker-latency/single/broker-latency.yaml index acc5f8e5727..b8397d90166 100644 --- a/test/performance/broker-latency/broker-latency.yaml +++ b/test/performance/broker-latency/single/broker-latency.yaml @@ -16,7 +16,6 @@ apiVersion: batch/v1 kind: Job metadata: name: broker-latency-send-receive - namespace: perf-eventing labels: role: broker-latency-consumer spec: @@ -55,6 +54,7 @@ spec: ports: - name: cloudevents containerPort: 8080 + terminationMessagePolicy: FallbackToLogsOnError --- @@ -62,7 +62,6 @@ apiVersion: v1 kind: Pod metadata: name: broker-latency-aggregator - namespace: perf-eventing labels: role: broker-latency-aggregator spec: diff --git a/test/performance/tools/common.sh b/test/performance/tools/common.sh index 51315ee0815..06681f013fa 100644 --- a/test/performance/tools/common.sh +++ b/test/performance/tools/common.sh @@ -19,6 +19,8 @@ export PROJECT_NAME="knative-eventing-performance" export USER_NAME="mako-job@knative-eventing-performance.iam.gserviceaccount.com" export TEST_ROOT_PATH="$GOPATH/src/knative.dev/eventing/test/performance" export KO_DOCKER_REPO="gcr.io/knative-eventing-performance" +export TEST_CONFIG_VARIANT="continuous" +export TEST_NAMESPACE="default" function header() { echo "***** $1 *****" @@ -74,16 +76,6 @@ function get_gke_credentials() { gcloud container clusters get-credentials ${name} --zone=${zone} --project=${PROJECT_NAME} || abort "Failed to get cluster creds" } -# Delete all the benchmark resources -# $1 -> cluster_name -function delete_benchmark_resources() { - name=$1 - - echo ">> Delete all existing jobs and test resources" - kubectl delete job --all - ko delete -f "${TEST_ROOT_PATH}/$name" -} - # Install the eventing resources from the repo function install_eventing_resources() { pushd . @@ -110,7 +102,7 @@ function install_eventing_resources() { # the kubectl context's default namespace. function install_mako_resources() { echo ">> Setting up config-mako ConfigMap" - cat < cluster_name, $2 -> test config variant +function delete_benchmark_resources() { + name=$1 + variant=$2 + + echo ">> Delete all existing jobs and test resources" + kubectl delete job --all + ko delete -n ${TEST_NAMESPACE} -f "${TEST_ROOT_PATH}/${name}/${variant}/" || abort "Failed to delete ${name}/${variant} resources" +} + +# Apply all the benchmark resources +# $1 -> cluster_name, $2 -> test config variant function apply_benchmark_resources() { name=$1 + variant=$2 echo ">> Applying $name benchmark yamls" # install the service and cronjob to run the benchmark # NOTE: this assumes we have a benchmark with the same name as the cluster # If service creation takes long time, we will have some initially unreachable errors in the test echo "Using ko version $(ko version)" - ko apply -f "$TEST_ROOT_PATH/$name" || abort "Failed to apply $name benchmark yamls" + ko apply -n ${TEST_NAMESPACE} -f "$TEST_ROOT_PATH/$name/${variant}/" || abort "Failed to apply ${name}/${variant} benchmark yamls" } # Update resources installed on the cluster with the up-to-date code. This # assumes the benchmark is running in the kubectl context's default namespace. -# $1 -> cluster_name, $2 -> cluster_zone +# $1 -> cluster_name function update_cluster() { name=$1 install_eventing_resources install_mako_resources - delete_benchmark_resources $name - apply_benchmark_resources $name + delete_benchmark_resources $name $TEST_CONFIG_VARIANT + apply_benchmark_resources $name $TEST_CONFIG_VARIANT } # Create a new cluster and install serving components and apply benchmark yamls. diff --git a/test/test_images/latencymako/kodata/prod.config b/test/test_images/latencymako/kodata/prod.config new file mode 120000 index 00000000000..6115d5c8651 --- /dev/null +++ b/test/test_images/latencymako/kodata/prod.config @@ -0,0 +1 @@ +../prod.config \ No newline at end of file diff --git a/test/test_images/latencymako/prod.config b/test/test_images/latencymako/prod.config new file mode 100644 index 00000000000..403ff3b7f36 --- /dev/null +++ b/test/test_images/latencymako/prod.config @@ -0,0 +1,42 @@ +### Creating this benchmark: +# mako create_benchmark test/performance/broker-latency/prod.config +### Updating this benchmark: +# mako update_benchmark test/performance/broker-latency/prod.config + +project_name: "Knative" +benchmark_name: "Eventing Broker Latency" +description: "Measure broker latency and reliability." +benchmark_key: "5036156928393216" + +# Human owners for manual benchmark adjustments. +owner_list: "grantrodgers@google.com" +owner_list: "chizhg@google.com" +owner_list: "srinivashegde@google.com" + +# Only this robot should publish data to Mako for this key! +owner_list: "mako-job@knative-eventing-performance.iam.gserviceaccount.com" + +# Define the name and type for x-axis of run charts +input_value_info: { + value_key: "t" + label: "time" + type: TIMESTAMP +} + +# Note: value_key is stored repeatedly and should be very short (ideally one or two characters). +metric_info_list: { + value_key: "pl" + label: "publish-latency" +} +metric_info_list: { + value_key: "pe" + label: "publish-errors" +} +metric_info_list: { + value_key: "dl" + label: "deliver-latency" +} +metric_info_list: { + value_key: "de" + label: "deliver-errors" +}