diff --git a/Gopkg.lock b/Gopkg.lock index fe4d8b7a92b..b874e3bebd3 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1201,7 +1201,7 @@ [[projects]] branch = "master" - digest = "1:03c8c36f66fba88d6184f87e0bcc24262a53efc9d47adbd9a071a69ad4c001fc" + digest = "1:2aee75bf4e9a4c7a79ab50bb2fe147f7ae8ab7e00967b4a848e7283363c79213" name = "knative.dev/pkg" packages = [ "apis", @@ -1266,6 +1266,7 @@ "profiling", "reconciler/testing", "signals", + "source", "system", "system/testing", "test", @@ -1284,7 +1285,7 @@ "webhook", ] pruneopts = "T" - revision = "c39ee25c42f53e4024d60511a2edef2065ea7333" + revision = "7f779625563a8117c7756af9fe40912b1d5e679e" [[projects]] branch = "master" @@ -1427,6 +1428,7 @@ "knative.dev/pkg/metrics/testing", "knative.dev/pkg/reconciler/testing", "knative.dev/pkg/signals", + "knative.dev/pkg/source", "knative.dev/pkg/system", "knative.dev/pkg/system/testing", "knative.dev/pkg/test", diff --git a/cmd/apiserver_receive_adapter/main.go b/cmd/apiserver_receive_adapter/main.go index 372a57bc882..9965b9074c0 100644 --- a/cmd/apiserver_receive_adapter/main.go +++ b/cmd/apiserver_receive_adapter/main.go @@ -36,6 +36,7 @@ import ( "knative.dev/pkg/logging" "knative.dev/pkg/metrics" "knative.dev/pkg/signals" + "knative.dev/pkg/source" ) const ( @@ -112,7 +113,7 @@ func main() { logger.Error("failed to create the metrics exporter", zap.Error(err)) } - reporter, err := metrics.NewStatsReporter() + reporter, err := source.NewStatsReporter() if err != nil { logger.Error("error building statsreporter", zap.Error(err)) } diff --git a/pkg/adapter/apiserver/adapter.go b/pkg/adapter/apiserver/adapter.go index 948d7afe61a..b1a98fad95b 100644 --- a/pkg/adapter/apiserver/adapter.go +++ b/pkg/adapter/apiserver/adapter.go @@ -20,6 +20,8 @@ import ( "fmt" "time" + cloudevents "github.com/cloudevents/sdk-go" + "go.uber.org/zap" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -27,10 +29,7 @@ import ( "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/dynamic" "k8s.io/client-go/tools/cache" - - cloudevents "github.com/cloudevents/sdk-go" - "go.uber.org/zap" - "knative.dev/pkg/metrics" + "knative.dev/pkg/source" ) type Adapter interface { @@ -70,13 +69,13 @@ type adapter struct { mode string delegate eventDelegate - reporter metrics.StatsReporter + reporter source.StatsReporter name string } func NewAdaptor(source string, k8sClient dynamic.Interface, ceClient cloudevents.Client, logger *zap.SugaredLogger, - opt Options, reporter metrics.StatsReporter, name string) Adapter { + opt Options, reporter source.StatsReporter, name string) Adapter { mode := opt.Mode switch mode { case ResourceMode, RefMode: diff --git a/pkg/adapter/apiserver/adapter_test.go b/pkg/adapter/apiserver/adapter_test.go index 4e5e0ca8665..6c12b56ca07 100644 --- a/pkg/adapter/apiserver/adapter_test.go +++ b/pkg/adapter/apiserver/adapter_test.go @@ -30,14 +30,14 @@ import ( dynamicfake "k8s.io/client-go/dynamic/fake" kncetesting "knative.dev/eventing/pkg/kncloudevents/testing" rectesting "knative.dev/eventing/pkg/reconciler/testing" - "knative.dev/pkg/metrics" + "knative.dev/pkg/source" ) type mockReporter struct { eventCount int } -func (r *mockReporter) ReportEventCount(args *metrics.ReportArgs, responseCode int) error { +func (r *mockReporter) ReportEventCount(args *source.ReportArgs, responseCode int) error { r.eventCount += 1 return nil } @@ -324,7 +324,7 @@ func makeRefAndTestingClient() (*ref, *kncetesting.TestCloudEventsClient) { }, ce } -func validateMetric(t *testing.T, reporter metrics.StatsReporter, want int) { +func validateMetric(t *testing.T, reporter source.StatsReporter, want int) { if mockReporter, ok := reporter.(*mockReporter); !ok { t.Errorf("reporter is not a mockReporter") } else if mockReporter.eventCount != want { diff --git a/pkg/adapter/apiserver/ref.go b/pkg/adapter/apiserver/ref.go index a3e7b7a1a79..0b8dda9f65b 100644 --- a/pkg/adapter/apiserver/ref.go +++ b/pkg/adapter/apiserver/ref.go @@ -18,17 +18,16 @@ package apiserver import ( "context" - "knative.dev/pkg/metrics" "reflect" + cloudevents "github.com/cloudevents/sdk-go" + "go.uber.org/zap" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/tools/cache" - - cloudevents "github.com/cloudevents/sdk-go" - "go.uber.org/zap" "knative.dev/eventing/pkg/adapter/apiserver/events" + "knative.dev/pkg/source" ) type ref struct { @@ -38,7 +37,7 @@ type ref struct { logger *zap.SugaredLogger controlledGVRs []schema.GroupVersionResource - reporter metrics.StatsReporter + reporter source.StatsReporter namespace string name string } @@ -106,7 +105,7 @@ func (a *ref) addControllerWatch(gvr schema.GroupVersionResource) { } func (a *ref) sendEvent(ctx context.Context, event *cloudevents.Event) error { - reportArgs := &metrics.ReportArgs{ + reportArgs := &source.ReportArgs{ Namespace: a.namespace, EventSource: event.Source(), EventType: event.Type(), diff --git a/pkg/adapter/apiserver/resource.go b/pkg/adapter/apiserver/resource.go index 8f129c5980b..557b0db122d 100644 --- a/pkg/adapter/apiserver/resource.go +++ b/pkg/adapter/apiserver/resource.go @@ -24,7 +24,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/tools/cache" "knative.dev/eventing/pkg/adapter/apiserver/events" - "knative.dev/pkg/metrics" + "knative.dev/pkg/source" ) type resource struct { @@ -32,7 +32,7 @@ type resource struct { source string eventType string logger *zap.SugaredLogger - reporter metrics.StatsReporter + reporter source.StatsReporter namespace string name string } @@ -72,7 +72,7 @@ func (a *resource) Delete(obj interface{}) error { } func (a *resource) sendEvent(ctx context.Context, event *cloudevents.Event) error { - reportArgs := &metrics.ReportArgs{ + reportArgs := &source.ReportArgs{ Namespace: a.namespace, EventSource: event.Source(), EventType: event.Type(), diff --git a/vendor/knative.dev/pkg/Gopkg.lock b/vendor/knative.dev/pkg/Gopkg.lock index 1274587e2cc..d831bab2554 100644 --- a/vendor/knative.dev/pkg/Gopkg.lock +++ b/vendor/knative.dev/pkg/Gopkg.lock @@ -1245,14 +1245,14 @@ [[projects]] branch = "master" - digest = "1:e873112f19e9ce823bc01ca715de355bd5cf8e4d6045ed5f34b3eb0d6ec2c655" + digest = "1:615f3c6b974179c583edf50234fd87c5731d02ef941f314cbd4dd2766d3a619a" name = "knative.dev/test-infra" packages = [ "scripts", "tools/dep-collector", ] pruneopts = "UT" - revision = "5449c8bad49d3528b5cb0b58a0cdce0f8a526f03" + revision = "816123d5e71fb88533d1564c331443387967e1dc" [[projects]] digest = "1:8730e0150dfb2b7e173890c8b9868e7a273082ef8e39f4940e3506a481cf895c" diff --git a/vendor/knative.dev/pkg/metrics/metricskey/constants_eventing.go b/vendor/knative.dev/pkg/metrics/metricskey/constants_eventing.go index 249c582f48b..ff35312f8c7 100644 --- a/vendor/knative.dev/pkg/metrics/metricskey/constants_eventing.go +++ b/vendor/knative.dev/pkg/metrics/metricskey/constants_eventing.go @@ -27,8 +27,8 @@ const ( // ResourceTypeKnativeBroker is the Stackdriver resource type for Knative Brokers. ResourceTypeKnativeBroker = "knative_broker" - // ResourceTypeKnativeImporter is the Stackdriver resource type for Knative Importers. - ResourceTypeKnativeImporter = "knative_importer" + // ResourceTypeKnativeSource is the Stackdriver resource type for Knative Sources. + ResourceTypeKnativeSource = "knative_source" // LabelTriggerName is the label for the name of the Trigger. LabelTriggerName = "trigger_name" @@ -48,11 +48,11 @@ const ( // LabelFilterSource is the label for the Trigger filter attribute "source". LabelFilterSource = "filter_source" - // LabelImporterName is the label for the name of the Importer. - LabelImporterName = "importer_name" + // LabelSourceName is the label for the name of the Source. + LabelSourceName = "source_name" - // LabelImporterResourceGroup is the name of the Importer CRD. - LabelImporterResourceGroup = "importer_resource_group" + // LabelSourceResourceGroup is the name of the Source CRD. + LabelSourceResourceGroup = "source_resource_group" ) var ( @@ -89,19 +89,19 @@ var ( "knative.dev/eventing/broker/event_count", ) - // KnativeImporterLabels stores the set of resource labels for resource type knative_importer. - KnativeImporterLabels = sets.NewString( + // KnativeSourceLabels stores the set of resource labels for resource type knative_source. + KnativeSourceLabels = sets.NewString( LabelProject, LabelLocation, LabelClusterName, LabelNamespaceName, - LabelImporterName, - LabelImporterResourceGroup, + LabelSourceName, + LabelSourceResourceGroup, ) - // KnativeImporterMetrics stores a set of metric types which are supported - // by resource type knative_importer. - KnativeImporterMetrics = sets.NewString( - "knative.dev/eventing/importer/event_count", + // KnativeSourceMetrics stores a set of metric types which are supported + // by resource type knative_source. + KnativeSourceMetrics = sets.NewString( + "knative.dev/eventing/source/event_count", ) ) diff --git a/vendor/knative.dev/pkg/metrics/monitored_resources_eventing.go b/vendor/knative.dev/pkg/metrics/monitored_resources_eventing.go index 30208ca9598..822848ceffd 100644 --- a/vendor/knative.dev/pkg/metrics/monitored_resources_eventing.go +++ b/vendor/knative.dev/pkg/metrics/monitored_resources_eventing.go @@ -44,13 +44,13 @@ type KnativeBroker struct { BrokerName string } -type KnativeImporter struct { - Project string - Location string - ClusterName string - NamespaceName string - ImporterName string - ImporterResourceGroup string +type KnativeSource struct { + Project string + Location string + ClusterName string + NamespaceName string + SourceName string + SourceResourceGroup string } func (kt *KnativeTrigger) MonitoredResource() (resType string, labels map[string]string) { @@ -76,16 +76,16 @@ func (kb *KnativeBroker) MonitoredResource() (resType string, labels map[string] return metricskey.ResourceTypeKnativeBroker, labels } -func (ki *KnativeImporter) MonitoredResource() (resType string, labels map[string]string) { +func (ki *KnativeSource) MonitoredResource() (resType string, labels map[string]string) { labels = map[string]string{ - metricskey.LabelProject: ki.Project, - metricskey.LabelLocation: ki.Location, - metricskey.LabelClusterName: ki.ClusterName, - metricskey.LabelNamespaceName: ki.NamespaceName, - metricskey.LabelImporterName: ki.ImporterName, - metricskey.LabelImporterResourceGroup: ki.ImporterResourceGroup, + metricskey.LabelProject: ki.Project, + metricskey.LabelLocation: ki.Location, + metricskey.LabelClusterName: ki.ClusterName, + metricskey.LabelNamespaceName: ki.NamespaceName, + metricskey.LabelSourceName: ki.SourceName, + metricskey.LabelSourceResourceGroup: ki.SourceResourceGroup, } - return metricskey.ResourceTypeKnativeImporter, labels + return metricskey.ResourceTypeKnativeSource, labels } func GetKnativeBrokerMonitoredResource( @@ -137,24 +137,24 @@ func GetKnativeTriggerMonitoredResource( return newTags, kt } -func GetKnativeImporterMonitoredResource( +func GetKnativeSourceMonitoredResource( v *view.View, tags []tag.Tag, gm *gcpMetadata) ([]tag.Tag, monitoredresource.Interface) { tagsMap := getTagsMap(tags) - ki := &KnativeImporter{ + ki := &KnativeSource{ // The first three resource labels are from metadata. Project: gm.project, Location: gm.location, ClusterName: gm.cluster, // The rest resource labels are from metrics labels. - NamespaceName: valueOrUnknown(metricskey.LabelNamespaceName, tagsMap), - ImporterName: valueOrUnknown(metricskey.LabelImporterName, tagsMap), - ImporterResourceGroup: valueOrUnknown(metricskey.LabelImporterResourceGroup, tagsMap), + NamespaceName: valueOrUnknown(metricskey.LabelNamespaceName, tagsMap), + SourceName: valueOrUnknown(metricskey.LabelSourceName, tagsMap), + SourceResourceGroup: valueOrUnknown(metricskey.LabelSourceResourceGroup, tagsMap), } var newTags []tag.Tag for _, t := range tags { // Keep the metrics labels that are not resource labels - if !metricskey.KnativeImporterLabels.Has(t.Key.Name()) { + if !metricskey.KnativeSourceLabels.Has(t.Key.Name()) { newTags = append(newTags, t) } } diff --git a/vendor/knative.dev/pkg/metrics/record.go b/vendor/knative.dev/pkg/metrics/record.go index 87e67fc3564..c342fe7eae3 100644 --- a/vendor/knative.dev/pkg/metrics/record.go +++ b/vendor/knative.dev/pkg/metrics/record.go @@ -36,7 +36,7 @@ import ( // 2) The backend is not Stackdriver. // 3) The backend is Stackdriver and it is allowed to use custom metrics. // 4) The backend is Stackdriver and the metric is one of the built-in metrics: "knative_revision", "knative_broker", -// "knative_trigger", "knative_importer". +// "knative_trigger", "knative_source". func Record(ctx context.Context, ms stats.Measurement) { mc := getCurMetricsConfig() @@ -57,7 +57,7 @@ func Record(ctx context.Context, ms stats.Measurement) { isServingBuiltIn := metricskey.KnativeRevisionMetrics.Has(metricType) isEventingBuiltIn := metricskey.KnativeTriggerMetrics.Has(metricType) || metricskey.KnativeBrokerMetrics.Has(metricType) || - metricskey.KnativeImporterMetrics.Has(metricType) + metricskey.KnativeSourceMetrics.Has(metricType) if isServingBuiltIn || isEventingBuiltIn { stats.Record(ctx, ms) diff --git a/vendor/knative.dev/pkg/metrics/stackdriver_exporter.go b/vendor/knative.dev/pkg/metrics/stackdriver_exporter.go index 80f830216a9..cc00fb46a92 100644 --- a/vendor/knative.dev/pkg/metrics/stackdriver_exporter.go +++ b/vendor/knative.dev/pkg/metrics/stackdriver_exporter.go @@ -87,10 +87,10 @@ func getMonitoredResourceFunc(metricTypePrefix string, gm *gcpMetadata) func(v * return GetKnativeBrokerMonitoredResource(view, tags, gm) } else if metricskey.KnativeTriggerMetrics.Has(metricType) { return GetKnativeTriggerMonitoredResource(view, tags, gm) - } else if metricskey.KnativeImporterMetrics.Has(metricType) { - return GetKnativeImporterMonitoredResource(view, tags, gm) + } else if metricskey.KnativeSourceMetrics.Has(metricType) { + return GetKnativeSourceMonitoredResource(view, tags, gm) } - // Unsupported metric by knative_revision, knative_broker, knative_trigger, and knative_importer, use "global" resource type. + // Unsupported metric by knative_revision, knative_broker, knative_trigger, and knative_source, use "global" resource type. return getGlobalMonitoredResource(view, tags) } } @@ -105,7 +105,7 @@ func getMetricTypeFunc(metricTypePrefix, customMetricTypePrefix string) func(vie inServing := metricskey.KnativeRevisionMetrics.Has(metricType) inEventing := metricskey.KnativeBrokerMetrics.Has(metricType) || metricskey.KnativeTriggerMetrics.Has(metricType) || - metricskey.KnativeImporterMetrics.Has(metricType) + metricskey.KnativeSourceMetrics.Has(metricType) if inServing || inEventing { return metricType } diff --git a/vendor/knative.dev/pkg/metrics/source_stats_reporter.go b/vendor/knative.dev/pkg/source/source_stats_reporter.go similarity index 53% rename from vendor/knative.dev/pkg/metrics/source_stats_reporter.go rename to vendor/knative.dev/pkg/source/source_stats_reporter.go index 6848230f847..059ad66dd82 100644 --- a/vendor/knative.dev/pkg/metrics/source_stats_reporter.go +++ b/vendor/knative.dev/pkg/source/source_stats_reporter.go @@ -14,14 +14,15 @@ * limitations under the License. */ -package metrics +package source import ( "context" + "go.opencensus.io/stats/view" + "knative.dev/pkg/metrics" "strconv" "go.opencensus.io/stats" - "go.opencensus.io/stats/view" "go.opencensus.io/tag" "knative.dev/pkg/metrics/metricskey" ) @@ -33,6 +34,19 @@ var ( "Number of events sent", stats.UnitDimensionless, ) + + // Create the tag keys that will be used to add tags to our measurements. + // Tag keys must conform to the restrictions described in + // go.opencensus.io/tag/validate.go. Currently those restrictions are: + // - length between 1 and 255 inclusive + // - characters are printable US-ASCII + namespaceKey = tag.MustNewKey(metricskey.LabelNamespaceName) + eventSourceKey = tag.MustNewKey(metricskey.LabelEventSource) + eventTypeKey = tag.MustNewKey(metricskey.LabelEventType) + sourceNameKey = tag.MustNewKey(metricskey.LabelSourceName) + sourceResourceGroupKey = tag.MustNewKey(metricskey.LabelSourceResourceGroup) + responseCodeKey = tag.MustNewKey(metricskey.LabelResponseCode) + responseCodeClassKey = tag.MustNewKey(metricskey.LabelResponseCodeClass) ) type ReportArgs struct { @@ -43,6 +57,10 @@ type ReportArgs struct { ResourceGroup string } +func init() { + register() +} + // StatsReporter defines the interface for sending source metrics. type StatsReporter interface { // ReportEventCount captures the event count. It records one per call. @@ -53,75 +71,18 @@ var _ StatsReporter = (*reporter)(nil) // reporter holds cached metric objects to report source metrics. type reporter struct { - namespaceTagKey tag.Key - eventSourceTagKey tag.Key - eventTypeTagKey tag.Key - sourceNameTagKey tag.Key - sourceResourceGroupTagKey tag.Key - responseCodeKey tag.Key - responseCodeClassKey tag.Key + ctx context.Context } // NewStatsReporter creates a reporter that collects and reports source metrics. func NewStatsReporter() (StatsReporter, error) { - var r = &reporter{} - - // Create the tag keys that will be used to add tags to our measurements. - nsTag, err := tag.NewKey(metricskey.LabelNamespaceName) - if err != nil { - return nil, err - } - r.namespaceTagKey = nsTag - - eventSourceTag, err := tag.NewKey(metricskey.LabelEventSource) - if err != nil { - return nil, err - } - r.eventSourceTagKey = eventSourceTag - - eventTypeTag, err := tag.NewKey(metricskey.LabelEventType) - if err != nil { - return nil, err - } - r.eventTypeTagKey = eventTypeTag - - nameTag, err := tag.NewKey(metricskey.LabelImporterName) - if err != nil { - return nil, err - } - r.sourceNameTagKey = nameTag - - resourceGroupTag, err := tag.NewKey(metricskey.LabelImporterResourceGroup) - if err != nil { - return nil, err - } - r.sourceResourceGroupTagKey = resourceGroupTag - - responseCodeTag, err := tag.NewKey(metricskey.LabelResponseCode) - if err != nil { - return nil, err - } - r.responseCodeKey = responseCodeTag - responseCodeClassTag, err := tag.NewKey(metricskey.LabelResponseCodeClass) - if err != nil { - return nil, err - } - r.responseCodeClassKey = responseCodeClassTag - - // Create view to see our measurements. - err = view.Register( - &view.View{ - Description: eventCountM.Description(), - Measure: eventCountM, - Aggregation: view.Count(), - TagKeys: []tag.Key{r.namespaceTagKey, r.eventSourceTagKey, r.eventTypeTagKey, r.sourceNameTagKey, r.sourceResourceGroupTagKey, r.responseCodeKey, r.responseCodeClassKey}, - }, + ctx, err := tag.New( + context.Background(), ) if err != nil { return nil, err } - - return r, nil + return &reporter{ctx: ctx}, nil } func (r *reporter) ReportEventCount(args *ReportArgs, responseCode int) error { @@ -129,18 +90,41 @@ func (r *reporter) ReportEventCount(args *ReportArgs, responseCode int) error { if err != nil { return err } - Record(ctx, eventCountM.M(1)) + metrics.Record(ctx, eventCountM.M(1)) return nil } func (r *reporter) generateTag(args *ReportArgs, responseCode int) (context.Context, error) { return tag.New( - context.Background(), - tag.Insert(r.namespaceTagKey, args.Namespace), - tag.Insert(r.eventSourceTagKey, args.EventSource), - tag.Insert(r.eventTypeTagKey, args.EventType), - tag.Insert(r.sourceNameTagKey, args.Name), - tag.Insert(r.sourceResourceGroupTagKey, args.ResourceGroup), - tag.Insert(r.responseCodeKey, strconv.Itoa(responseCode)), - tag.Insert(r.responseCodeClassKey, ResponseCodeClass(responseCode))) + r.ctx, + tag.Insert(namespaceKey, args.Namespace), + tag.Insert(eventSourceKey, args.EventSource), + tag.Insert(eventTypeKey, args.EventType), + tag.Insert(sourceNameKey, args.Name), + tag.Insert(sourceResourceGroupKey, args.ResourceGroup), + tag.Insert(responseCodeKey, strconv.Itoa(responseCode)), + tag.Insert(responseCodeClassKey, metrics.ResponseCodeClass(responseCode))) +} + +func register() { + tagKeys := []tag.Key{ + namespaceKey, + eventSourceKey, + eventTypeKey, + sourceNameKey, + sourceResourceGroupKey, + responseCodeKey, + responseCodeClassKey} + + // Create view to see our measurements. + if err := view.Register( + &view.View{ + Description: eventCountM.Description(), + Measure: eventCountM, + Aggregation: view.Count(), + TagKeys: tagKeys, + }, + ); err != nil { + panic(err) + } } diff --git a/vendor/knative.dev/pkg/test/webhook-apicoverage/coveragecalculator/calculator.go b/vendor/knative.dev/pkg/test/webhook-apicoverage/coveragecalculator/calculator.go index 2dbfb23361e..5c5262471bc 100644 --- a/vendor/knative.dev/pkg/test/webhook-apicoverage/coveragecalculator/calculator.go +++ b/vendor/knative.dev/pkg/test/webhook-apicoverage/coveragecalculator/calculator.go @@ -16,6 +16,10 @@ limitations under the License. package coveragecalculator +import ( + "math" +) + // CoverageValues encapsulates all the coverage related values. type CoverageValues struct { TotalFields int @@ -25,12 +29,37 @@ type CoverageValues struct { PercentCoverage float64 } +// CoveragePercentages encapsulate percentage coverage for resources. +type CoveragePercentages struct { + + // ResourceCoverages maps percentage coverage per resource. + ResourceCoverages map[string]float64 +} + +// CalculatePercentageValue calculates percentage value based on other fields. func (c *CoverageValues) CalculatePercentageValue() { if c.TotalFields > 0 { c.PercentCoverage = (float64(c.CoveredFields) / float64(c.TotalFields-c.IgnoredFields)) * 100 } } +// GetAndRemoveResourceValue utility method to implement "get and delete" +// semantics. This makes templating operations easy. +func (c *CoveragePercentages) GetAndRemoveResourceValue(resource string) float64 { + if resourcePercentage, ok := c.ResourceCoverages[resource]; ok { + delete(c.ResourceCoverages, resource) + return resourcePercentage + } + + return 0.0 +} + +// IsFailedBuild utility method to indicate if CoveragePercentages indicate +// values of a failed build. +func (c *CoveragePercentages) IsFailedBuild() bool { + return math.Abs(c.ResourceCoverages["Overall"]-0) == 0 +} + // CalculateTypeCoverage calculates aggregate coverage values based on provided []TypeCoverage func CalculateTypeCoverage(typeCoverage []TypeCoverage) *CoverageValues { cv := CoverageValues{} diff --git a/vendor/knative.dev/pkg/test/webhook-apicoverage/tools/tools.go b/vendor/knative.dev/pkg/test/webhook-apicoverage/tools/tools.go index b8207b348ca..b97014e6d20 100644 --- a/vendor/knative.dev/pkg/test/webhook-apicoverage/tools/tools.go +++ b/vendor/knative.dev/pkg/test/webhook-apicoverage/tools/tools.go @@ -25,6 +25,7 @@ import ( "os/user" "path" + "github.com/pkg/errors" "knative.dev/pkg/test/webhook-apicoverage/coveragecalculator" "knative.dev/pkg/test/webhook-apicoverage/view" "knative.dev/pkg/test/webhook-apicoverage/webhook" @@ -47,6 +48,10 @@ const ( // WebhookTotalCoverageEndPoint constant for total coverage API endpoint. WebhookTotalCoverageEndPoint = "https://%s:443" + webhook.TotalCoverageEndPoint + + // WebhookResourcePercentageCoverageEndPoint constant for + // ResourcePercentageCoverage API endpoint. + WebhookResourcePercentageCoverageEndPoint = "https://%s:443" + webhook.ResourcePercentageCoverageEndPoint ) // GetDefaultKubePath helper method to fetch kubeconfig path. @@ -107,14 +112,15 @@ func GetResourceCoverage(webhookIP string, resourceName string) (string, error) } resp, err := client.Get(fmt.Sprintf(WebhookResourceCoverageEndPoint, webhookIP, resourceName)) if err != nil { - return "", fmt.Errorf("encountered error making resource coverage request: %v", err) - } else if resp.StatusCode != http.StatusOK { + return "", errors.Wrap(err, "encountered error making resource coverage request") + } + if resp.StatusCode != http.StatusOK { return "", fmt.Errorf("invalid HTTP Status received for resource coverage request. Status: %d", resp.StatusCode) } var body []byte if body, err = ioutil.ReadAll(resp.Body); err != nil { - return "", fmt.Errorf("error reading resource coverage response: %v", err) + return "", errors.Wrap(err, "Failed reading resource coverage response") } return string(body), nil @@ -131,11 +137,7 @@ func GetAndWriteResourceCoverage(webhookIP string, resourceName string, outputFi return err } - if err = ioutil.WriteFile(outputFile, []byte(resourceCoverage), 0400); err != nil { - return fmt.Errorf("error writing resource coverage to output file: %s, error: %v coverage: %s", outputFile, err, resourceCoverage) - } - - return nil + return ioutil.WriteFile(outputFile, []byte(resourceCoverage), 0400) } // GetTotalCoverage calls the total coverage API to retrieve total coverage values. @@ -147,8 +149,9 @@ func GetTotalCoverage(webhookIP string) (*coveragecalculator.CoverageValues, err resp, err := client.Get(fmt.Sprintf(WebhookTotalCoverageEndPoint, webhookIP)) if err != nil { - return nil, fmt.Errorf("encountered error making total coverage request: %v", err) - } else if resp.StatusCode != http.StatusOK { + return nil, errors.Wrap(err, "encountered error making total coverage request") + } + if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("invalid HTTP Status received for total coverage request. Status: %d", resp.StatusCode) } @@ -159,7 +162,7 @@ func GetTotalCoverage(webhookIP string) (*coveragecalculator.CoverageValues, err var coverage coveragecalculator.CoverageValues if err = json.Unmarshal(body, &coverage); err != nil { - return nil, fmt.Errorf("error unmarshalling response to CoverageValues instance: %v", err) + return nil, errors.Wrap(err, "Failed unmarshalling response to CoverageValues instance") } return &coverage, nil @@ -176,13 +179,53 @@ func GetAndWriteTotalCoverage(webhookIP string, outputFile string) error { return err } - if htmlData, err := view.GetHTMLCoverageValuesDisplay(totalCoverage); err != nil { - return fmt.Errorf("error building html file from total coverage. error: %v", err) - } else { - if err = ioutil.WriteFile(outputFile, []byte(htmlData), 0400); err != nil { - return fmt.Errorf("error writing total coverage to output file: %s, error: %v", outputFile, err) - } + htmlData, err := view.GetHTMLCoverageValuesDisplay(totalCoverage) + if err != nil { + return errors.Wrap(err, "Failed building html file from total coverage. error") + } + + return ioutil.WriteFile(outputFile, []byte(htmlData), 0400) +} + +// GetResourcePercentages calls resource percentage coverage API to retrieve +// percentage values. +func GetResourcePercentages(webhookIP string) ( + *coveragecalculator.CoveragePercentages, error) { + client := &http.Client{Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + } + + resp, err := client.Get(fmt.Sprintf(WebhookResourcePercentageCoverageEndPoint, + webhookIP)) + if err != nil { + return nil, errors.Wrap(err, "encountered error making resource percentage coverage request") + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("Invalid HTTP Status received for resource"+ + " percentage coverage request. Status: %d", resp.StatusCode) + } + + var body []byte + if body, err = ioutil.ReadAll(resp.Body); err != nil { + return nil, errors.Wrap(err, "Failed reading resource percentage coverage response") + } + + coveragePercentages := &coveragecalculator.CoveragePercentages{} + if err = json.Unmarshal(body, coveragePercentages); err != nil { + return nil, errors.Wrap(err, "Failed unmarshalling response to CoveragePercentages instance") + } + + return coveragePercentages, nil +} + +// WriteResourcePercentages writes CoveragePercentages to junit_xml output file. +func WriteResourcePercentages(outputFile string, + coveragePercentages *coveragecalculator.CoveragePercentages) error { + htmlData, err := view.GetCoveragePercentageXMLDisplay(coveragePercentages) + if err != nil { + errors.Wrap(err, "Failed building coverage percentage xml file") } - return nil + return ioutil.WriteFile(outputFile, []byte(htmlData), 0400) } diff --git a/vendor/knative.dev/pkg/test/webhook-apicoverage/view/README.md b/vendor/knative.dev/pkg/test/webhook-apicoverage/view/README.md index db7cb941f36..fd22df58895 100644 --- a/vendor/knative.dev/pkg/test/webhook-apicoverage/view/README.md +++ b/vendor/knative.dev/pkg/test/webhook-apicoverage/view/README.md @@ -37,3 +37,8 @@ Covered Fields: Ignored Fields: Coverage Percentage: ``` + +`GetCoveragePercentageXMLDisplay()` is a utility method that can be used by +repos to produce coverage percentage for each resource in a Junit XML results +file. The method takes [CoveragePercentages](../coveragecalculator/calculator.go) +as input and produces a Junit result file format. diff --git a/vendor/knative.dev/pkg/test/webhook-apicoverage/view/aggregate_coverage.html b/vendor/knative.dev/pkg/test/webhook-apicoverage/view/aggregate_coverage.html deleted file mode 100644 index 98b64764cc1..00000000000 --- a/vendor/knative.dev/pkg/test/webhook-apicoverage/view/aggregate_coverage.html +++ /dev/null @@ -1,25 +0,0 @@ - - - - - - - - - -
Total Fields{{ .TotalFields }}
Covered Fields{{ .CoveredFields }}
Ignored Fields{{ .IgnoredFields }}
Coverage Percentage{{ .PercentCoverage }}
- - diff --git a/vendor/knative.dev/pkg/test/webhook-apicoverage/view/html_display.go b/vendor/knative.dev/pkg/test/webhook-apicoverage/view/html_display.go index 36d60979aa5..1c93c9bd067 100644 --- a/vendor/knative.dev/pkg/test/webhook-apicoverage/view/html_display.go +++ b/vendor/knative.dev/pkg/test/webhook-apicoverage/view/html_display.go @@ -28,15 +28,16 @@ type HtmlDisplayData struct { CoverageNumbers *coveragecalculator.CoverageValues } -// GetHTMLDisplay is a helper method to display API Coverage details in json-like format inside a HTML page. -func GetHTMLDisplay(coverageData []coveragecalculator.TypeCoverage, coverageValues *coveragecalculator.CoverageValues) (string, error) { - +// GetHTMLDisplay is a helper method to display API Coverage details in +// json-like format inside a HTML page. +func GetHTMLDisplay(coverageData []coveragecalculator.TypeCoverage, + coverageValues *coveragecalculator.CoverageValues) (string, error) { htmlData := HtmlDisplayData{ TypeCoverages: coverageData, CoverageNumbers: coverageValues, } - tmpl, err := template.ParseFiles("type_coverage.html") + tmpl, err := template.New("TypeCoverage").Parse(TypeCoverageTempl) if err != nil { return "", err } @@ -52,8 +53,7 @@ func GetHTMLDisplay(coverageData []coveragecalculator.TypeCoverage, coverageValu // GetHTMLCoverageValuesDisplay is a helper method to display coverage values inside a HTML table. func GetHTMLCoverageValuesDisplay(coverageValues *coveragecalculator.CoverageValues) (string, error) { - - tmpl, err := template.ParseFiles("aggregate_coverage.html") + tmpl, err := template.New("AggregateCoverage").Parse(AggregateCoverageTmpl) if err != nil { return "", err } diff --git a/vendor/knative.dev/pkg/test/webhook-apicoverage/view/type_coverage.html b/vendor/knative.dev/pkg/test/webhook-apicoverage/view/html_template.go similarity index 55% rename from vendor/knative.dev/pkg/test/webhook-apicoverage/view/type_coverage.html rename to vendor/knative.dev/pkg/test/webhook-apicoverage/view/html_template.go index a55bbe36d9f..89084c8fe1e 100644 --- a/vendor/knative.dev/pkg/test/webhook-apicoverage/view/type_coverage.html +++ b/vendor/knative.dev/pkg/test/webhook-apicoverage/view/html_template.go @@ -1,4 +1,26 @@ - +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package view + +import ( + "fmt" +) + +var TypeCoverageTempl = fmt.Sprint(` + + + + + + +
Total Fields{{ .TotalFields }}
Covered Fields{{ .CoveredFields }}
Ignored Fields{{ .IgnoredFields }}
Coverage Percentage{{ .PercentCoverage }}
+ + +`) diff --git a/vendor/knative.dev/pkg/test/webhook-apicoverage/view/xml_display.go b/vendor/knative.dev/pkg/test/webhook-apicoverage/view/xml_display.go new file mode 100644 index 00000000000..146312a9ac2 --- /dev/null +++ b/vendor/knative.dev/pkg/test/webhook-apicoverage/view/xml_display.go @@ -0,0 +1,42 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package view + +import ( + "strings" + "text/template" + + "knative.dev/pkg/test/webhook-apicoverage/coveragecalculator" +) + +// GetCoveragePercentageXMLDisplay is a helper method to write resource coverage +// percentage values to junit xml file format. +func GetCoveragePercentageXMLDisplay( + percentageCoverages *coveragecalculator.CoveragePercentages) (string, error) { + tmpl, err := template.New("JunitResult").Parse(JunitResultTmpl) + if err != nil { + return "", err + } + + var buffer strings.Builder + err = tmpl.Execute(&buffer, percentageCoverages) + if err != nil { + return "", err + } + + return buffer.String(), nil +} diff --git a/vendor/knative.dev/pkg/test/webhook-apicoverage/view/xml_template.go b/vendor/knative.dev/pkg/test/webhook-apicoverage/view/xml_template.go new file mode 100644 index 00000000000..019945e713f --- /dev/null +++ b/vendor/knative.dev/pkg/test/webhook-apicoverage/view/xml_template.go @@ -0,0 +1,41 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package view + +import ( + "fmt" +) + +var JunitResultTmpl = fmt.Sprint(` + + + {{ if .IsFailedBuild }} + true + {{ end }} + + + + + {{ range $key, $value := .ResourceCoverages }} + + + + + + {{end}} + +`) diff --git a/vendor/knative.dev/pkg/test/webhook-apicoverage/webhook/apicoverage_recorder.go b/vendor/knative.dev/pkg/test/webhook-apicoverage/webhook/apicoverage_recorder.go index 5e9b63636cc..0a0fade15af 100644 --- a/vendor/knative.dev/pkg/test/webhook-apicoverage/webhook/apicoverage_recorder.go +++ b/vendor/knative.dev/pkg/test/webhook-apicoverage/webhook/apicoverage_recorder.go @@ -50,6 +50,10 @@ const ( // TotalCoverageEndPoint is the endpoint for Total Coverage API TotalCoverageEndPoint = "/totalcoverage" + // ResourcePercentageCoverageEndPoint is the end point for Resource Percentage + // coverages API + ResourcePercentageCoverageEndPoint = "/resourcepercentagecoverage" + // resourceChannelQueueSize size of the queue maintained for resource channel. resourceChannelQueueSize = 10 ) @@ -209,3 +213,51 @@ func (a *APICoverageRecorder) GetTotalCoverage(w http.ResponseWriter, r *http.Re fmt.Fprintf(w, "error writing total coverage response: %v", err) } } + +// GetResourceCoveragePercentags goes over all the resources setup for the +// apicoverage tool and returns percentage coverage for each resource. +func (a *APICoverageRecorder) GetResourceCoveragePercentages( + w http.ResponseWriter, r *http.Request) { + var ( + ignoredFields coveragecalculator.IgnoredFields + err error + ) + + ignoredFieldsFilePath := + os.Getenv("KO_DATA_PATH") + "/ignoredfields.yaml" + if err = ignoredFields.ReadFromFile(ignoredFieldsFilePath); err != nil { + a.Logger.Errorf("Error reading file %s: %v", + ignoredFieldsFilePath, err) + } + + totalCoverage := coveragecalculator.CoverageValues{} + percentCoverages := make(map[string]float64) + for resource := range a.ResourceMap { + tree := a.ResourceForest.TopLevelTrees[resource.Kind] + typeCoverage := tree.BuildCoverageData(a.NodeRules, a.FieldRules, + ignoredFields) + coverageValues := coveragecalculator.CalculateTypeCoverage(typeCoverage) + coverageValues.CalculatePercentageValue() + percentCoverages[resource.Kind] = coverageValues.PercentCoverage + totalCoverage.TotalFields += coverageValues.TotalFields + totalCoverage.CoveredFields += coverageValues.CoveredFields + totalCoverage.IgnoredFields += coverageValues.IgnoredFields + } + totalCoverage.CalculatePercentageValue() + percentCoverages["Overall"] = totalCoverage.PercentCoverage + + var body []byte + if body, err = json.Marshal( + coveragecalculator.CoveragePercentages{ + ResourceCoverages: percentCoverages, + }); err != nil { + fmt.Fprintf(w, "error marshalling percentage coverage response: %v", + err) + return + } + + if _, err = w.Write(body); err != nil { + fmt.Fprintf(w, "error writing percentage coverage response: %v", + err) + } +} diff --git a/vendor/knative.dev/pkg/tracing/opencensus.go b/vendor/knative.dev/pkg/tracing/opencensus.go index 42b21cb4a79..9a6586f474a 100644 --- a/vendor/knative.dev/pkg/tracing/opencensus.go +++ b/vendor/knative.dev/pkg/tracing/opencensus.go @@ -2,12 +2,14 @@ package tracing import ( "errors" + "fmt" "io" + "os" "sync" "contrib.go.opencensus.io/exporter/stackdriver" oczipkin "contrib.go.opencensus.io/exporter/zipkin" - zipkin "github.com/openzipkin/zipkin-go" + "github.com/openzipkin/zipkin-go" httpreporter "github.com/openzipkin/zipkin-go/reporter/http" "go.opencensus.io/trace" "go.uber.org/zap" @@ -16,7 +18,7 @@ import ( ) // ConfigOption is the interface for adding additional exporters and configuring opencensus tracing. -type ConfigOption func(*config.Config) +type ConfigOption func(*config.Config) error // OpenCensusTracer is responsible for managing and updating configuration of OpenCensus tracing type OpenCensusTracer struct { @@ -46,14 +48,16 @@ func (oct *OpenCensusTracer) ApplyConfig(cfg *config.Config) error { return err } - // Short circuit if our config hasnt changed + // Short circuit if our config hasn't changed. if oct.curCfg != nil && oct.curCfg.Equals(cfg) { return nil } // Apply config options for _, configOpt := range oct.configOptions { - configOpt(cfg) + if err = configOpt(cfg); err != nil { + return err + } } // Set config @@ -70,7 +74,9 @@ func (oct *OpenCensusTracer) Finish() error { } for _, configOpt := range oct.configOptions { - configOpt(nil) + if err = configOpt(nil); err != nil { + return err + } } globalOct = nil @@ -108,7 +114,7 @@ func createOCTConfig(cfg *config.Config) *trace.Config { // WithExporter returns a ConfigOption for use with NewOpenCensusTracer that configures // it to export traces based on the configuration read from config-tracing. func WithExporter(name string, logger *zap.SugaredLogger) ConfigOption { - return func(cfg *config.Config) { + return func(cfg *config.Config) error { var ( exporter trace.Exporter closer io.Closer @@ -120,15 +126,25 @@ func WithExporter(name string, logger *zap.SugaredLogger) ConfigOption { }) if err != nil { logger.Errorw("error reading project-id from metadata", zap.Error(err)) - return + return err } exporter = exp case config.Zipkin: + // If name isn't specified, then zipkin.NewEndpoint will return an error saying that it + // can't find the host named ''. So, if not specified, default it to this machine's + // hostname. + if name == "" { + n, err := os.Hostname() + if err != nil { + return fmt.Errorf("unable to get hostname: %v", err) + } + name = n + } hostPort := name + ":80" zipEP, err := zipkin.NewEndpoint(name, hostPort) if err != nil { logger.Errorw("error building zipkin endpoint", zap.Error(err)) - return + return err } reporter := httpreporter.NewReporter(cfg.ZipkinEndpoint) exporter = oczipkin.NewExporter(reporter, zipEP) @@ -149,5 +165,7 @@ func WithExporter(name string, logger *zap.SugaredLogger) ConfigOption { globalOct.exporter = exporter globalOct.closer = closer + + return nil } } diff --git a/vendor/knative.dev/pkg/tracing/zipkin.go b/vendor/knative.dev/pkg/tracing/zipkin.go index f5932be211e..75a97ae3bfc 100644 --- a/vendor/knative.dev/pkg/tracing/zipkin.go +++ b/vendor/knative.dev/pkg/tracing/zipkin.go @@ -32,7 +32,7 @@ type ZipkinReporterFactory func(*config.Config) (zipkinreporter.Reporter, error) // - WithExporter() in production code // - testing/FakeZipkinExporter() in test code. func WithZipkinExporter(reporterFact ZipkinReporterFactory, endpoint *zipkinmodel.Endpoint) ConfigOption { - return func(cfg *config.Config) { + return func(cfg *config.Config) error { var ( reporter zipkinreporter.Reporter exporter trace.Exporter @@ -43,8 +43,7 @@ func WithZipkinExporter(reporterFact ZipkinReporterFactory, endpoint *zipkinmode // do this before cleanup to minimize time where we have duplicate exporters reporter, err := reporterFact(cfg) if err != nil { - // TODO(greghaynes) log this error - return + return err } exporter := zipkin.NewExporter(reporter, endpoint) trace.RegisterExporter(exporter) @@ -63,5 +62,7 @@ func WithZipkinExporter(reporterFact ZipkinReporterFactory, endpoint *zipkinmode oct.closer = reporter oct.exporter = exporter + + return nil } }