From 488189f1c59c5c0ec88dfde1069a9dcebc9b5f6d Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Sun, 26 Apr 2020 21:59:09 -0600 Subject: [PATCH 01/30] Sort tc.CRConfigDeliveryService fields --- lib/go-tc/crconfig.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/go-tc/crconfig.go b/lib/go-tc/crconfig.go index 20547fb43a..fbb4f55e24 100644 --- a/lib/go-tc/crconfig.go +++ b/lib/go-tc/crconfig.go @@ -105,31 +105,31 @@ type CRConfigTrafficOpsServer struct { //TODO: drichardson - reconcile this with the DeliveryService struct in deliveryservices.go type CRConfigDeliveryService struct { AnonymousBlockingEnabled *string `json:"anonymousBlockingEnabled,omitempty"` + BypassDestination map[string]*CRConfigBypassDestination `json:"bypassDestination,omitempty"` ConsistentHashQueryParams []string `json:"consistentHashQueryParams,omitempty"` ConsistentHashRegex *string `json:"consistentHashRegex,omitempty"` CoverageZoneOnly bool `json:"coverageZoneOnly,string"` + DeepCachingType *DeepCachingType `json:"deepCachingType"` Dispersion *CRConfigDispersion `json:"dispersion,omitempty"` Domains []string `json:"domains,omitempty"` + EcsEnabled *bool `json:"ecsEnabled,string,omitempty"` + GeoEnabled []CRConfigGeoEnabled `json:"geoEnabled,omitempty"` + GeoLimitRedirectURL *string `json:"geoLimitRedirectURL,omitempty"` GeoLocationProvider *string `json:"geolocationProvider,omitempty"` + IP6RoutingEnabled *bool `json:"ip6RoutingEnabled,string,omitempty"` MatchSets []*MatchSet `json:"matchsets,omitempty"` + MaxDNSIPsForLocation *int `json:"maxDnsIpsForLocation,omitempty"` MissLocation *CRConfigLatitudeLongitudeShort `json:"missLocation,omitempty"` Protocol *CRConfigDeliveryServiceProtocol `json:"protocol,omitempty"` RegionalGeoBlocking *string `json:"regionalGeoBlocking,omitempty"` - ResponseHeaders map[string]string `json:"responseHeaders,omitempty"` RequestHeaders []string `json:"requestHeaders,omitempty"` + ResponseHeaders map[string]string `json:"responseHeaders,omitempty"` + RoutingName *string `json:"routingName,omitempty"` Soa *SOA `json:"soa,omitempty"` SSLEnabled bool `json:"sslEnabled,string"` + StaticDNSEntries []CRConfigStaticDNSEntry `json:"staticDnsEntries,omitempty"` TTL *int `json:"ttl,omitempty"` TTLs *CRConfigTTL `json:"ttls,omitempty"` - MaxDNSIPsForLocation *int `json:"maxDnsIpsForLocation,omitempty"` - IP6RoutingEnabled *bool `json:"ip6RoutingEnabled,string,omitempty"` - EcsEnabled *bool `json:"ecsEnabled,string,omitempty"` - RoutingName *string `json:"routingName,omitempty"` - BypassDestination map[string]*CRConfigBypassDestination `json:"bypassDestination,omitempty"` - DeepCachingType *DeepCachingType `json:"deepCachingType"` - GeoEnabled []CRConfigGeoEnabled `json:"geoEnabled,omitempty"` - GeoLimitRedirectURL *string `json:"geoLimitRedirectURL,omitempty"` - StaticDNSEntries []CRConfigStaticDNSEntry `json:"staticDnsEntries,omitempty"` } type CRConfigGeoEnabled struct { From d6739f8b4f979480791f98584ba9b4524822f77d Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Sun, 26 Apr 2020 22:53:36 -0600 Subject: [PATCH 02/30] Sort CRConfigDeliveryService query/scan by column name --- .../crconfig/deliveryservice.go | 88 +++++++++---------- .../crconfig/deliveryservice_test.go | 72 +++++++-------- 2 files changed, 80 insertions(+), 80 deletions(-) diff --git a/traffic_ops/traffic_ops_golang/crconfig/deliveryservice.go b/traffic_ops/traffic_ops_golang/crconfig/deliveryservice.go index 6e766a226a..379e9bf66f 100644 --- a/traffic_ops/traffic_ops_golang/crconfig/deliveryservice.go +++ b/traffic_ops/traffic_ops_golang/crconfig/deliveryservice.go @@ -92,37 +92,37 @@ func makeDSes(cdn string, domain string, tx *sql.Tx) (map[string]tc.CRConfigDeli } q := ` -SELECT d.xml_id, - d.miss_lat, - d.miss_long, - d.protocol, - d.ccr_dns_ttl AS ttl, - d.routing_name, - d.geo_provider, - t.name AS type, - d.geo_limit, - d.geo_limit_countries, - d.geolimit_redirect_url, +SELECT d.anonymous_blocking_enabled, + d.consistent_hash_regex, + d.deep_caching_type, d.initial_dispersion, - d.regional_geo_blocking, - d.tr_response_headers, - d.max_dns_answers, - p.name AS profile, + d.dns_bypass_cname, d.dns_bypass_ip, d.dns_bypass_ip6, d.dns_bypass_ttl, - d.dns_bypass_cname, + (SELECT ARRAY_AGG(name ORDER BY name) + FROM deliveryservice_consistent_hash_query_param + WHERE deliveryservice_id = d.id) AS query_keys, + d.routing_name, + d.ccr_dns_ttl AS ttl, + d.ecs_enabled, + d.regional_geo_blocking, + d.geo_limit, + d.geo_limit_countries, + d.geolimit_redirect_url, + d.geo_provider, d.http_bypass_fqdn, d.ipv6_routing_enabled, - d.ecs_enabled, - d.deep_caching_type, + d.max_dns_answers, + d.miss_lat, + d.miss_long, + p.name AS profile, + d.protocol, d.tr_request_headers, d.tr_response_headers, - d.anonymous_blocking_enabled, - d.consistent_hash_regex, - (SELECT ARRAY_AGG(name ORDER BY name) - FROM deliveryservice_consistent_hash_query_param - WHERE deliveryservice_id = d.id) AS query_keys + d.tr_response_headers, + t.name AS type, + d.xml_id FROM deliveryservice AS d INNER JOIN type AS t ON t.id = d.type LEFT OUTER JOIN profile AS p ON p.id = d.profile @@ -173,35 +173,35 @@ AND d.active = true anonymousBlocking := false consistentHashRegex := sql.NullString{} err := rows.Scan( - &xmlID, - &missLat, - &missLon, - &protocol, - &ds.TTL, - &ds.RoutingName, - &geoProvider, - &ttype, - &geoLimit, - &geoLimitCountries, - &geoLimitRedirectURL, + &anonymousBlocking, + &consistentHashRegex, + &deepCachingType, &dispersion, - &geoBlocking, - &trRespHdrsStr, - &maxDNSAnswers, - &profile, + &dnsBypassCName, &dnsBypassIP, &dnsBypassIP6, &dnsBypassTTL, - &dnsBypassCName, + pq.Array(&ds.ConsistentHashQueryParams), + &ds.RoutingName, + &ds.TTL, + &ecsEnabled, + &geoBlocking, + &geoLimit, + &geoLimitCountries, + &geoLimitRedirectURL, + &geoProvider, &httpBypassFQDN, &ip6RoutingEnabled, - &ecsEnabled, - &deepCachingType, + &maxDNSAnswers, + &missLat, + &missLon, + &profile, + &protocol, &trRequestHeaders, + &trRespHdrsStr, &trResponseHeaders, - &anonymousBlocking, - &consistentHashRegex, - pq.Array(&ds.ConsistentHashQueryParams), + &ttype, + &xmlID, ) if err != nil { return nil, errors.New("scanning deliveryservice: " + err.Error()) diff --git a/traffic_ops/traffic_ops_golang/crconfig/deliveryservice_test.go b/traffic_ops/traffic_ops_golang/crconfig/deliveryservice_test.go index d272164f84..90ecbd3424 100644 --- a/traffic_ops/traffic_ops_golang/crconfig/deliveryservice_test.go +++ b/traffic_ops/traffic_ops_golang/crconfig/deliveryservice_test.go @@ -123,68 +123,68 @@ func ExpectedMakeDSes() map[string]tc.CRConfigDeliveryService { func MockMakeDSes(mock sqlmock.Sqlmock, expected map[string]tc.CRConfigDeliveryService, cdn string) { rows := sqlmock.NewRows([]string{ - "xml_id", - "miss_lat", - "miss_long", - "protocol", - "ttl", - "routing_name", - "geo_provider", - "type", - "geo_limit", - "geo_limit_countries", - "geeo_limit_redirect_url", + "anonymous_blocking_enabled", + "consistent_hash_regex", + "deep_caching_type", "initial_dispersion", - "regional_geo_blocking", - "tr_response_headers", - "max_dns_answers", - "profile", + "dns_bypass_cname", "dns_bypass_ip", "dns_bypass_ip6", "dns_bypass_ttl", - "dns_bypass_cname", + "query_keys", + "routing_name", + "ttl", + "ecs_enabled", + "regional_geo_blocking", + "geo_limit", + "geo_limit_countries", + "geeo_limit_redirect_url", + "geo_provider", "http_bypass_fqdn", "ipv6_routing_enabled", - "ecs_enabled", - "deep_caching_type", + "max_dns_answers", + "miss_lat", + "miss_long", + "profile", + "protocol", "tr_request_headers", "tr_response_headers", - "anonymous_blocking_enabled", - "consistent_hash_regex", - "query_keys"}) + "tr_response_headers", + "type", + "xml_id"}) for dsName, ds := range expected { queryParams := "{" + strings.Join(ds.ConsistentHashQueryParams, ",") + "}" rows = rows.AddRow( - dsName, - ds.MissLocation.Lat, - ds.MissLocation.Lon, - 0, - *ds.TTL, - *ds.RoutingName, - 0, - "HTTP", - 0, - "", - "", - 42, false, "", nil, + 42, "", "", "", 0, + queryParams, + *ds.RoutingName, + *ds.TTL, + *ds.EcsEnabled, + false, + 0, + "", "", + 0, *ds.BypassDestination["HTTP"].FQDN, *ds.IP6RoutingEnabled, - *ds.EcsEnabled, nil, + ds.MissLocation.Lat, + ds.MissLocation.Lon, "", + 0, "", - false, "", - queryParams) + "", + "HTTP", + dsName) } mock.ExpectQuery("select").WithArgs(cdn).WillReturnRows(rows) } From 0b71e8fac4ebd746d6a84a51bcee51495f081c16 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Mon, 27 Apr 2020 02:24:23 -0600 Subject: [PATCH 03/30] Add contentServers capabilities to CRConfig --- docs/source/api/v3/cdns_name_snapshot.rst | 1 + docs/source/api/v3/cdns_name_snapshot_new.rst | 1 + lib/go-tc/crconfig.go | 1 + .../traffic_ops_golang/crconfig/servers.go | 21 +++++++++++++----- .../crconfig/servers_test.go | 22 +++++++++++++++---- 5 files changed, 37 insertions(+), 9 deletions(-) diff --git a/docs/source/api/v3/cdns_name_snapshot.rst b/docs/source/api/v3/cdns_name_snapshot.rst index 942f32d451..188967b84d 100644 --- a/docs/source/api/v3/cdns_name_snapshot.rst +++ b/docs/source/api/v3/cdns_name_snapshot.rst @@ -119,6 +119,7 @@ Response Structure :contentServers: An object containing keys which are the (short) hostnames of the :term:`Edge-tier cache servers` in the CDN; the values corresponding to those keys are routing information for said servers :cacheGroup: A string that is the :ref:`cache-group-name` of the :term:`Cache Group` to which the server belongs + :capabilities: An array of this Cache Group's :term:`Server Capabilities`. If the Cache Group has no Server Capabilities, this field is omitted. :deliveryServices: An object containing keys which are the names of :term:`Delivery Services` to which this :term:`cache server` is assigned; the values corresponding to those keys are arrays of :abbr:`FQDNs (Fully Qualified Domain Names)` that resolve to this :term:`cache server` .. note:: Only :term:`Edge-tier cache servers` can be assigned to a :term:`Delivery Service`, and therefore this field will only be present when ``type`` is ``"EDGE"``. diff --git a/docs/source/api/v3/cdns_name_snapshot_new.rst b/docs/source/api/v3/cdns_name_snapshot_new.rst index 0920421c44..8543fd51ff 100644 --- a/docs/source/api/v3/cdns_name_snapshot_new.rst +++ b/docs/source/api/v3/cdns_name_snapshot_new.rst @@ -118,6 +118,7 @@ Response Structure :contentServers: An object containing keys which are the (short) hostnames of the :term:`Edge-tier cache servers` in the CDN; the values corresponding to those keys are routing information for said servers :cacheGroup: A string that is the :ref:`cache-group-name` of the :term:`Cache Group` to which the server belongs + :capabilities: An array of this Cache Group's :term:`Server Capabilities`. If the Cache Group has no Server Capabilities, this field is omitted. :deliveryServices: An object containing keys which are the names of :term:`Delivery Services` to which this :term:`cache server` is assigned; the values corresponding to those keys are arrays of :abbr:`FQDNs (Fully Qualified Domain Names)` that resolve to this :term:`cache server` .. note:: Only :term:`Edge-tier cache servers` can be assigned to a :term:`Delivery Service`, and therefore this field will only be present when ``type`` is ``"EDGE"``. diff --git a/lib/go-tc/crconfig.go b/lib/go-tc/crconfig.go index fbb4f55e24..dde21f1854 100644 --- a/lib/go-tc/crconfig.go +++ b/lib/go-tc/crconfig.go @@ -86,6 +86,7 @@ type CRConfigServerStatus string type CRConfigTrafficOpsServer struct { CacheGroup *string `json:"cacheGroup,omitempty"` + Capabilities []string `json:"capabilities,omitempty"` Fqdn *string `json:"fqdn,omitempty"` HashCount *int `json:"hashCount,omitempty"` HashId *string `json:"hashId,omitempty"` diff --git a/traffic_ops/traffic_ops_golang/crconfig/servers.go b/traffic_ops/traffic_ops_golang/crconfig/servers.go index 9c7c6447b2..0b0265ac85 100644 --- a/traffic_ops/traffic_ops_golang/crconfig/servers.go +++ b/traffic_ops/traffic_ops_golang/crconfig/servers.go @@ -26,6 +26,8 @@ import ( "strconv" "strings" + "github.com/lib/pq" + "github.com/apache/trafficcontrol/lib/go-log" "github.com/apache/trafficcontrol/lib/go-tc" @@ -102,7 +104,7 @@ type ServerUnion struct { type ServerAndHost struct { Server ServerUnion - Host string + Host string } const DefaultWeightMultiplier = 1000.0 @@ -127,7 +129,10 @@ func getAllServers(cdn string, tx *sql.Tx) (map[string]ServerUnion, error) { p.name AS profile_name, cast(p.routing_disabled AS int), st.name AS status, - t.name AS type + t.name AS type, + (SELECT ARRAY_AGG(server_capability ORDER BY server_capability) + FROM server_server_capability + WHERE server = s.id) AS capabilities FROM server AS s INNER JOIN cachegroup AS cg ON cg.id = s.cachegroup INNER JOIN type AS t on t.id = s.type @@ -153,7 +158,7 @@ func getAllServers(cdn string, tx *sql.Tx) (map[string]ServerUnion, error) { var status string var id int - if err := rows.Scan(&id, &s.Host, &s.Server.CacheGroup, &s.Server.Fqdn, &hashId, &httpsPort, &port, &s.Server.Profile, &s.Server.RoutingDisabled, &status, &s.Server.ServerType); err != nil { + if err := rows.Scan(&id, &s.Host, &s.Server.CacheGroup, &s.Server.Fqdn, &hashId, &httpsPort, &port, &s.Server.Profile, &s.Server.RoutingDisabled, &status, &s.Server.ServerType, pq.Array(&s.Server.Capabilities)); err != nil { return nil, errors.New("Error scanning server: " + err.Error()) } @@ -300,7 +305,8 @@ func getServerDSes(cdn string, tx *sql.Tx, domain string) (map[tc.CacheName]map[ } q := ` -select ds.xml_id as ds, dt.name as ds_type, ds.routing_name, r.pattern as pattern +select ds.xml_id as ds, dt.name as ds_type, ds.routing_name, r.pattern as pattern, +ds.topology IS NOT NULL as has_topology from regex as r inner join type as rt on r.type = rt.id inner join deliveryservice_regex as dsr on dsr.regex = r.id @@ -326,10 +332,15 @@ order by dsr.set_number asc dsType := "" dsPattern := "" dsRoutingName := "" + var hasTopology bool inf := DSRouteInfo{} - if err := rows.Scan(&ds, &dsType, &dsRoutingName, &dsPattern); err != nil { + if err := rows.Scan(&ds, &dsType, &dsRoutingName, &dsPattern, &hasTopology); err != nil { return nil, errors.New("Error scanning server deliveryservices: " + err.Error()) } + // Topology-based delivery services do not use the contentServers.deliveryServices field + if hasTopology { + continue + } inf.IsDNS = strings.HasPrefix(dsType, "DNS") inf.IsRaw = !strings.Contains(dsPattern, `.*`) if !inf.IsRaw { diff --git a/traffic_ops/traffic_ops_golang/crconfig/servers_test.go b/traffic_ops/traffic_ops_golang/crconfig/servers_test.go index a83b4053ac..8579c2bf99 100644 --- a/traffic_ops/traffic_ops_golang/crconfig/servers_test.go +++ b/traffic_ops/traffic_ops_golang/crconfig/servers_test.go @@ -22,9 +22,11 @@ package crconfig import ( "context" "fmt" + "github.com/lib/pq" "math/rand" "net" "reflect" + "strings" "testing" "time" @@ -34,6 +36,8 @@ import ( "gopkg.in/DATA-DOG/go-sqlmock.v1" ) +type StringArray pq.StringArray + func randBool() *bool { b := rand.Int()%2 == 0 return &b @@ -47,6 +51,14 @@ func randStr() *string { } return &s } +func randStrArray() []string { + num := 100 + sArray := make([]string, num) + for i := 0; i < num; i++ { + sArray[i] = *randStr() + } + return sArray +} func randInt() *int { i := rand.Int() return &i @@ -109,6 +121,7 @@ func randServer(ipService bool, ip6Service bool) tc.CRConfigTrafficOpsServer { return tc.CRConfigTrafficOpsServer{ CacheGroup: cachegroup, + Capabilities: randStrArray(), Fqdn: randStr(), HashCount: randInt(), HashId: randStr(), @@ -217,12 +230,13 @@ func ExpectedGetAllServers(params map[string]ServerParams, ipIsService bool, ip6 } func MockGetAllServers(mock sqlmock.Sqlmock, expected map[string]ServerUnion, cdn string, ipIsService bool, ip6IsService bool) { - serverRows := sqlmock.NewRows([]string{"id", "host_name", "cachegroup", "fqdn", "hashid", "https_port", "tcp_port", "profile_name", "routing_disabled", "status", "type"}) + serverRows := sqlmock.NewRows([]string{"id", "host_name", "cachegroup", "fqdn", "hashid", "https_port", "tcp_port", "profile_name", "routing_disabled", "status", "type", "capabilities"}) interfaceRows := sqlmock.NewRows([]string{"max_bandwidth", "monitor", "mtu", "name", "server"}) ipRows := sqlmock.NewRows([]string{"address", "gateway", "service_address", "interface", "server"}) i := 1 for name, s := range expected { - serverRows = serverRows.AddRow(i, name, *s.CacheGroup, *s.Fqdn, *s.HashId, *s.HttpsPort, *s.Port, *s.Profile, s.RoutingDisabled, *s.ServerStatus, *s.ServerType) + capabilities := "{" + strings.Join(s.Capabilities, ",") + "}" + serverRows = serverRows.AddRow(i, name, *s.CacheGroup, *s.Fqdn, *s.HashId, *s.HttpsPort, *s.Port, *s.Profile, s.RoutingDisabled, *s.ServerStatus, *s.ServerType, capabilities) if s.InterfaceName == nil { i++ continue @@ -538,7 +552,7 @@ func ExpectedGetServerDSes(expectedGetServerDSNames map[tc.CacheName][]tc.Delive } func MockGetServerDSes(mock sqlmock.Sqlmock, expected map[tc.CacheName]map[string][]string, cdn string) { - rows := sqlmock.NewRows([]string{"ds", "ds_type", "routing_name", "pattern"}) + rows := sqlmock.NewRows([]string{"ds", "ds_type", "routing_name", "pattern", "hasTopology"}) dsmap := map[string][]string{} for _, dses := range expected { for ds, patterns := range dses { @@ -548,7 +562,7 @@ func MockGetServerDSes(mock sqlmock.Sqlmock, expected map[tc.CacheName]map[strin for ds, patterns := range dsmap { for _, pattern := range patterns { - rows = rows.AddRow(ds, "DNS", "", pattern) + rows = rows.AddRow(ds, "DNS", "", pattern, false) } } mock.ExpectQuery("select").WithArgs(cdn).WillReturnRows(rows) From a5b76b5a6dc3651b88d1eee62d70342e9146ab11 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Mon, 27 Apr 2020 02:30:05 -0600 Subject: [PATCH 04/30] Add requiredCapabilities and topology fields to deliveryServices section in CRConfig --- docs/source/api/v3/cdns_name_snapshot.rst | 3 ++- docs/source/api/v3/cdns_name_snapshot_new.rst | 2 ++ lib/go-tc/crconfig.go | 2 ++ .../traffic_ops_golang/crconfig/deliveryservice.go | 6 ++++++ .../crconfig/deliveryservice_test.go | 12 +++++++++--- 5 files changed, 21 insertions(+), 4 deletions(-) diff --git a/docs/source/api/v3/cdns_name_snapshot.rst b/docs/source/api/v3/cdns_name_snapshot.rst index 188967b84d..21b573bb8d 100644 --- a/docs/source/api/v3/cdns_name_snapshot.rst +++ b/docs/source/api/v3/cdns_name_snapshot.rst @@ -247,7 +247,7 @@ Response Structure Regional Geographic Blocking is used by this :term:`Delivery Service` .. seealso:: :ref:`regionalgeo-qht` - + :requiredCapabilities: An array of this Delivery Service's :term:`required capabilities `. If there are no required capabilities, this field is omitted. :routingName: A string that is this :ref:`Delivery Service's Routing Name ` :soa: An object defining the :abbr:`SOA (Start of Authority)` record for the :term:`Delivery Service`'s :abbr:`TLDs (Top-Level Domains)` (defined in ``domains``) @@ -273,6 +273,7 @@ Response Structure .. seealso:: :ref:`ds-protocol` + :topology: The name of the :term:`Topology` that this :term:`Delivery Service` is assigned to. If the Delivery Service is not assigned to a topology, this field is omitted. :ttls: An object that contains keys which are types of DNS records that have values which are strings containing integers that specify the time for which a response to the specific type of record request should remain valid .. note:: This overrides ``config.ttls``. diff --git a/docs/source/api/v3/cdns_name_snapshot_new.rst b/docs/source/api/v3/cdns_name_snapshot_new.rst index 8543fd51ff..ff81715229 100644 --- a/docs/source/api/v3/cdns_name_snapshot_new.rst +++ b/docs/source/api/v3/cdns_name_snapshot_new.rst @@ -247,6 +247,7 @@ Response Structure .. seealso:: :ref:`regionalgeo-qht` + :requiredCapabilities: An array of this Delivery Service's :term:`required capabilities `. If there are no required capabilities, this field is omitted. :routingName: A string that is this :ref:`Delivery Service's Routing Name ` :soa: An object defining the :abbr:`SOA (Start of Authority)` record for the :term:`Delivery Service`'s :abbr:`TLDs (Top-Level Domains)` (defined in ``domains``) @@ -272,6 +273,7 @@ Response Structure .. seealso:: :ref:`ds-protocol` + :topology: The name of the :term:`Topology` that this :term:`Delivery Service` is assigned to. If the Delivery Service is not assigned to a topology, this field is omitted. :ttls: An object that contains keys which are types of DNS records that have values which are strings containing integers that specify the time for which a response to the specific type of record request should remain valid .. note:: This overrides ``config.ttls``. diff --git a/lib/go-tc/crconfig.go b/lib/go-tc/crconfig.go index dde21f1854..0d08542f41 100644 --- a/lib/go-tc/crconfig.go +++ b/lib/go-tc/crconfig.go @@ -124,11 +124,13 @@ type CRConfigDeliveryService struct { Protocol *CRConfigDeliveryServiceProtocol `json:"protocol,omitempty"` RegionalGeoBlocking *string `json:"regionalGeoBlocking,omitempty"` RequestHeaders []string `json:"requestHeaders,omitempty"` + RequiredCapabilities []string `json:"requiredCapabilities,omitempty"` ResponseHeaders map[string]string `json:"responseHeaders,omitempty"` RoutingName *string `json:"routingName,omitempty"` Soa *SOA `json:"soa,omitempty"` SSLEnabled bool `json:"sslEnabled,string"` StaticDNSEntries []CRConfigStaticDNSEntry `json:"staticDnsEntries,omitempty"` + Topology *string `json:"topology,omitempty"` TTL *int `json:"ttl,omitempty"` TTLs *CRConfigTTL `json:"ttls,omitempty"` } diff --git a/traffic_ops/traffic_ops_golang/crconfig/deliveryservice.go b/traffic_ops/traffic_ops_golang/crconfig/deliveryservice.go index 379e9bf66f..96fd2771ea 100644 --- a/traffic_ops/traffic_ops_golang/crconfig/deliveryservice.go +++ b/traffic_ops/traffic_ops_golang/crconfig/deliveryservice.go @@ -118,6 +118,10 @@ SELECT d.anonymous_blocking_enabled, d.miss_long, p.name AS profile, d.protocol, + (SELECT ARRAY_AGG(required_capability ORDER BY required_capability) + FROM deliveryservices_required_capability + WHERE deliveryservice_id = d.id) AS required_capabilities, + d.topology, d.tr_request_headers, d.tr_response_headers, d.tr_response_headers, @@ -197,6 +201,8 @@ AND d.active = true &missLon, &profile, &protocol, + pq.Array(&ds.RequiredCapabilities), + &ds.Topology, &trRequestHeaders, &trRespHdrsStr, &trResponseHeaders, diff --git a/traffic_ops/traffic_ops_golang/crconfig/deliveryservice_test.go b/traffic_ops/traffic_ops_golang/crconfig/deliveryservice_test.go index 90ecbd3424..0940044d13 100644 --- a/traffic_ops/traffic_ops_golang/crconfig/deliveryservice_test.go +++ b/traffic_ops/traffic_ops_golang/crconfig/deliveryservice_test.go @@ -68,9 +68,10 @@ func randDS() tc.CRConfigDeliveryService { AcceptHTTPS: false, RedirectOnHTTPS: false, }, - RegionalGeoBlocking: &falseStrPtr, - ResponseHeaders: nil, - RequestHeaders: nil, + RegionalGeoBlocking: &falseStrPtr, + ResponseHeaders: nil, + RequestHeaders: nil, + RequiredCapabilities: randStrArray(), Soa: &tc.SOA{ Admin: &ttlAdmin, ExpireSeconds: &ttlExpire, @@ -80,6 +81,7 @@ func randDS() tc.CRConfigDeliveryService { }, SSLEnabled: false, EcsEnabled: &ecsEnabled, + Topology: randStr(), TTL: ttl, TTLs: &tc.CRConfigTTL{ ASeconds: &ttlStr, @@ -147,6 +149,8 @@ func MockMakeDSes(mock sqlmock.Sqlmock, expected map[string]tc.CRConfigDeliveryS "miss_long", "profile", "protocol", + "required_capabilities", + "topology", "tr_request_headers", "tr_response_headers", "tr_response_headers", @@ -180,6 +184,8 @@ func MockMakeDSes(mock sqlmock.Sqlmock, expected map[string]tc.CRConfigDeliveryS ds.MissLocation.Lon, "", 0, + "{"+strings.Join(ds.RequiredCapabilities, ",")+"}", + ds.Topology, "", "", "", From 760d6e13ee02d649c047943f1e69d10c4f98945b Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Mon, 27 Apr 2020 02:31:29 -0600 Subject: [PATCH 05/30] Add topologies section to CRConfig --- docs/source/api/v3/cdns_name_snapshot.rst | 349 ++++++++------- docs/source/api/v3/cdns_name_snapshot_new.rst | 398 ++++++++++-------- docs/source/glossary.rst | 2 + lib/go-tc/crconfig.go | 5 + .../traffic_ops_golang/crconfig/crconfig.go | 3 + .../traffic_ops_golang/crconfig/topologies.go | 66 +++ .../crconfig/topologies_test.go | 83 ++++ 7 files changed, 559 insertions(+), 347 deletions(-) create mode 100644 traffic_ops/traffic_ops_golang/crconfig/topologies.go create mode 100644 traffic_ops/traffic_ops_golang/crconfig/topologies_test.go diff --git a/docs/source/api/v3/cdns_name_snapshot.rst b/docs/source/api/v3/cdns_name_snapshot.rst index 21b573bb8d..e85edf88c2 100644 --- a/docs/source/api/v3/cdns_name_snapshot.rst +++ b/docs/source/api/v3/cdns_name_snapshot.rst @@ -42,9 +42,10 @@ Request Structure :caption: Request Example GET /api/3.0/cdns/CDN-in-a-Box/snapshot HTTP/1.1 - Host: trafficops.infra.ciab.test - User-Agent: curl/7.47.0 + User-Agent: python-requests/2.23.0 + Accept-Encoding: gzip, deflate Accept: */* + Connection: keep-alive Cookie: mojolicious=... Response Structure @@ -320,6 +321,10 @@ Response Structure :tm_user: The username of the currently logged-in user :tm_version: The full version number of the Traffic Ops server, including release number, git commit hash, and supported Enterprise Linux version +:topologies: An array of :term:`Topologies` where each key is the name of that Topology. + + :nodes: An array of the names of the :term:`Edge-Tier` :term:`Cache Groups` in this :term:`Topology`. :term:`Mid-Tier` Cache Groups in the topology are not included. + :trafficRouterLocations: An object containing keys which are the :ref:`names of Cache Groups ` within the CDN which contain Traffic Routers :backupLocations: An object that describes this :ref:`Cache Group's Fallbacks ` @@ -343,15 +348,19 @@ Response Structure Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept, Set-Cookie, Cookie Access-Control-Allow-Methods: POST,GET,OPTIONS,PUT,DELETE Access-Control-Allow-Origin: * + Content-Encoding: gzip Content-Type: application/json - Set-Cookie: mojolicious=...; Path=/; Expires=Mon, 18 Nov 2019 17:40:54 GMT; Max-Age=3600; HttpOnly - Whole-Content-Sha512: 220bc4XXwaj+s7ODd3QAF5leGj06lnApiN5E8H/B2RgxSphnQIfnwy6WWbBDjonWXPV1IWDCjBMO+rR+lAabMg== + Set-Cookie: mojolicious=...; Path=/; Expires=Wed, 27 May 2020 18:33:17 GMT; Max-Age=3600; HttpOnly + Vary: Accept-Encoding + Whole-Content-Sha512: B5qdN9URIfu11gQxPZ8YaaMvy2HMrzsnrpt6vF037yv6OQiKCRyrUMX6wYs7QW4YVaeUrvmS2ya5l2YC0kvNAg== X-Server-Name: traffic_ops_golang/ - Date: Wed, 12 Dec 2018 17:36:25 GMT - Transfer-Encoding: chunked + Date: Wed, 27 May 2020 17:33:17 GMT + Content-Length: 1360 - { "response": { - "config": { + + { + "response": { + "config": { "api.cache-control.max-age": "10", "certificates.polling.interval": "300000", "consistent.dns.routing": "true", @@ -361,184 +370,204 @@ Response Structure "dnssec.enabled": "false", "domain_name": "mycdn.ciab.test", "federationmapping.polling.interval": "60000", - "federationmapping.polling.url": "https://${toHostname}/api/3.0/federations", + "federationmapping.polling.url": "https://${toHostname}/api/2.0/federations/all", "geolocation.polling.interval": "86400000", "geolocation.polling.url": "https://trafficops.infra.ciab.test:443/GeoLite2-City.mmdb.gz", "keystore.maintenance.interval": "300", "neustar.polling.interval": "86400000", "neustar.polling.url": "https://trafficops.infra.ciab.test:443/neustar.tar.gz", "soa": { - "admin": "twelve_monkeys", - "expire": "604800", - "minimum": "30", - "refresh": "28800", - "retry": "7200" + "admin": "twelve_monkeys", + "expire": "604800", + "minimum": "30", + "refresh": "28800", + "retry": "7200" }, "steeringmapping.polling.interval": "60000", "ttls": { - "A": "3600", - "AAAA": "3600", - "DNSKEY": "30", - "DS": "30", - "NS": "3600", - "SOA": "86400" + "A": "3600", + "AAAA": "3600", + "DNSKEY": "30", + "DS": "30", + "NS": "3600", + "SOA": "86400" }, "zonemanager.cache.maintenance.interval": "300", "zonemanager.threadpool.scale": "0.50" - }, - "contentServers": { + }, + "contentRouters": { + "trafficrouter": { + "api.port": "3333", + "fqdn": "trafficrouter.infra.ciab.test", + "httpsPort": 443, + "ip": "172.26.0.15", + "ip6": "", + "location": "CDN_in_a_Box_Edge", + "port": 80, + "profile": "CCR_CIAB", + "secure.api.port": "3443", + "status": "ONLINE" + } + }, + "contentServers": { "edge": { - "cacheGroup": "CDN_in_a_Box_Edge", - "fqdn": "edge.infra.ciab.test", - "hashCount": 999, - "hashId": "edge", - "httpsPort": 443, - "interfaceName": "eth0", - "ip": "172.16.239.100", - "ip6": "fc01:9400:1000:8::100", - "locationId": "CDN_in_a_Box_Edge", - "port": 80, - "profile": "ATS_EDGE_TIER_CACHE", - "status": "REPORTED", - "type": "EDGE", - "deliveryServices": { - "demo1": [ - "edge.demo1.mycdn.ciab.test" - ] - }, - "routingDisabled": 0 + "cacheGroup": "CDN_in_a_Box_Edge", + "capabilities": [ + "heat-vision" + ], + "fqdn": "edge.infra.ciab.test", + "hashCount": 999, + "hashId": "edge", + "httpsPort": 443, + "interfaceName": "eth0", + "ip": "172.26.0.3", + "ip6": "", + "locationId": "CDN_in_a_Box_Edge", + "port": 80, + "profile": "ATS_EDGE_TIER_CACHE", + "routingDisabled": 0, + "status": "REPORTED", + "type": "EDGE" }, "mid": { - "cacheGroup": "CDN_in_a_Box_Mid", - "fqdn": "mid.infra.ciab.test", - "hashCount": 999, - "hashId": "mid", - "httpsPort": 443, - "interfaceName": "eth0", - "ip": "172.16.239.120", - "ip6": "fc01:9400:1000:8::120", - "locationId": "CDN_in_a_Box_Mid", - "port": 80, - "profile": "ATS_MID_TIER_CACHE", - "status": "REPORTED", - "type": "MID", - "routingDisabled": 0 - } - }, - "contentRouters": { - "trafficrouter": { - "api.port": "3333", - "secure.api.port": "3443", - "fqdn": "trafficrouter.infra.ciab.test", - "httpsPort": 443, - "ip": "172.16.239.60", - "ip6": "fc01:9400:1000:8::60", - "location": "CDN_in_a_Box_Edge", - "port": 80, - "profile": "CCR_CIAB", - "status": "ONLINE" + "cacheGroup": "CDN_in_a_Box_Mid", + "capabilities": [ + "heat-vision" + ], + "fqdn": "mid.infra.ciab.test", + "hashCount": 999, + "hashId": "mid", + "httpsPort": 443, + "interfaceName": "eth0", + "ip": "172.26.0.4", + "ip6": "", + "locationId": "CDN_in_a_Box_Mid", + "port": 80, + "profile": "ATS_MID_TIER_CACHE", + "routingDisabled": 0, + "status": "REPORTED", + "type": "MID" } - }, - "deliveryServices": { + }, + "deliveryServices": { "demo1": { - "anonymousBlockingEnabled": "false", - "coverageZoneOnly": "false", - "dispersion": { - "limit": 1, - "shuffled": "true" - }, - "domains": [ - "demo1.mycdn.ciab.test" - ], - "geolocationProvider": "maxmindGeolocationService", - "matchsets": [ + "anonymousBlockingEnabled": "false", + "consistentHashQueryParams": [ + "abc", + "pdq", + "xxx", + "zyx" + ], + "coverageZoneOnly": "false", + "deepCachingType": "NEVER", + "dispersion": { + "limit": 1, + "shuffled": "true" + }, + "domains": [ + "demo1.mycdn.ciab.test" + ], + "ecsEnabled": "false", + "geolocationProvider": "maxmindGeolocationService", + "ip6RoutingEnabled": "true", + "matchsets": [ + { + "matchlist": [ { - "protocol": "HTTP", - "matchlist": [ - { - "regex": ".*\\.demo1\\..*", - "match-type": "HOST" - } - ] + "match-type": "HOST", + "regex": ".*\\.demo1\\..*" } - ], - "missLocation": { - "lat": 42, - "long": -88 - }, - "protocol": { - "acceptHttps": "false", - "redirectToHttps": "false" - }, - "regionalGeoBlocking": "false", - "soa": { - "admin": "traffic_ops", - "expire": "604800", - "minimum": "30", - "refresh": "28800", - "retry": "7200" - }, - "sslEnabled": "false", - "ttls": { - "A": "", - "AAAA": "", - "NS": "3600", - "SOA": "86400" - }, - "ip6RoutingEnabled": "true", - "ecsEnabled": "false", - "routingName": "video", - "deepCachingType": "NEVER" - } - }, - "edgeLocations": { - "CDN_in_a_Box_Edge": { - "latitude": 38.897663, - "longitude": -77.036574, - "backupLocations": { - "fallbackToClosest": "true" - }, - "localizationMethods": [ - "GEO", - "CZ", - "DEEP_CZ" - ] + ], + "protocol": "HTTP" + } + ], + "missLocation": { + "lat": 42, + "long": -88 + }, + "protocol": { + "acceptHttps": "true", + "redirectToHttps": "false" + }, + "regionalGeoBlocking": "false", + "requiredCapabilities": [ + "heat-vision" + ], + "routingName": "video", + "soa": { + "admin": "traffic_ops", + "expire": "604800", + "minimum": "30", + "refresh": "28800", + "retry": "7200" + }, + "sslEnabled": "true", + "topology": "my-topology", + "ttls": { + "A": "", + "AAAA": "", + "NS": "3600", + "SOA": "86400" + } } - }, - "trafficRouterLocations": { + }, + "edgeLocations": { "CDN_in_a_Box_Edge": { - "latitude": 38.897663, - "longitude": -77.036574, - "backupLocations": { - "fallbackToClosest": "false" - }, - "localizationMethods": [ - "GEO", - "CZ", - "DEEP_CZ" - ] + "backupLocations": { + "fallbackToClosest": "true" + }, + "latitude": 38.897663, + "localizationMethods": [ + "GEO", + "CZ", + "DEEP_CZ" + ], + "longitude": -77.036574 } - }, - "monitors": { + }, + "monitors": { "trafficmonitor": { - "fqdn": "trafficmonitor.infra.ciab.test", - "httpsPort": 443, - "ip": "172.16.239.40", - "ip6": "fc01:9400:1000:8::40", - "location": "CDN_in_a_Box_Edge", - "port": 80, - "profile": "RASCAL-Traffic_Monitor", - "status": "ONLINE" + "fqdn": "trafficmonitor.infra.ciab.test", + "httpsPort": 443, + "ip": "172.26.0.14", + "ip6": "", + "location": "CDN_in_a_Box_Edge", + "port": 80, + "profile": "RASCAL-Traffic_Monitor", + "status": "ONLINE" } - }, - "stats": { + }, + "stats": { "CDN_name": "CDN-in-a-Box", - "date": 1544635937, - "tm_host": "trafficops.infra.ciab.test", - "tm_path": "/tools/write_crconfig/CDN-in-a-Box", + "date": 1590600715, + "tm_host": "trafficops.infra.ciab.test:443", + "tm_path": "/api/3.0/snapshot", "tm_user": "admin", - "tm_version": "traffic_ops-3.0.0-9813.8ad7bd8e.el7" + "tm_version": "development" + }, + "topologies": { + "my-topology": { + "nodes": [ + "CDN_in_a_Box_Edge" + ] + } + }, + "trafficRouterLocations": { + "CDN_in_a_Box_Edge": { + "backupLocations": { + "fallbackToClosest": "false" + }, + "latitude": 38.897663, + "localizationMethods": [ + "GEO", + "CZ", + "DEEP_CZ" + ], + "longitude": -77.036574 + } + } } - }} + } + .. [#httpOnly] These only apply to HTTP-:ref:`routed ` :term:`Delivery Services` diff --git a/docs/source/api/v3/cdns_name_snapshot_new.rst b/docs/source/api/v3/cdns_name_snapshot_new.rst index ff81715229..e251b08c10 100644 --- a/docs/source/api/v3/cdns_name_snapshot_new.rst +++ b/docs/source/api/v3/cdns_name_snapshot_new.rst @@ -41,9 +41,10 @@ Request Structure :caption: Request Example GET /api/3.0/cdns/CDN-in-a-Box/snapshot/new HTTP/1.1 - Host: trafficops.infra.ciab.test - User-Agent: curl/7.47.0 + User-Agent: python-requests/2.23.0 + Accept-Encoding: gzip, deflate Accept: */* + Connection: keep-alive Cookie: mojolicious=... Response Structure @@ -320,6 +321,10 @@ Response Structure :tm_user: The username of the currently logged-in user :tm_version: The full version number of the Traffic Ops server, including release number, git commit hash, and supported Enterprise Linux version +:topologies: An array of :term:`Topologies` where each key is the name of that Topology. + + :nodes: An array of the names of the :term:`Edge-Tier` :term:`Cache Groups` in this :term:`Topology`. :term:`Mid-Tier` Cache Groups in the topology are not included. + :trafficRouterLocations: An object containing keys which are the :ref:`names of Cache Groups ` within the CDN which contain Traffic Routers :backupLocations: An object that describes this :ref:`Cache Group's Fallbacks ` @@ -344,205 +349,224 @@ Response Structure Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept, Set-Cookie, Cookie Access-Control-Allow-Methods: POST,GET,OPTIONS,PUT,DELETE Access-Control-Allow-Origin: * + Content-Encoding: gzip Content-Type: application/json - Set-Cookie: mojolicious=...; Path=/; Expires=Mon, 18 Nov 2019 17:40:54 GMT; Max-Age=3600; HttpOnly - Whole-Content-Sha512: MWzgAYngmU1IEIxRa0C6VfY+MMuu7T9OCiIj1Aul58pA7J7DiS6r8wjVRVVW8W2Eu2V9BC7OEacR1fQyuIsRWg== + Set-Cookie: mojolicious=...; Path=/; Expires=Wed, 27 May 2020 20:31:13 GMT; Max-Age=3600; HttpOnly + Vary: Accept-Encoding + Whole-Content-Sha512: M6uhE2oPpjpTUR7gALsPOnM2CepD+VCAjp4dj5Xnppo0G5zL31PQgiteD23q67r7/bq/JJpMvIvdaENVYFtrqQ== X-Server-Name: traffic_ops_golang/ - Date: Wed, 12 Dec 2018 21:41:48 GMT - Transfer-Encoding: chunked - - { "response": { - "config": { - "api.cache-control.max-age": "10", - "certificates.polling.interval": "300000", - "consistent.dns.routing": "true", - "coveragezone.polling.interval": "3600000", - "coveragezone.polling.url": "https://trafficops.infra.ciab.test:443/coverage-zone.json", - "dnssec.dynamic.response.expiration": "300s", - "dnssec.enabled": "false", - "domain_name": "mycdn.ciab.test", - "federationmapping.polling.interval": "60000", - "federationmapping.polling.url": "https://${toHostname}/api/3.0/federations", - "geolocation.polling.interval": "86400000", - "geolocation.polling.url": "https://trafficops.infra.ciab.test:443/GeoLite2-City.mmdb.gz", - "keystore.maintenance.interval": "300", - "neustar.polling.interval": "86400000", - "neustar.polling.url": "https://trafficops.infra.ciab.test:443/neustar.tar.gz", - "soa": { - "admin": "twelve_monkeys", - "expire": "604800", - "minimum": "30", - "refresh": "28800", - "retry": "7200" - }, - "steeringmapping.polling.interval": "60000", - "ttls": { - "A": "3600", - "AAAA": "3600", - "DNSKEY": "30", - "DS": "30", - "NS": "3600", - "SOA": "86400" - }, - "zonemanager.cache.maintenance.interval": "300", - "zonemanager.threadpool.scale": "0.50" - }, - "contentServers": { - "edge": { - "cacheGroup": "CDN_in_a_Box_Edge", - "fqdn": "edge.infra.ciab.test", - "hashCount": 999, - "hashId": "edge", - "httpsPort": 443, - "interfaceName": "eth0", - "ip": "172.16.239.100", - "ip6": "fc01:9400:1000:8::100", - "locationId": "CDN_in_a_Box_Edge", - "port": 80, - "profile": "ATS_EDGE_TIER_CACHE", - "status": "REPORTED", - "type": "EDGE", - "deliveryServices": { - "demo1": [ - "edge.demo1.mycdn.ciab.test" - ] - }, - "routingDisabled": 0 - }, - "mid": { - "cacheGroup": "CDN_in_a_Box_Mid", - "fqdn": "mid.infra.ciab.test", - "hashCount": 999, - "hashId": "mid", - "httpsPort": 443, - "interfaceName": "eth0", - "ip": "172.16.239.120", - "ip6": "fc01:9400:1000:8::120", - "locationId": "CDN_in_a_Box_Mid", - "port": 80, - "profile": "ATS_MID_TIER_CACHE", - "status": "REPORTED", - "type": "MID", - "routingDisabled": 0 - } - }, - "contentRouters": { - "trafficrouter": { - "api.port": "3333", - "secure.api.port": "3443", - "fqdn": "trafficrouter.infra.ciab.test", - "httpsPort": 443, - "ip": "172.16.239.60", - "ip6": "fc01:9400:1000:8::60", - "location": "CDN_in_a_Box_Edge", - "port": 80, - "profile": "CCR_CIAB", - "status": "ONLINE" - } - }, - "deliveryServices": { - "demo1": { - "anonymousBlockingEnabled": "false", - "coverageZoneOnly": "false", - "dispersion": { - "limit": 1, - "shuffled": "true" - }, - "domains": [ - "demo1.mycdn.ciab.test" - ], - "geolocationProvider": "maxmindGeolocationService", - "matchsets": [ - { - "protocol": "HTTP", - "matchlist": [ - { - "regex": ".*\\.demo1\\..*", - "match-type": "HOST" - } - ] - } - ], - "missLocation": { - "lat": 42, - "long": -88 - }, - "protocol": { - "acceptHttps": "false", - "redirectToHttps": "false" - }, - "regionalGeoBlocking": "false", + Date: Wed, 27 May 2020 19:31:13 GMT + Content-Length: 1374 + + { + "response": { + "config": { + "api.cache-control.max-age": "10", + "certificates.polling.interval": "300000", + "consistent.dns.routing": "true", + "coveragezone.polling.interval": "3600000", + "coveragezone.polling.url": "https://trafficops.infra.ciab.test:443/coverage-zone.json", + "dnssec.dynamic.response.expiration": "300s", + "dnssec.enabled": "false", + "domain_name": "mycdn.ciab.test", + "federationmapping.polling.interval": "60000", + "federationmapping.polling.url": "https://${toHostname}/api/2.0/federations/all", + "geolocation.polling.interval": "86400000", + "geolocation.polling.url": "https://trafficops.infra.ciab.test:443/GeoLite2-City.mmdb.gz", + "keystore.maintenance.interval": "300", + "neustar.polling.interval": "86400000", + "neustar.polling.url": "https://trafficops.infra.ciab.test:443/neustar.tar.gz", "soa": { - "admin": "traffic_ops", + "admin": "twelve_monkeys", "expire": "604800", "minimum": "30", "refresh": "28800", "retry": "7200" }, - "sslEnabled": "false", + "steeringmapping.polling.interval": "60000", "ttls": { - "A": "", - "AAAA": "", + "A": "3600", + "AAAA": "3600", + "DNSKEY": "30", + "DS": "30", "NS": "3600", "SOA": "86400" }, - "ip6RoutingEnabled": "true", - "ecsEnabled": "false", - "routingName": "video", - "deepCachingType": "NEVER" - } - }, - "edgeLocations": { - "CDN_in_a_Box_Edge": { - "latitude": 38.897663, - "longitude": -77.036574, - "backupLocations": { - "fallbackToClosest": "true", - "list": [ - "test" - ] - }, - "localizationMethods": [ - "GEO", - "CZ", - "DEEP_CZ" - ] - } - }, - "trafficRouterLocations": { - "CDN_in_a_Box_Edge": { - "latitude": 38.897663, - "longitude": -77.036574, - "backupLocations": { - "fallbackToClosest": "false" + "zonemanager.cache.maintenance.interval": "300", + "zonemanager.threadpool.scale": "0.50" + }, + "contentServers": { + "edge": { + "cacheGroup": "CDN_in_a_Box_Edge", + "capabilities": [ + "heat-vision" + ], + "fqdn": "edge.infra.ciab.test", + "hashCount": 999, + "hashId": "edge", + "httpsPort": 443, + "interfaceName": "eth0", + "ip": "172.26.0.3", + "ip6": "", + "locationId": "CDN_in_a_Box_Edge", + "port": 80, + "profile": "ATS_EDGE_TIER_CACHE", + "status": "REPORTED", + "type": "EDGE", + "routingDisabled": 0 }, - "localizationMethods": [ - "GEO", - "CZ", - "DEEP_CZ" - ] - } - }, - "monitors": { - "trafficmonitor": { - "fqdn": "trafficmonitor.infra.ciab.test", - "httpsPort": 443, - "ip": "172.16.239.40", - "ip6": "fc01:9400:1000:8::40", - "location": "CDN_in_a_Box_Edge", - "port": 80, - "profile": "RASCAL-Traffic_Monitor", - "status": "ONLINE" + "mid": { + "cacheGroup": "CDN_in_a_Box_Mid", + "capabilities": [ + "heat-vision" + ], + "fqdn": "mid.infra.ciab.test", + "hashCount": 999, + "hashId": "mid", + "httpsPort": 443, + "interfaceName": "eth0", + "ip": "172.26.0.4", + "ip6": "", + "locationId": "CDN_in_a_Box_Mid", + "port": 80, + "profile": "ATS_MID_TIER_CACHE", + "status": "REPORTED", + "type": "MID", + "routingDisabled": 0 + } + }, + "contentRouters": { + "trafficrouter": { + "api.port": "3333", + "fqdn": "trafficrouter.infra.ciab.test", + "httpsPort": 443, + "ip": "172.26.0.15", + "ip6": "", + "location": "CDN_in_a_Box_Edge", + "port": 80, + "profile": "CCR_CIAB", + "secure.api.port": "3443", + "status": "ONLINE" + } + }, + "deliveryServices": { + "demo1": { + "anonymousBlockingEnabled": "false", + "consistentHashQueryParams": [ + "abc", + "pdq", + "xxx", + "zyx" + ], + "coverageZoneOnly": "false", + "deepCachingType": "NEVER", + "dispersion": { + "limit": 1, + "shuffled": "true" + }, + "domains": [ + "demo1.mycdn.ciab.test" + ], + "ecsEnabled": "false", + "geolocationProvider": "maxmindGeolocationService", + "ip6RoutingEnabled": "true", + "matchsets": [ + { + "protocol": "HTTP", + "matchlist": [ + { + "regex": ".*\\.demo1\\..*", + "match-type": "HOST" + } + ] + } + ], + "missLocation": { + "lat": 42, + "long": -88 + }, + "protocol": { + "acceptHttps": "true", + "redirectToHttps": "false" + }, + "regionalGeoBlocking": "false", + "requiredCapabilities": [ + "heat-vision" + ], + "routingName": "video", + "soa": { + "admin": "traffic_ops", + "expire": "604800", + "minimum": "30", + "refresh": "28800", + "retry": "7200" + }, + "sslEnabled": "true", + "topology": "my-topology", + "ttls": { + "A": "", + "AAAA": "", + "NS": "3600", + "SOA": "86400" + } + } + }, + "edgeLocations": { + "CDN_in_a_Box_Edge": { + "latitude": 38.897663, + "longitude": -77.036574, + "backupLocations": { + "fallbackToClosest": "true" + }, + "localizationMethods": [ + "GEO", + "CZ", + "DEEP_CZ" + ] + } + }, + "trafficRouterLocations": { + "CDN_in_a_Box_Edge": { + "latitude": 38.897663, + "longitude": -77.036574, + "backupLocations": { + "fallbackToClosest": "false" + }, + "localizationMethods": [ + "GEO", + "CZ", + "DEEP_CZ" + ] + } + }, + "monitors": { + "trafficmonitor": { + "fqdn": "trafficmonitor.infra.ciab.test", + "httpsPort": 443, + "ip": "172.26.0.14", + "ip6": "", + "location": "CDN_in_a_Box_Edge", + "port": 80, + "profile": "RASCAL-Traffic_Monitor", + "status": "ONLINE" + } + }, + "stats": { + "CDN_name": "CDN-in-a-Box", + "date": 1590607873, + "tm_host": "trafficops.infra.ciab.test:443", + "tm_path": "/api/3.0/cdns/CDN-in-a-Box/snapshot/new", + "tm_user": "admin", + "tm_version": "development" + }, + "topologies": { + "my-topology": { + "nodes": [ + "CDN_in_a_Box_Edge" + ] + } } - }, - "stats": { - "CDN_name": "CDN-in-a-Box", - "date": 1544650908, - "tm_host": "ipcdn-cache-51.cdnlab.comcast.net:6443", - "tm_path": "/tools/write_crconfig/CDN-in-a-Box", - "tm_user": "admin", - "tm_version": "traffic_ops-3.0.0-9813.8ad7bd8e.el7" } - }} + } .. [#httpOnly] These only apply to HTTP-:ref:`routed ` :term:`Delivery Services` diff --git a/docs/source/glossary.rst b/docs/source/glossary.rst index 1c2553b78c..ce6d748fb5 100644 --- a/docs/source/glossary.rst +++ b/docs/source/glossary.rst @@ -123,6 +123,7 @@ Glossary Edge Edge-tier + Edge-Tier Edge-tier cache Edge-tier caches Edge-tier cache server @@ -230,6 +231,7 @@ Glossary Mid Mid-tier + Mid-Tier Mid-tier cache Mid-tier caches Mid-tier cache server diff --git a/lib/go-tc/crconfig.go b/lib/go-tc/crconfig.go index 0d08542f41..652be79d5a 100644 --- a/lib/go-tc/crconfig.go +++ b/lib/go-tc/crconfig.go @@ -30,6 +30,7 @@ type CRConfig struct { RouterLocations map[string]CRConfigLatitudeLongitude `json:"trafficRouterLocations,omitempty"` Monitors map[string]CRConfigMonitor `json:"monitors,omitempty"` Stats CRConfigStats `json:"stats,omitempty"` + Topologies map[string]CRConfigTopology `json:"topologies,omitempty"` } // CRConfigConfig used to be the type of CRConfig's Config field, though @@ -135,6 +136,10 @@ type CRConfigDeliveryService struct { TTLs *CRConfigTTL `json:"ttls,omitempty"` } +type CRConfigTopology struct { + Nodes []string `json:"nodes"` +} + type CRConfigGeoEnabled struct { CountryCode string `json:"countryCode"` } diff --git a/traffic_ops/traffic_ops_golang/crconfig/crconfig.go b/traffic_ops/traffic_ops_golang/crconfig/crconfig.go index 55d3d217ba..c89f93b599 100644 --- a/traffic_ops/traffic_ops_golang/crconfig/crconfig.go +++ b/traffic_ops/traffic_ops_golang/crconfig/crconfig.go @@ -52,6 +52,9 @@ func Make(tx *sql.Tx, cdn, user, toHost, reqPath, toVersion string, useClientReq if crc.DeliveryServices, err = makeDSes(cdn, cdnDomain, tx); err != nil { return nil, errors.New("Error getting Delivery Services: " + err.Error()) } + if crc.Topologies, err = makeTopologies(tx); err != nil { + return nil, errors.New("Error getting Topologies: " + err.Error()) + } if !useClientReqHost { paramTMURL, ok, err := getGlobalParam(tx, "tm.url") diff --git a/traffic_ops/traffic_ops_golang/crconfig/topologies.go b/traffic_ops/traffic_ops_golang/crconfig/topologies.go new file mode 100644 index 0000000000..82db3e32fc --- /dev/null +++ b/traffic_ops/traffic_ops_golang/crconfig/topologies.go @@ -0,0 +1,66 @@ +package crconfig + +import ( + "database/sql" + "errors" + "fmt" + "github.com/apache/trafficcontrol/lib/go-tc" + "github.com/lib/pq" +) + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +func makeTopologies(tx *sql.Tx) (map[string]tc.CRConfigTopology, error) { + query := ` +SELECT + t.name, + (SELECT ARRAY_AGG(tc.cachegroup ORDER BY tc.cachegroup) + FROM topology_cachegroup tc + JOIN cachegroup c ON c.name = tc.cachegroup + JOIN type ON type.id = c.type + WHERE t.name = tc.topology + AND type.name = $1 + ) AS nodes +FROM topology t +ORDER BY t.name +` + var rows *sql.Rows + var err error + if rows, err = tx.Query(query, tc.CacheGroupEdgeTypeName); err != nil { + return nil, errors.New("querying topologies: " + err.Error()) + } + + topologies := map[string]tc.CRConfigTopology{} + for rows.Next() { + topology := tc.CRConfigTopology{} + var name string + if err = rows.Scan( + &name, + pq.Array(&topology.Nodes), + ); err != nil { + return nil, errors.New("scanning topology: " + err.Error()) + } + topologies[name] = topology + } + if err = rows.Close(); err != nil { + return nil, fmt.Errorf("closing rows: %s", err.Error()) + } + return topologies, nil +} diff --git a/traffic_ops/traffic_ops_golang/crconfig/topologies_test.go b/traffic_ops/traffic_ops_golang/crconfig/topologies_test.go new file mode 100644 index 0000000000..3c11fd75c6 --- /dev/null +++ b/traffic_ops/traffic_ops_golang/crconfig/topologies_test.go @@ -0,0 +1,83 @@ +package crconfig + +import ( + "context" + "encoding/json" + "github.com/apache/trafficcontrol/lib/go-tc" + "gopkg.in/DATA-DOG/go-sqlmock.v1" + "reflect" + "strings" + "testing" + "time" +) + +func randTopology() tc.CRConfigTopology { + return tc.CRConfigTopology{ + Nodes: randStrArray(), + } +} + +func ExpectedMakeTops() map[string]tc.CRConfigTopology { + return map[string]tc.CRConfigTopology{ + "top1": randTopology(), + "top2": randTopology(), + } +} + +func MockMakeTops(mock sqlmock.Sqlmock, expected map[string]tc.CRConfigTopology) { + rows := sqlmock.NewRows([]string{ + "name", + "nodes"}) + + for topName, top := range expected { + nodes := "{" + strings.Join(top.Nodes, ",") + "}" + rows = rows.AddRow( + topName, + nodes) + } + mock.ExpectQuery("SELECT").WillReturnRows(rows) +} + +func TestMakeTops(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) + } + + expected := ExpectedMakeTops() + mock.ExpectBegin() + MockMakeTops(mock, expected) + mock.ExpectCommit() + + dbCtx, _ := context.WithTimeout(context.TODO(), time.Duration(10)*time.Second) + tx, err := db.BeginTx(dbCtx, nil) + if err != nil { + t.Fatal("creating transaction: ", err) + } + + actual, err := makeTopologies(tx) + if err != nil { + t.Fatal("makeTopologies expected: nil error, actual: ", err) + } + + if err = db.Close(); err != nil { + t.Fatal("closing db: ", err) + } + + if len(actual) != len(expected) { + t.Fatalf("makeTopologies len expected: %v, actual: %v", len(expected), len(actual)) + } + + for topName, top := range expected { + actualTop, ok := actual[topName] + if !ok { + t.Errorf("makeTopologies expected: %v, actual: missing", topName) + continue + } + expectedBts, _ := json.MarshalIndent(top, " ", " ") + actualBts, _ := json.MarshalIndent(actualTop, " ", " ") + if !reflect.DeepEqual(expectedBts, actualBts) { + t.Errorf("makeDSes ds %+v expected: %+v\n\nactual: %+v\n\n\n", topName, string(expectedBts), string(actualBts)) + } + } +} From 322fc09c70c628fe89bcc834c53df7002b9598da Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Tue, 5 May 2020 02:55:04 -0600 Subject: [PATCH 06/30] Add requiredCapabilities field to DeliveryService --- .../core/ds/DeliveryService.java | 20 +++++++++++++++++++ .../core/ds/DeliveryServiceTest.java | 14 ++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java index 4a5a8cb9cb..a199735bf5 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java @@ -83,6 +83,7 @@ public class DeliveryService { private final JsonNode props; private boolean isDns; private final String routingName; + private final Set requiredCapabilities; private final boolean shouldAppendQueryString; private final Geolocation missLocation; private final Dispersion dispersion; @@ -133,6 +134,21 @@ public DeliveryService(final String id, final JsonNode dsJo) throws JsonUtilsExc this.shouldAppendQueryString = JsonUtils.optBoolean(dsJo, "appendQueryString", true); this.ecsEnabled = JsonUtils.optBoolean(dsJo, "ecsEnabled"); + this.requiredCapabilities = new HashSet<>(); + if (dsJo.has("requiredCapabilities")) { + final JsonNode requiredCapabilitiesNode = dsJo.get("requiredCapabilities"); + if (!requiredCapabilitiesNode.isArray()) { + LOGGER.error("Delivery Service '" + id + "' has malformed requiredCapabilities. Disregarding."); + } else { + requiredCapabilitiesNode.forEach((requiredCapabilityNode) -> { + final String requiredCapability = requiredCapabilityNode.asText(); + if (!requiredCapability.isEmpty()) { + this.requiredCapabilities.add(requiredCapability); + } + }); + } + } + this.consistentHashQueryParams = new HashSet(); if (dsJo.has("consistentHashQueryParams")) { final JsonNode cqpNode = dsJo.get("consistentHashQueryParams"); @@ -591,6 +607,10 @@ public String getRoutingName() { return routingName; } + public boolean hasRequiredCapabilities(final Set capabilities) { + return this.requiredCapabilities.containsAll(capabilities); + } + public Dispersion getDispersion() { return dispersion; } diff --git a/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryServiceTest.java b/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryServiceTest.java index 59da1c97e1..2306fcdd69 100644 --- a/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryServiceTest.java +++ b/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryServiceTest.java @@ -19,6 +19,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.Test; +import org.powermock.reflect.Whitebox; + +import java.util.HashSet; +import java.util.Set; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; @@ -59,7 +63,7 @@ public void itHandlesDuplicatesInConsistentHashQueryParams() throws Exception { public void itExtractsQueryParams() throws Exception { final JsonNode json = (new ObjectMapper()).readTree("{\"routingName\":\"edge\",\"coverageZoneOnly\":false,\"consistentHashQueryParams\":[\"test\", \"quest\"]}"); final HTTPRequest r = new HTTPRequest(); - r.setPath("/path1234/some_stream_name1234/some_other_info.m3u8"); + r.setPath("/path1234/some_stream_name1234/some_other_info.m3u8"); r.setQueryString("test=value&foo=fizz&quest=oth%20ervalue&bar=buzz"); assert (new DeliveryService("test", json)).extractSignificantQueryParams(r).equals("quest=oth ervaluetest=value"); } @@ -75,4 +79,12 @@ public void itConfiguresRequestHeadersFromJSON() throws Exception { assertThat(deliveryService.getRequestHeaders(), containsInAnyOrder("Cache-Control", "Cookie", "Content-Type", "If-Modified-Since")); } + @Test + public void itAddsRequiredCapabilities() throws Exception { + final ObjectMapper mapper = new ObjectMapper(); + final JsonNode jsonConfiguration = mapper.readTree("{\"requiredCapabilities\":[\"all-read\",\"all-write\",\"cdn-read\"],\"routingName\":\"edge\",\"coverageZoneOnly\":false}"); + final DeliveryService deliveryService = new DeliveryService("has-required-capabilities", jsonConfiguration); + + assertThat(Whitebox.getInternalState(deliveryService, "requiredCapabilities"), containsInAnyOrder("all-read", "all-write", "cdn-read")); + } } From 8f4058d43666951e4fea0fa0883d7d865401d1b7 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Tue, 5 May 2020 04:10:21 -0600 Subject: [PATCH 07/30] Add capabilities field to Cache --- .../core/config/ConfigHandler.java | 16 +++++++++++ .../traffic_router/core/edge/Node.java | 27 +++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java index 7cc3ee94d5..c515371f4f 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java @@ -357,6 +357,22 @@ private void parseCacheConfig(final JsonNode contentServers, final CacheRegister cache.setFqdn(JsonUtils.getString(jo, "fqdn")); cache.setPort(JsonUtils.getInt(jo, "port")); + if (jo.has("capabilities")) { + final Set capabilities = new HashSet<>(); + final JsonNode capabilitiesNode = jo.get("capabilities"); + if (!capabilitiesNode.isArray()) { + LOGGER.error("Server '" + hashId + "' has malformed capabilities. Disregarding."); + } else { + capabilitiesNode.forEach((capabilityNode) -> { + final String capability = capabilityNode.asText(); + if (!capability.isEmpty()) { + capabilities.add(capability); + } + }); + } + cache.addCapabilities(capabilities); + } + final String ip = JsonUtils.getString(jo, "ip"); final String ip6 = JsonUtils.optString(jo, "ip6"); diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/edge/Node.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/edge/Node.java index ea4158f58a..c59366a01a 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/edge/Node.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/edge/Node.java @@ -19,7 +19,12 @@ import java.net.InetAddress; import java.net.UnknownHostException; import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; import com.comcast.cdn.traffic_control.traffic_router.core.util.JsonUtils; import com.fasterxml.jackson.databind.JsonNode; @@ -41,6 +46,10 @@ public enum IpVersions { private List ipAddresses; private List unavailableIpAddresses; private int port; + private final Map deliveryServices = new HashMap(); + private final Set capabilities = new HashSet<>(); + private final Geolocation geolocation; + private final Hashable hashable = new DefaultHashable(); private int httpsPort = 443; public Node(final String id) { @@ -119,6 +128,24 @@ public int hashCode() { .toHashCode(); } + public void addCapabilities(final Set capabilities) { + this.capabilities.addAll(capabilities); + } + + public Set getCapabilities() { + return this.capabilities; + } + + public void setDeliveryServices(final Collection deliveryServices) { + for (final DeliveryServiceReference deliveryServiceReference : deliveryServices) { + this.deliveryServices.put(deliveryServiceReference.getDeliveryServiceId(), deliveryServiceReference); + } + } + + public boolean hasDeliveryService(final String deliveryServiceId) { + return deliveryServices.containsKey(deliveryServiceId); + } + public void setFqdn(final String fqdn) { this.fqdn = fqdn; } From d522946c3c8809f2e9e2e1f6dfc68f55748269eb Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Tue, 5 May 2020 02:55:18 -0600 Subject: [PATCH 08/30] Add topology field to DeliveryService --- .../traffic_router/core/ds/DeliveryService.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java index a199735bf5..1fad374200 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java @@ -83,6 +83,7 @@ public class DeliveryService { private final JsonNode props; private boolean isDns; private final String routingName; + private String topology; private final Set requiredCapabilities; private final boolean shouldAppendQueryString; private final Geolocation missLocation; @@ -134,6 +135,9 @@ public DeliveryService(final String id, final JsonNode dsJo) throws JsonUtilsExc this.shouldAppendQueryString = JsonUtils.optBoolean(dsJo, "appendQueryString", true); this.ecsEnabled = JsonUtils.optBoolean(dsJo, "ecsEnabled"); + if (dsJo.has("topology")) { + this.topology = JsonUtils.optString(dsJo, "topology"); + } this.requiredCapabilities = new HashSet<>(); if (dsJo.has("requiredCapabilities")) { final JsonNode requiredCapabilitiesNode = dsJo.get("requiredCapabilities"); @@ -607,6 +611,10 @@ public String getRoutingName() { return routingName; } + public String getTopology() { + return topology; + } + public boolean hasRequiredCapabilities(final Set capabilities) { return this.requiredCapabilities.containsAll(capabilities); } From 669bd23d75dc83aeea8ae45d569860f3d5ef12fd Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Tue, 5 May 2020 04:15:09 -0600 Subject: [PATCH 09/30] Assign a delivery service to all servers in its topology that have its required capabilities --- CHANGELOG.md | 1 + .../core/config/ConfigHandler.java | 47 ++++++++++++----- .../core/config/ConfigHandlerTest.java | 51 +++++++++++++++++-- 3 files changed, 81 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e052f082fd..e8b2afc7e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Traffic Portal: Added the ability to create, read, update and delete flexible topologies. - Traffic Portal: Added the ability to assign topologies to delivery services. - Traffic Portal: Added the ability to view all delivery services and cache groups associated with a topology. + - Traffic Router: Added support for topology-based delivery services - Updated /servers/details to use multiple interfaces in API v3 - Added [Edge Traffic Routing](https://traffic-control-cdn.readthedocs.io/en/latest/admin/traffic_router.html#edge-traffic-routing) feature which allows Traffic Router to localize more DNS record types than just the routing name for DNS delivery services - Astats csv support - astats will now respond to `Accept: text/csv` and return a csv formatted stats list diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java index c515371f4f..991438fbaa 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java @@ -18,19 +18,11 @@ import java.io.IOException; import java.net.UnknownHostException; import java.net.URL; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; -import java.util.Iterator; +import java.util.*; import java.util.concurrent.BlockingQueue; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; +import java.util.stream.Stream; import com.comcast.cdn.traffic_control.traffic_router.core.ds.LetsEncryptDnsChallengeWatcher; import com.comcast.cdn.traffic_control.traffic_router.core.ds.SteeringWatcher; @@ -76,6 +68,7 @@ public class ConfigHandler { private static long lastSnapshotTimestamp = 0; private static Object configSync = new Object(); public static String deliveryServicesKey = "deliveryServices"; + public static String topologiesKey = "topologies"; private TrafficRouterManager trafficRouterManager; private GeolocationDatabaseUpdater geolocationDatabaseUpdater; @@ -227,6 +220,9 @@ public boolean processConfig(final String jsonStr) throws JsonUtilsException, IO parseEdgeTrafficRouterLocations(jo, cacheRegister); parseCacheConfig(JsonUtils.getJsonNode(jo, "contentServers"), cacheRegister); + if (jo.has(topologiesKey)) { + parseTopologyConfig(JsonUtils.getJsonNode(jo, topologiesKey), deliveryServiceMap, cacheRegister); + } parseMonitorConfig(JsonUtils.getJsonNode(jo, "monitors")); federationsWatcher.configure(config); @@ -300,7 +296,7 @@ public void setRegionalGeoUpdater(final RegionalGeoUpdater regionalGeoUpdater) { public void setAnonymousIpConfigUpdater(final AnonymousIpConfigUpdater anonymousIpConfigUpdater) { this.anonymousIpConfigUpdater = anonymousIpConfigUpdater; } - + public void setAnonymousIpDatabaseUpdater(final AnonymousIpDatabaseUpdater anonymousIpDatabaseUpdater) { this.anonymousIpDatabaseUpdater = anonymousIpDatabaseUpdater; } @@ -475,6 +471,31 @@ private Map parseDeliveryServiceConfig(final JsonNode a return deliveryServiceMap; } + private void parseTopologyConfig(final JsonNode allTopologies, final Map deliveryServiceMap, final CacheRegister cacheRegister) { + final Map> topologyMap = new HashMap<>(); + allTopologies.fieldNames().forEachRemaining((String topologyName) -> { + final List nodes = new ArrayList<>(); + allTopologies.get(topologyName).get("nodes").forEach((JsonNode cache) -> nodes.add(cache.textValue())); + topologyMap.put(topologyName, nodes); + }); + + deliveryServiceMap.forEach((xmlId, ds) -> { + final List dsReferences = new ArrayList<>(); + try { + dsReferences.add(new DeliveryServiceReference(ds.getId(), ds.getDomain())); + } catch (ParseException e) { + LOGGER.error("Unable to create a DeliveryServiceReference from DeliveryService '" + ds.getId() + "'", e); + return; + } + Stream.of(ds.getTopology()) + .filter((topologyName) -> !Objects.isNull(topologyName) && topologyMap.containsKey(topologyName)) + .flatMap((topologyName) -> topologyMap.get(topologyName).stream()) + .flatMap((node) -> cacheRegister.getCacheLocation(node).getCaches().stream()) + .filter((cache) -> ds.hasRequiredCapabilities(cache.getCapabilities())) + .forEach((cache) -> cache.setDeliveryServices(dsReferences)); + }); + } + private void parseDeliveryServiceMatchSets(final JsonNode allDeliveryServices, final Map deliveryServiceMap, final CacheRegister cacheRegister) throws JsonUtilsException { final TreeSet deliveryServiceMatchers = new TreeSet<>(); final JsonNode config = cacheRegister.getConfig(); @@ -557,7 +578,7 @@ private void initGeoFailedRedirect(final Map dsMap, fin /** * Parses the geolocation database configuration and updates the database if the URL has * changed. - * + * * @param config * the {@link TrafficRouterConfiguration} * @throws JsonUtilsException @@ -609,7 +630,7 @@ private void parseAnonymousIpConfig(final JsonNode jo) throws JsonUtilsException AnonymousIp.getCurrentConfig().enabled = false; return; } - + if (databaseUrl == null) { LOGGER.info(anonymousPollingUrl + " not configured; stopping service updater and disabling feature"); getAnonymousIpDatabaseUpdater().stopServiceUpdater(); diff --git a/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandlerTest.java b/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandlerTest.java index 9e9e000dd3..3967146651 100644 --- a/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandlerTest.java +++ b/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandlerTest.java @@ -15,12 +15,11 @@ package com.comcast.cdn.traffic_control.traffic_router.core.config; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; +import java.util.*; +import com.comcast.cdn.traffic_control.traffic_router.core.cache.Cache; +import com.comcast.cdn.traffic_control.traffic_router.core.cache.CacheLocation; +import com.comcast.cdn.traffic_control.traffic_router.geolocation.Geolocation; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.comcast.cdn.traffic_control.traffic_router.core.edge.CacheLocation.LocalizationMethod; @@ -248,6 +247,48 @@ public Void answer(InvocationOnMock invocation) { assertThat(typeUrl[0], equalTo(path)); } + @Test + public void itParsesTheTopologiesConfig() throws Exception { + /* Make the CacheLocation, add a Cache, and add the CacheLocation to the CacheRegister */ + final String cacheId = "edge"; + final Cache cache = new Cache(cacheId, cacheId, 0); + final String location = "CDN_in_a_Box_Edge"; + final CacheLocation cacheLocation = new CacheLocation(location, new Geolocation(38.897663, 38.897663)); + cacheLocation.addCache(cache); + final Set locations = new HashSet<>(); + locations.add(cacheLocation); + final CacheRegister register = new CacheRegister(); + register.setConfiguredLocations(locations); + + /* Add a capability to the Cache */ + final String capability = "a-capability"; + final Set capabilities = new HashSet<>(); + capabilities.add(capability); + cache.addCapabilities(capabilities); + + /* Mock a DeliveryService and add it to our DeliveryService Map */ + final String dsId = "top-ds"; + final String domain = "ds.cdn.com"; + final String topology = "foo"; + final DeliveryService ds = mock(DeliveryService.class); + when(ds.getId()).thenReturn(dsId); + when(ds.getDomain()).thenReturn(domain); + when(ds.getTopology()).thenReturn(topology); + when(ds.hasRequiredCapabilities(capabilities)).thenReturn(true); + final Map dsMap = new HashMap<>(); + dsMap.put(dsId, ds); + + /* Parse the Topologies config JSON */ + final ObjectMapper mapper = new ObjectMapper(); + final JsonNode allTopologiesJson = mapper.readTree("{\"" + topology + "\":{\"nodes\":[\"" + location + "\"]}}"); + Whitebox.invokeMethod(handler, "parseTopologyConfig", allTopologiesJson, dsMap, register); + + /* Assert that the DeliveryService was assigned to the Cache */ + Collection dsReferences = cache.getDeliveryServices(); + assertThat(dsReferences.size(), equalTo(1)); + assertThat(dsReferences.iterator().next().getDeliveryServiceId(), equalTo(dsId)); + } + @Test public void testParseLocalizationMethods() throws Exception { LocalizationMethod[] allMethods = new LocalizationMethod[] { From b75109d01fb9f2d4a5a74e1d81205cf48d3869c6 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Wed, 27 May 2020 15:37:20 -0600 Subject: [PATCH 10/30] Add Apache license header --- .../crconfig/topologies_test.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/traffic_ops/traffic_ops_golang/crconfig/topologies_test.go b/traffic_ops/traffic_ops_golang/crconfig/topologies_test.go index 3c11fd75c6..9c9b637110 100644 --- a/traffic_ops/traffic_ops_golang/crconfig/topologies_test.go +++ b/traffic_ops/traffic_ops_golang/crconfig/topologies_test.go @@ -1,5 +1,24 @@ package crconfig +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + import ( "context" "encoding/json" From b8cbd20307637ecc0592a7cb9d02c4200f6f72f6 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Thu, 4 Jun 2020 16:49:06 -0600 Subject: [PATCH 11/30] Test whether server capabilities are a superset of required capabilities --- .../traffic_router/core/ds/DeliveryService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java index 1fad374200..facbf6b082 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java @@ -615,8 +615,8 @@ public String getTopology() { return topology; } - public boolean hasRequiredCapabilities(final Set capabilities) { - return this.requiredCapabilities.containsAll(capabilities); + public boolean hasRequiredCapabilities(final Set serverCapabilities) { + return serverCapabilities.containsAll(requiredCapabilities); } public Dispersion getDispersion() { From 1025e3f84a70b2ce66d60e28745ef65a0d1cae2c Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Thu, 4 Jun 2020 16:59:53 -0600 Subject: [PATCH 12/30] Expand wildcard imports --- .../traffic_router/core/config/ConfigHandler.java | 12 +++++++++++- .../core/config/ConfigHandlerTest.java | 7 ++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java index 991438fbaa..4924bfabab 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java @@ -18,7 +18,17 @@ import java.io.IOException; import java.net.UnknownHostException; import java.net.URL; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; +import java.util.Iterator; import java.util.concurrent.BlockingQueue; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; diff --git a/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandlerTest.java b/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandlerTest.java index 3967146651..2c53244f9c 100644 --- a/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandlerTest.java +++ b/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandlerTest.java @@ -15,7 +15,12 @@ package com.comcast.cdn.traffic_control.traffic_router.core.config; -import java.util.*; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; import com.comcast.cdn.traffic_control.traffic_router.core.cache.Cache; import com.comcast.cdn.traffic_control.traffic_router.core.cache.CacheLocation; From a0b28f711b4213b2468ae41df051d21278c79ab1 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Thu, 4 Jun 2020 17:08:57 -0600 Subject: [PATCH 13/30] Defer row-closing --- traffic_ops/traffic_ops_golang/crconfig/topologies.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/traffic_ops/traffic_ops_golang/crconfig/topologies.go b/traffic_ops/traffic_ops_golang/crconfig/topologies.go index 82db3e32fc..c1203a42b5 100644 --- a/traffic_ops/traffic_ops_golang/crconfig/topologies.go +++ b/traffic_ops/traffic_ops_golang/crconfig/topologies.go @@ -4,6 +4,7 @@ import ( "database/sql" "errors" "fmt" + "github.com/apache/trafficcontrol/lib/go-log" "github.com/apache/trafficcontrol/lib/go-tc" "github.com/lib/pq" ) @@ -46,6 +47,7 @@ ORDER BY t.name if rows, err = tx.Query(query, tc.CacheGroupEdgeTypeName); err != nil { return nil, errors.New("querying topologies: " + err.Error()) } + defer log.Close(rows, "unable to close DB connection") topologies := map[string]tc.CRConfigTopology{} for rows.Next() { @@ -59,8 +61,5 @@ ORDER BY t.name } topologies[name] = topology } - if err = rows.Close(); err != nil { - return nil, fmt.Errorf("closing rows: %s", err.Error()) - } return topologies, nil } From 3c2e85636c8cdfd09493b0ef67ee1c9655734ce3 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Thu, 4 Jun 2020 17:10:47 -0600 Subject: [PATCH 14/30] Move license header --- .../traffic_ops_golang/crconfig/topologies.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/traffic_ops/traffic_ops_golang/crconfig/topologies.go b/traffic_ops/traffic_ops_golang/crconfig/topologies.go index c1203a42b5..d86cdc1fb6 100644 --- a/traffic_ops/traffic_ops_golang/crconfig/topologies.go +++ b/traffic_ops/traffic_ops_golang/crconfig/topologies.go @@ -1,14 +1,5 @@ package crconfig -import ( - "database/sql" - "errors" - "fmt" - "github.com/apache/trafficcontrol/lib/go-log" - "github.com/apache/trafficcontrol/lib/go-tc" - "github.com/lib/pq" -) - /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -28,6 +19,14 @@ import ( * under the License. */ +import ( + "database/sql" + "errors" + "github.com/apache/trafficcontrol/lib/go-log" + "github.com/apache/trafficcontrol/lib/go-tc" + "github.com/lib/pq" +) + func makeTopologies(tx *sql.Tx) (map[string]tc.CRConfigTopology, error) { query := ` SELECT From 4dae880e21e5cbbd954ec344a6b92ee90aa52e55 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Thu, 4 Jun 2020 17:31:05 -0600 Subject: [PATCH 15/30] Cache Group -> Cache Server --- docs/source/api/v3/cdns_name_snapshot.rst | 2 +- docs/source/api/v3/cdns_name_snapshot_new.rst | 2 +- docs/source/glossary.rst | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/source/api/v3/cdns_name_snapshot.rst b/docs/source/api/v3/cdns_name_snapshot.rst index e85edf88c2..5452f93adf 100644 --- a/docs/source/api/v3/cdns_name_snapshot.rst +++ b/docs/source/api/v3/cdns_name_snapshot.rst @@ -120,7 +120,7 @@ Response Structure :contentServers: An object containing keys which are the (short) hostnames of the :term:`Edge-tier cache servers` in the CDN; the values corresponding to those keys are routing information for said servers :cacheGroup: A string that is the :ref:`cache-group-name` of the :term:`Cache Group` to which the server belongs - :capabilities: An array of this Cache Group's :term:`Server Capabilities`. If the Cache Group has no Server Capabilities, this field is omitted. + :capabilities: An array of this :ref:`Cache Server`'s :term:`Server Capabilities`. If the Cache Server has no Server Capabilities, this field is omitted. :deliveryServices: An object containing keys which are the names of :term:`Delivery Services` to which this :term:`cache server` is assigned; the values corresponding to those keys are arrays of :abbr:`FQDNs (Fully Qualified Domain Names)` that resolve to this :term:`cache server` .. note:: Only :term:`Edge-tier cache servers` can be assigned to a :term:`Delivery Service`, and therefore this field will only be present when ``type`` is ``"EDGE"``. diff --git a/docs/source/api/v3/cdns_name_snapshot_new.rst b/docs/source/api/v3/cdns_name_snapshot_new.rst index e251b08c10..0e3644e0f8 100644 --- a/docs/source/api/v3/cdns_name_snapshot_new.rst +++ b/docs/source/api/v3/cdns_name_snapshot_new.rst @@ -119,7 +119,7 @@ Response Structure :contentServers: An object containing keys which are the (short) hostnames of the :term:`Edge-tier cache servers` in the CDN; the values corresponding to those keys are routing information for said servers :cacheGroup: A string that is the :ref:`cache-group-name` of the :term:`Cache Group` to which the server belongs - :capabilities: An array of this Cache Group's :term:`Server Capabilities`. If the Cache Group has no Server Capabilities, this field is omitted. + :capabilities: An array of this :ref:`Cache Server`'s :term:`Server Capabilities`. If the Cache Server has no Server Capabilities, this field is omitted. :deliveryServices: An object containing keys which are the names of :term:`Delivery Services` to which this :term:`cache server` is assigned; the values corresponding to those keys are arrays of :abbr:`FQDNs (Fully Qualified Domain Names)` that resolve to this :term:`cache server` .. note:: Only :term:`Edge-tier cache servers` can be assigned to a :term:`Delivery Service`, and therefore this field will only be present when ``type`` is ``"EDGE"``. diff --git a/docs/source/glossary.rst b/docs/source/glossary.rst index ce6d748fb5..f330d7ad63 100644 --- a/docs/source/glossary.rst +++ b/docs/source/glossary.rst @@ -28,6 +28,7 @@ Glossary astats (stats_over_http) An :abbr:`ATS (Apache Traffic Server)` plugin that allows you to monitor vitals of the :abbr:`ATS (Apache Traffic Server)` server. See :ref:`astats`. + Cache Server cache server cache servers The main function of a CDN is to proxy requests from clients to :term:`origin servers` and cache the results. To proxy, in the CDN context, is to obtain content using HTTP from an :term:`origin server` on behalf of a client. To cache is to store the results so they can be reused when other clients are requesting the same content. There are three types of proxies in use on the Internet today: From e98c07af463c49a99a185957b22cd032b01f9efd Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Mon, 8 Jun 2020 06:18:53 -0600 Subject: [PATCH 16/30] Different server capability --- docs/source/api/v3/cdns_name_snapshot.rst | 6 +++--- docs/source/api/v3/cdns_name_snapshot_new.rst | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/source/api/v3/cdns_name_snapshot.rst b/docs/source/api/v3/cdns_name_snapshot.rst index 5452f93adf..a2bdd01ff2 100644 --- a/docs/source/api/v3/cdns_name_snapshot.rst +++ b/docs/source/api/v3/cdns_name_snapshot.rst @@ -413,7 +413,7 @@ Response Structure "edge": { "cacheGroup": "CDN_in_a_Box_Edge", "capabilities": [ - "heat-vision" + "RAM_DISK_STORAGE" ], "fqdn": "edge.infra.ciab.test", "hashCount": 999, @@ -432,7 +432,7 @@ Response Structure "mid": { "cacheGroup": "CDN_in_a_Box_Mid", "capabilities": [ - "heat-vision" + "RAM_DISK_STORAGE" ], "fqdn": "mid.infra.ciab.test", "hashCount": 999, @@ -491,7 +491,7 @@ Response Structure }, "regionalGeoBlocking": "false", "requiredCapabilities": [ - "heat-vision" + "RAM_DISK_STORAGE" ], "routingName": "video", "soa": { diff --git a/docs/source/api/v3/cdns_name_snapshot_new.rst b/docs/source/api/v3/cdns_name_snapshot_new.rst index 0e3644e0f8..049cbde585 100644 --- a/docs/source/api/v3/cdns_name_snapshot_new.rst +++ b/docs/source/api/v3/cdns_name_snapshot_new.rst @@ -399,7 +399,7 @@ Response Structure "edge": { "cacheGroup": "CDN_in_a_Box_Edge", "capabilities": [ - "heat-vision" + "RAM_DISK_STORAGE" ], "fqdn": "edge.infra.ciab.test", "hashCount": 999, @@ -418,7 +418,7 @@ Response Structure "mid": { "cacheGroup": "CDN_in_a_Box_Mid", "capabilities": [ - "heat-vision" + "RAM_DISK_STORAGE" ], "fqdn": "mid.infra.ciab.test", "hashCount": 999, @@ -491,7 +491,7 @@ Response Structure }, "regionalGeoBlocking": "false", "requiredCapabilities": [ - "heat-vision" + "RAM_DISK_STORAGE" ], "routingName": "video", "soa": { From a5cb26926f9efbf035c2c97665ed79a5d7c7df8a Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Wed, 10 Jun 2020 14:01:00 -0600 Subject: [PATCH 17/30] Fix imports and class members that broke in the rebase --- .../traffic_control/traffic_router/core/edge/Node.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/edge/Node.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/edge/Node.java index c59366a01a..b40c1a9619 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/edge/Node.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/edge/Node.java @@ -46,10 +46,8 @@ public enum IpVersions { private List ipAddresses; private List unavailableIpAddresses; private int port; - private final Map deliveryServices = new HashMap(); + private final Map deliveryServices = new HashMap<>(); private final Set capabilities = new HashSet<>(); - private final Geolocation geolocation; - private final Hashable hashable = new DefaultHashable(); private int httpsPort = 443; public Node(final String id) { @@ -136,8 +134,8 @@ public Set getCapabilities() { return this.capabilities; } - public void setDeliveryServices(final Collection deliveryServices) { - for (final DeliveryServiceReference deliveryServiceReference : deliveryServices) { + public void setDeliveryServices(final Collection deliveryServices) { + for (final Cache.DeliveryServiceReference deliveryServiceReference : deliveryServices) { this.deliveryServices.put(deliveryServiceReference.getDeliveryServiceId(), deliveryServiceReference); } } From c6b7ed474da0c9c37675a9b358fd93cb49282a75 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Mon, 15 Jun 2020 13:06:33 -0600 Subject: [PATCH 18/30] Generate delivery service server FQDN for DeliveryServiceReference --- .../traffic_router/core/config/ConfigHandler.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java index 4924bfabab..766da1cb89 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java @@ -491,18 +491,19 @@ private void parseTopologyConfig(final JsonNode allTopologies, final Map { final List dsReferences = new ArrayList<>(); - try { - dsReferences.add(new DeliveryServiceReference(ds.getId(), ds.getDomain())); - } catch (ParseException e) { - LOGGER.error("Unable to create a DeliveryServiceReference from DeliveryService '" + ds.getId() + "'", e); - return; - } Stream.of(ds.getTopology()) .filter((topologyName) -> !Objects.isNull(topologyName) && topologyMap.containsKey(topologyName)) .flatMap((topologyName) -> topologyMap.get(topologyName).stream()) .flatMap((node) -> cacheRegister.getCacheLocation(node).getCaches().stream()) .filter((cache) -> ds.hasRequiredCapabilities(cache.getCapabilities())) - .forEach((cache) -> cache.setDeliveryServices(dsReferences)); + .forEach((cache) -> { + cache.setDeliveryServices(dsReferences); + try { + dsReferences.add(new DeliveryServiceReference(ds.getId(), cache.getId() + "." + ds.getDomain())); + } catch (ParseException e) { + LOGGER.error("Unable to create a DeliveryServiceReference from DeliveryService '" + ds.getId() + "'", e); + } + }); }); } From c63191d340c7238c6858c7b1201850f533111c53 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Mon, 15 Jun 2020 13:39:57 -0600 Subject: [PATCH 19/30] Add DeliveryServiceReferences to the Cache --- .../traffic_router/core/config/ConfigHandler.java | 1 + 1 file changed, 1 insertion(+) diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java index 766da1cb89..ec80df0d95 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java @@ -500,6 +500,7 @@ private void parseTopologyConfig(final JsonNode allTopologies, final Map Date: Mon, 15 Jun 2020 13:46:44 -0600 Subject: [PATCH 20/30] Fix ConfigHandlerTest imports --- .../traffic_router/core/config/ConfigHandlerTest.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandlerTest.java b/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandlerTest.java index 2c53244f9c..98e22212e0 100644 --- a/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandlerTest.java +++ b/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandlerTest.java @@ -22,8 +22,8 @@ import java.util.Map; import java.util.Set; -import com.comcast.cdn.traffic_control.traffic_router.core.cache.Cache; -import com.comcast.cdn.traffic_control.traffic_router.core.cache.CacheLocation; +import com.comcast.cdn.traffic_control.traffic_router.core.edge.Cache; +import com.comcast.cdn.traffic_control.traffic_router.core.edge.CacheLocation; import com.comcast.cdn.traffic_control.traffic_router.geolocation.Geolocation; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -34,7 +34,6 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; -import static org.mockito.Matchers.anyBoolean; import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; From 705bca4fc8d432b8f4bd12c543296d115b1f27bd Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Tue, 16 Jun 2020 12:37:52 -0600 Subject: [PATCH 21/30] Remove redundant parentheses --- .../traffic_router/core/config/ConfigHandler.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java index ec80df0d95..dcfc199d1b 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java @@ -492,11 +492,11 @@ private void parseTopologyConfig(final JsonNode allTopologies, final Map { final List dsReferences = new ArrayList<>(); Stream.of(ds.getTopology()) - .filter((topologyName) -> !Objects.isNull(topologyName) && topologyMap.containsKey(topologyName)) - .flatMap((topologyName) -> topologyMap.get(topologyName).stream()) - .flatMap((node) -> cacheRegister.getCacheLocation(node).getCaches().stream()) - .filter((cache) -> ds.hasRequiredCapabilities(cache.getCapabilities())) - .forEach((cache) -> { + .filter(topologyName -> !Objects.isNull(topologyName) && topologyMap.containsKey(topologyName)) + .flatMap(topologyName -> topologyMap.get(topologyName).stream()) + .flatMap(node -> cacheRegister.getCacheLocation(node).getCaches().stream()) + .filter(cache -> ds.hasRequiredCapabilities(cache.getCapabilities())) + .forEach(cache -> { cache.setDeliveryServices(dsReferences); try { dsReferences.add(new DeliveryServiceReference(ds.getId(), cache.getId() + "." + ds.getDomain())); From af6c35ecddbc2fd7fdbbc08345a72f7dbb0730da Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Thu, 18 Jun 2020 14:38:54 -0600 Subject: [PATCH 22/30] Associate delivery service URLs with the correct delivery service for Traffic Router stat tracking --- .../core/config/ConfigHandler.java | 51 ++++++++++++------- .../core/ds/DeliveryService.java | 13 +++++ .../core/ds/DeliveryServiceMatcher.java | 4 ++ .../core/edge/CacheRegister.java | 7 +++ .../core/request/RequestMatcher.java | 8 +++ 5 files changed, 65 insertions(+), 18 deletions(-) diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java index dcfc199d1b..42eedd6011 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java @@ -416,17 +416,9 @@ private void parseCacheConfig(final JsonNode contentServers, final CacheRegister } final String tld = JsonUtils.optString(cacheRegister.getConfig(), "domain_name"); - - if (name.endsWith(tld)) { - final String reName = name.replaceAll("^.*?\\.", ""); - - if (!dsNames.contains(reName)) { - dsNames.add(reName); - } - } else { - if (!dsNames.contains(name)) { - dsNames.add(name); - } + final String dsName = getDsName( name, tld); + if (!dsNames.contains(dsName)) { + dsNames.add(dsName); } i++; @@ -481,8 +473,16 @@ private Map parseDeliveryServiceConfig(final JsonNode a return deliveryServiceMap; } + private String getDsName(final String name, final String tld) { + return name.endsWith(tld) + ? name.replaceAll("^.*?\\.", "") + : name; + } + private void parseTopologyConfig(final JsonNode allTopologies, final Map deliveryServiceMap, final CacheRegister cacheRegister) { final Map> topologyMap = new HashMap<>(); + final Map> statMap = new HashMap<>(); + final String tld = JsonUtils.optString(cacheRegister.getConfig(), "domain_name"); allTopologies.fieldNames().forEachRemaining((String topologyName) -> { final List nodes = new ArrayList<>(); allTopologies.get(topologyName).get("nodes").forEach((JsonNode cache) -> nodes.add(cache.textValue())); @@ -491,21 +491,36 @@ private void parseTopologyConfig(final JsonNode allTopologies, final Map { final List dsReferences = new ArrayList<>(); + final List dsNames = new ArrayList<>(); // for stats Stream.of(ds.getTopology()) .filter(topologyName -> !Objects.isNull(topologyName) && topologyMap.containsKey(topologyName)) - .flatMap(topologyName -> topologyMap.get(topologyName).stream()) + .flatMap(topologyName -> { + statMap.put(ds.getId(), dsNames); + return topologyMap.get(topologyName).stream(); + }) .flatMap(node -> cacheRegister.getCacheLocation(node).getCaches().stream()) .filter(cache -> ds.hasRequiredCapabilities(cache.getCapabilities())) .forEach(cache -> { + cacheRegister.getDeliveryServiceMatchers(ds).stream() + .flatMap(deliveryServiceMatcher -> deliveryServiceMatcher.getRequestMatchers().stream()) + .map(requestMatcher -> requestMatcher.getPattern().pattern()) + .forEach(pattern -> { + final String remap = ds.getRemap(pattern); + final String fqdn = pattern.contains(".*") && !ds.isDns() + ? cache.getId() + "." + remap + : remap; + dsNames.add(getDsName(fqdn, tld)); + try { + dsReferences.add(new DeliveryServiceReference(ds.getId(), fqdn)); + } catch (ParseException e) { + LOGGER.error("Unable to create a DeliveryServiceReference from DeliveryService '" + ds.getId() + "'", e); + } + }); cache.setDeliveryServices(dsReferences); - try { - dsReferences.add(new DeliveryServiceReference(ds.getId(), cache.getId() + "." + ds.getDomain())); - cache.setDeliveryServices(dsReferences); - } catch (ParseException e) { - LOGGER.error("Unable to create a DeliveryServiceReference from DeliveryService '" + ds.getId() + "'", e); - } }); + }); + statTracker.initialize(statMap, cacheRegister); } private void parseDeliveryServiceMatchSets(final JsonNode allDeliveryServices, final Map deliveryServiceMap, final CacheRegister cacheRegister) throws JsonUtilsException { diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java index facbf6b082..23eccd8479 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java @@ -76,6 +76,8 @@ public class DeliveryService { @JsonIgnore private final String domain; @JsonIgnore + private final String tld; + @JsonIgnore private final JsonNode bypassDestination; @JsonIgnore private final JsonNode soa; @@ -131,6 +133,7 @@ public DeliveryService(final String id, final JsonNode dsJo) throws JsonUtilsExc this.bypassDestination = dsJo.get("bypassDestination"); this.routingName = JsonUtils.getString(dsJo, "routingName").toLowerCase(); this.domain = getDomainFromJson(dsJo.get("domains")); + this.tld = this.domain.replaceAll("^.*?\\.", ""); this.soa = dsJo.get("soa"); this.shouldAppendQueryString = JsonUtils.optBoolean(dsJo, "appendQueryString", true); this.ecsEnabled = JsonUtils.optBoolean(dsJo, "ecsEnabled"); @@ -378,6 +381,16 @@ public String createURIString(final HTTPRequest request, final String alternateP return uri.toString(); } + public String getRemap(final String pattern) { + if (!pattern.contains(".*")) { + return pattern; + } + final String host = pattern.replaceAll("^\\(\\.\\*\\\\\\.\\|\\^\\)|^\\.\\*\\\\\\.|\\\\\\.\\.\\*", "") + "." + tld; + return this.isDns() + ? this.getId() + "." + host + : host; + } + private String getFQDN(final Cache cache) { for (final DeliveryServiceReference dsRef : cache.getDeliveryServices()) { if (dsRef.getDeliveryServiceId().equals(this.getId())) { diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryServiceMatcher.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryServiceMatcher.java index 716fb5ae85..c0253ee489 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryServiceMatcher.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryServiceMatcher.java @@ -48,6 +48,10 @@ public void addMatch(final Type type, final String string, final String target) requestMatchers.add(new RequestMatcher(type, string, target)); } + public List getRequestMatchers() { + return new ArrayList<>(this.requestMatchers); + } + public boolean matches(final Request request) { for (final RequestMatcher matcher : requestMatchers) { if (!matcher.matches(request)) { diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/edge/CacheRegister.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/edge/CacheRegister.java index c802a84c9b..ab207e3a91 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/edge/CacheRegister.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/edge/CacheRegister.java @@ -16,6 +16,7 @@ package com.comcast.cdn.traffic_control.traffic_router.core.edge; import java.util.*; +import java.util.stream.Collectors; import com.fasterxml.jackson.databind.JsonNode; @@ -129,6 +130,12 @@ public Map getCacheMap() { return allCaches; } + public Set getDeliveryServiceMatchers(final DeliveryService deliveryService) { + return this.deliveryServiceMatchers.stream() + .filter(deliveryServiceMatcher -> deliveryServiceMatcher.getDeliveryService() == deliveryService) + .collect(Collectors.toCollection(TreeSet::new)); + } + public void setDeliveryServiceMatchers(final TreeSet matchers) { this.deliveryServiceMatchers = matchers; } diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/request/RequestMatcher.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/request/RequestMatcher.java index 14fca52eeb..d3bd57952e 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/request/RequestMatcher.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/request/RequestMatcher.java @@ -59,6 +59,14 @@ public boolean matches(final Request request) { return pattern.matcher(target).matches(); } + public Type getType() { + return type; + } + + public Pattern getPattern() { + return pattern; + } + private String getTarget(final Request request) { if (type == Type.HOST) { return request.getHostname(); From 2e509d420ca4615a494d810e6317bc5704256842 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Thu, 18 Jun 2020 16:25:40 -0600 Subject: [PATCH 23/30] Do not create DeliveryServiceReferences for aliases --- .../traffic_router/core/config/ConfigHandler.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java index 42eedd6011..092f3d2297 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java @@ -510,6 +510,9 @@ private void parseTopologyConfig(final JsonNode allTopologies, final Map Date: Thu, 18 Jun 2020 18:39:59 -0600 Subject: [PATCH 24/30] Prefix routing name for DNS DS, not XML ID --- .../traffic_control/traffic_router/core/ds/DeliveryService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java index 23eccd8479..fb8e662219 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java @@ -387,7 +387,7 @@ public String getRemap(final String pattern) { } final String host = pattern.replaceAll("^\\(\\.\\*\\\\\\.\\|\\^\\)|^\\.\\*\\\\\\.|\\\\\\.\\.\\*", "") + "." + tld; return this.isDns() - ? this.getId() + "." + host + ? this.routingName + "." + host : host; } From 18f2f3a15caeaaf444c04ef09cdd3b158cb73831 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Thu, 18 Jun 2020 19:30:48 -0600 Subject: [PATCH 25/30] Correct alias-detecting condition for DNS delivery services --- .../traffic_router/core/config/ConfigHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java index 092f3d2297..d1a213608d 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java @@ -510,7 +510,7 @@ private void parseTopologyConfig(final JsonNode allTopologies, final Map Date: Thu, 18 Jun 2020 19:34:32 -0600 Subject: [PATCH 26/30] Precompile wildcard-matching pattern for reuse --- .../traffic_router/core/ds/DeliveryService.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java index fb8e662219..186020cce9 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java @@ -36,6 +36,7 @@ import java.util.TreeSet; import java.util.Iterator; import java.util.concurrent.atomic.AtomicInteger; +import java.util.regex.Pattern; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.JsonNode; @@ -78,6 +79,8 @@ public class DeliveryService { @JsonIgnore private final String tld; @JsonIgnore + private final Pattern wildcardPattern = Pattern.compile("^\\(\\.\\*\\\\\\.\\|\\^\\)|^\\.\\*\\\\\\.|\\\\\\.\\.\\*"); + @JsonIgnore private final JsonNode bypassDestination; @JsonIgnore private final JsonNode soa; @@ -381,11 +384,11 @@ public String createURIString(final HTTPRequest request, final String alternateP return uri.toString(); } - public String getRemap(final String pattern) { - if (!pattern.contains(".*")) { - return pattern; + public String getRemap(final String dsPattern) { + if (!dsPattern.contains(".*")) { + return dsPattern; } - final String host = pattern.replaceAll("^\\(\\.\\*\\\\\\.\\|\\^\\)|^\\.\\*\\\\\\.|\\\\\\.\\.\\*", "") + "." + tld; + final String host = wildcardPattern.matcher(dsPattern).replaceAll("") + "." + tld; return this.isDns() ? this.routingName + "." + host : host; From 1f9d31e44440e3acc4602636255fce40fc03ecd6 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Tue, 23 Jun 2020 16:11:10 -0600 Subject: [PATCH 27/30] Filter DeliveryServiceMatchers by the DS's xmlId --- .../traffic_control/traffic_router/core/edge/CacheRegister.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/edge/CacheRegister.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/edge/CacheRegister.java index ab207e3a91..3e243a4369 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/edge/CacheRegister.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/edge/CacheRegister.java @@ -132,7 +132,7 @@ public Map getCacheMap() { public Set getDeliveryServiceMatchers(final DeliveryService deliveryService) { return this.deliveryServiceMatchers.stream() - .filter(deliveryServiceMatcher -> deliveryServiceMatcher.getDeliveryService() == deliveryService) + .filter(deliveryServiceMatcher -> deliveryServiceMatcher.getDeliveryService().getId().equals(deliveryService.getId())) .collect(Collectors.toCollection(TreeSet::new)); } From 067427f9752431c353b8cf9b7013310e5a5de8a5 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Tue, 23 Jun 2020 16:11:43 -0600 Subject: [PATCH 28/30] Include a more readable version of the regex and explain it --- .../traffic_control/traffic_router/core/ds/DeliveryService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java index 186020cce9..fcce2b6e73 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java @@ -79,6 +79,8 @@ public class DeliveryService { @JsonIgnore private final String tld; @JsonIgnore + // Matches the beginning of a HOST_REGEXP pattern with or without confighandler.regex.superhack.enabled. + // ^\(\.\*\\\.\|\^\)|^\.\*\\\.|\\\.\.\* private final Pattern wildcardPattern = Pattern.compile("^\\(\\.\\*\\\\\\.\\|\\^\\)|^\\.\\*\\\\\\.|\\\\\\.\\.\\*"); @JsonIgnore private final JsonNode bypassDestination; From 68c2181c0a91f03a92dac7412edce937ff61fb52 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Tue, 23 Jun 2020 19:31:24 -0600 Subject: [PATCH 29/30] Extend mocks for ConfigHandlerTest.itParsesTheTopologiesConfig() --- .../core/config/ConfigHandlerTest.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandlerTest.java b/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandlerTest.java index 98e22212e0..bdb9c9947a 100644 --- a/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandlerTest.java +++ b/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandlerTest.java @@ -21,9 +21,12 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.TreeSet; +import com.comcast.cdn.traffic_control.traffic_router.core.ds.DeliveryServiceMatcher; import com.comcast.cdn.traffic_control.traffic_router.core.edge.Cache; import com.comcast.cdn.traffic_control.traffic_router.core.edge.CacheLocation; +import com.comcast.cdn.traffic_control.traffic_router.core.router.StatTracker; import com.comcast.cdn.traffic_control.traffic_router.geolocation.Geolocation; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -272,19 +275,32 @@ public void itParsesTheTopologiesConfig() throws Exception { /* Mock a DeliveryService and add it to our DeliveryService Map */ final String dsId = "top-ds"; - final String domain = "ds.cdn.com"; + final String routingName = "cdn"; + final String domain = "ds.site.com"; final String topology = "foo"; + final String superHackedRegexp = "(.*\\.|^)" + dsId + "\\..*"; final DeliveryService ds = mock(DeliveryService.class); when(ds.getId()).thenReturn(dsId); when(ds.getDomain()).thenReturn(domain); + when(ds.getRemap(superHackedRegexp)).thenReturn(domain); + when(ds.getRoutingName()).thenReturn(routingName); when(ds.getTopology()).thenReturn(topology); when(ds.hasRequiredCapabilities(capabilities)).thenReturn(true); + when(ds.isDns()).thenReturn(false); final Map dsMap = new HashMap<>(); dsMap.put(dsId, ds); + final DeliveryServiceMatcher dsMatcher = new DeliveryServiceMatcher(ds); + dsMatcher.addMatch(DeliveryServiceMatcher.Type.HOST, superHackedRegexp, ""); + final TreeSet dsMatchers = new TreeSet<>(); + dsMatchers.add(dsMatcher); + register.setDeliveryServiceMap(dsMap); + register.setDeliveryServiceMatchers(dsMatchers); + /* Parse the Topologies config JSON */ final ObjectMapper mapper = new ObjectMapper(); final JsonNode allTopologiesJson = mapper.readTree("{\"" + topology + "\":{\"nodes\":[\"" + location + "\"]}}"); + Whitebox.setInternalState(handler, "statTracker", new StatTracker()); Whitebox.invokeMethod(handler, "parseTopologyConfig", allTopologiesJson, dsMap, register); /* Assert that the DeliveryService was assigned to the Cache */ From e162d16bd09c2f73d5c9a9b280b24a3641ef50eb Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Wed, 24 Jun 2020 10:08:39 -0600 Subject: [PATCH 30/30] Only initialize `tld` if `domain` is non-null --- .../traffic_router/core/ds/DeliveryService.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java index fcce2b6e73..dc155fa004 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java @@ -138,7 +138,9 @@ public DeliveryService(final String id, final JsonNode dsJo) throws JsonUtilsExc this.bypassDestination = dsJo.get("bypassDestination"); this.routingName = JsonUtils.getString(dsJo, "routingName").toLowerCase(); this.domain = getDomainFromJson(dsJo.get("domains")); - this.tld = this.domain.replaceAll("^.*?\\.", ""); + this.tld = this.domain != null + ? this.domain.replaceAll("^.*?\\.", "") + : null; this.soa = dsJo.get("soa"); this.shouldAppendQueryString = JsonUtils.optBoolean(dsJo, "appendQueryString", true); this.ecsEnabled = JsonUtils.optBoolean(dsJo, "ecsEnabled");