diff --git a/CHANGELOG.md b/CHANGELOG.md index e052f082fd..e8b2afc7e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Traffic Portal: Added the ability to create, read, update and delete flexible topologies. - Traffic Portal: Added the ability to assign topologies to delivery services. - Traffic Portal: Added the ability to view all delivery services and cache groups associated with a topology. + - Traffic Router: Added support for topology-based delivery services - Updated /servers/details to use multiple interfaces in API v3 - Added [Edge Traffic Routing](https://traffic-control-cdn.readthedocs.io/en/latest/admin/traffic_router.html#edge-traffic-routing) feature which allows Traffic Router to localize more DNS record types than just the routing name for DNS delivery services - Astats csv support - astats will now respond to `Accept: text/csv` and return a csv formatted stats list diff --git a/docs/source/api/v3/cdns_name_snapshot.rst b/docs/source/api/v3/cdns_name_snapshot.rst index 942f32d451..a2bdd01ff2 100644 --- a/docs/source/api/v3/cdns_name_snapshot.rst +++ b/docs/source/api/v3/cdns_name_snapshot.rst @@ -42,9 +42,10 @@ Request Structure :caption: Request Example GET /api/3.0/cdns/CDN-in-a-Box/snapshot HTTP/1.1 - Host: trafficops.infra.ciab.test - User-Agent: curl/7.47.0 + User-Agent: python-requests/2.23.0 + Accept-Encoding: gzip, deflate Accept: */* + Connection: keep-alive Cookie: mojolicious=... Response Structure @@ -119,6 +120,7 @@ Response Structure :contentServers: An object containing keys which are the (short) hostnames of the :term:`Edge-tier cache servers` in the CDN; the values corresponding to those keys are routing information for said servers :cacheGroup: A string that is the :ref:`cache-group-name` of the :term:`Cache Group` to which the server belongs + :capabilities: An array of this :ref:`Cache Server`'s :term:`Server Capabilities`. If the Cache Server has no Server Capabilities, this field is omitted. :deliveryServices: An object containing keys which are the names of :term:`Delivery Services` to which this :term:`cache server` is assigned; the values corresponding to those keys are arrays of :abbr:`FQDNs (Fully Qualified Domain Names)` that resolve to this :term:`cache server` .. note:: Only :term:`Edge-tier cache servers` can be assigned to a :term:`Delivery Service`, and therefore this field will only be present when ``type`` is ``"EDGE"``. @@ -246,7 +248,7 @@ Response Structure Regional Geographic Blocking is used by this :term:`Delivery Service` .. seealso:: :ref:`regionalgeo-qht` - + :requiredCapabilities: An array of this Delivery Service's :term:`required capabilities `. If there are no required capabilities, this field is omitted. :routingName: A string that is this :ref:`Delivery Service's Routing Name ` :soa: An object defining the :abbr:`SOA (Start of Authority)` record for the :term:`Delivery Service`'s :abbr:`TLDs (Top-Level Domains)` (defined in ``domains``) @@ -272,6 +274,7 @@ Response Structure .. seealso:: :ref:`ds-protocol` + :topology: The name of the :term:`Topology` that this :term:`Delivery Service` is assigned to. If the Delivery Service is not assigned to a topology, this field is omitted. :ttls: An object that contains keys which are types of DNS records that have values which are strings containing integers that specify the time for which a response to the specific type of record request should remain valid .. note:: This overrides ``config.ttls``. @@ -318,6 +321,10 @@ Response Structure :tm_user: The username of the currently logged-in user :tm_version: The full version number of the Traffic Ops server, including release number, git commit hash, and supported Enterprise Linux version +:topologies: An array of :term:`Topologies` where each key is the name of that Topology. + + :nodes: An array of the names of the :term:`Edge-Tier` :term:`Cache Groups` in this :term:`Topology`. :term:`Mid-Tier` Cache Groups in the topology are not included. + :trafficRouterLocations: An object containing keys which are the :ref:`names of Cache Groups ` within the CDN which contain Traffic Routers :backupLocations: An object that describes this :ref:`Cache Group's Fallbacks ` @@ -341,15 +348,19 @@ Response Structure Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept, Set-Cookie, Cookie Access-Control-Allow-Methods: POST,GET,OPTIONS,PUT,DELETE Access-Control-Allow-Origin: * + Content-Encoding: gzip Content-Type: application/json - Set-Cookie: mojolicious=...; Path=/; Expires=Mon, 18 Nov 2019 17:40:54 GMT; Max-Age=3600; HttpOnly - Whole-Content-Sha512: 220bc4XXwaj+s7ODd3QAF5leGj06lnApiN5E8H/B2RgxSphnQIfnwy6WWbBDjonWXPV1IWDCjBMO+rR+lAabMg== + Set-Cookie: mojolicious=...; Path=/; Expires=Wed, 27 May 2020 18:33:17 GMT; Max-Age=3600; HttpOnly + Vary: Accept-Encoding + Whole-Content-Sha512: B5qdN9URIfu11gQxPZ8YaaMvy2HMrzsnrpt6vF037yv6OQiKCRyrUMX6wYs7QW4YVaeUrvmS2ya5l2YC0kvNAg== X-Server-Name: traffic_ops_golang/ - Date: Wed, 12 Dec 2018 17:36:25 GMT - Transfer-Encoding: chunked + Date: Wed, 27 May 2020 17:33:17 GMT + Content-Length: 1360 + - { "response": { - "config": { + { + "response": { + "config": { "api.cache-control.max-age": "10", "certificates.polling.interval": "300000", "consistent.dns.routing": "true", @@ -359,184 +370,204 @@ Response Structure "dnssec.enabled": "false", "domain_name": "mycdn.ciab.test", "federationmapping.polling.interval": "60000", - "federationmapping.polling.url": "https://${toHostname}/api/3.0/federations", + "federationmapping.polling.url": "https://${toHostname}/api/2.0/federations/all", "geolocation.polling.interval": "86400000", "geolocation.polling.url": "https://trafficops.infra.ciab.test:443/GeoLite2-City.mmdb.gz", "keystore.maintenance.interval": "300", "neustar.polling.interval": "86400000", "neustar.polling.url": "https://trafficops.infra.ciab.test:443/neustar.tar.gz", "soa": { - "admin": "twelve_monkeys", - "expire": "604800", - "minimum": "30", - "refresh": "28800", - "retry": "7200" + "admin": "twelve_monkeys", + "expire": "604800", + "minimum": "30", + "refresh": "28800", + "retry": "7200" }, "steeringmapping.polling.interval": "60000", "ttls": { - "A": "3600", - "AAAA": "3600", - "DNSKEY": "30", - "DS": "30", - "NS": "3600", - "SOA": "86400" + "A": "3600", + "AAAA": "3600", + "DNSKEY": "30", + "DS": "30", + "NS": "3600", + "SOA": "86400" }, "zonemanager.cache.maintenance.interval": "300", "zonemanager.threadpool.scale": "0.50" - }, - "contentServers": { + }, + "contentRouters": { + "trafficrouter": { + "api.port": "3333", + "fqdn": "trafficrouter.infra.ciab.test", + "httpsPort": 443, + "ip": "172.26.0.15", + "ip6": "", + "location": "CDN_in_a_Box_Edge", + "port": 80, + "profile": "CCR_CIAB", + "secure.api.port": "3443", + "status": "ONLINE" + } + }, + "contentServers": { "edge": { - "cacheGroup": "CDN_in_a_Box_Edge", - "fqdn": "edge.infra.ciab.test", - "hashCount": 999, - "hashId": "edge", - "httpsPort": 443, - "interfaceName": "eth0", - "ip": "172.16.239.100", - "ip6": "fc01:9400:1000:8::100", - "locationId": "CDN_in_a_Box_Edge", - "port": 80, - "profile": "ATS_EDGE_TIER_CACHE", - "status": "REPORTED", - "type": "EDGE", - "deliveryServices": { - "demo1": [ - "edge.demo1.mycdn.ciab.test" - ] - }, - "routingDisabled": 0 + "cacheGroup": "CDN_in_a_Box_Edge", + "capabilities": [ + "RAM_DISK_STORAGE" + ], + "fqdn": "edge.infra.ciab.test", + "hashCount": 999, + "hashId": "edge", + "httpsPort": 443, + "interfaceName": "eth0", + "ip": "172.26.0.3", + "ip6": "", + "locationId": "CDN_in_a_Box_Edge", + "port": 80, + "profile": "ATS_EDGE_TIER_CACHE", + "routingDisabled": 0, + "status": "REPORTED", + "type": "EDGE" }, "mid": { - "cacheGroup": "CDN_in_a_Box_Mid", - "fqdn": "mid.infra.ciab.test", - "hashCount": 999, - "hashId": "mid", - "httpsPort": 443, - "interfaceName": "eth0", - "ip": "172.16.239.120", - "ip6": "fc01:9400:1000:8::120", - "locationId": "CDN_in_a_Box_Mid", - "port": 80, - "profile": "ATS_MID_TIER_CACHE", - "status": "REPORTED", - "type": "MID", - "routingDisabled": 0 - } - }, - "contentRouters": { - "trafficrouter": { - "api.port": "3333", - "secure.api.port": "3443", - "fqdn": "trafficrouter.infra.ciab.test", - "httpsPort": 443, - "ip": "172.16.239.60", - "ip6": "fc01:9400:1000:8::60", - "location": "CDN_in_a_Box_Edge", - "port": 80, - "profile": "CCR_CIAB", - "status": "ONLINE" + "cacheGroup": "CDN_in_a_Box_Mid", + "capabilities": [ + "RAM_DISK_STORAGE" + ], + "fqdn": "mid.infra.ciab.test", + "hashCount": 999, + "hashId": "mid", + "httpsPort": 443, + "interfaceName": "eth0", + "ip": "172.26.0.4", + "ip6": "", + "locationId": "CDN_in_a_Box_Mid", + "port": 80, + "profile": "ATS_MID_TIER_CACHE", + "routingDisabled": 0, + "status": "REPORTED", + "type": "MID" } - }, - "deliveryServices": { + }, + "deliveryServices": { "demo1": { - "anonymousBlockingEnabled": "false", - "coverageZoneOnly": "false", - "dispersion": { - "limit": 1, - "shuffled": "true" - }, - "domains": [ - "demo1.mycdn.ciab.test" - ], - "geolocationProvider": "maxmindGeolocationService", - "matchsets": [ + "anonymousBlockingEnabled": "false", + "consistentHashQueryParams": [ + "abc", + "pdq", + "xxx", + "zyx" + ], + "coverageZoneOnly": "false", + "deepCachingType": "NEVER", + "dispersion": { + "limit": 1, + "shuffled": "true" + }, + "domains": [ + "demo1.mycdn.ciab.test" + ], + "ecsEnabled": "false", + "geolocationProvider": "maxmindGeolocationService", + "ip6RoutingEnabled": "true", + "matchsets": [ + { + "matchlist": [ { - "protocol": "HTTP", - "matchlist": [ - { - "regex": ".*\\.demo1\\..*", - "match-type": "HOST" - } - ] + "match-type": "HOST", + "regex": ".*\\.demo1\\..*" } - ], - "missLocation": { - "lat": 42, - "long": -88 - }, - "protocol": { - "acceptHttps": "false", - "redirectToHttps": "false" - }, - "regionalGeoBlocking": "false", - "soa": { - "admin": "traffic_ops", - "expire": "604800", - "minimum": "30", - "refresh": "28800", - "retry": "7200" - }, - "sslEnabled": "false", - "ttls": { - "A": "", - "AAAA": "", - "NS": "3600", - "SOA": "86400" - }, - "ip6RoutingEnabled": "true", - "ecsEnabled": "false", - "routingName": "video", - "deepCachingType": "NEVER" - } - }, - "edgeLocations": { - "CDN_in_a_Box_Edge": { - "latitude": 38.897663, - "longitude": -77.036574, - "backupLocations": { - "fallbackToClosest": "true" - }, - "localizationMethods": [ - "GEO", - "CZ", - "DEEP_CZ" - ] + ], + "protocol": "HTTP" + } + ], + "missLocation": { + "lat": 42, + "long": -88 + }, + "protocol": { + "acceptHttps": "true", + "redirectToHttps": "false" + }, + "regionalGeoBlocking": "false", + "requiredCapabilities": [ + "RAM_DISK_STORAGE" + ], + "routingName": "video", + "soa": { + "admin": "traffic_ops", + "expire": "604800", + "minimum": "30", + "refresh": "28800", + "retry": "7200" + }, + "sslEnabled": "true", + "topology": "my-topology", + "ttls": { + "A": "", + "AAAA": "", + "NS": "3600", + "SOA": "86400" + } } - }, - "trafficRouterLocations": { + }, + "edgeLocations": { "CDN_in_a_Box_Edge": { - "latitude": 38.897663, - "longitude": -77.036574, - "backupLocations": { - "fallbackToClosest": "false" - }, - "localizationMethods": [ - "GEO", - "CZ", - "DEEP_CZ" - ] + "backupLocations": { + "fallbackToClosest": "true" + }, + "latitude": 38.897663, + "localizationMethods": [ + "GEO", + "CZ", + "DEEP_CZ" + ], + "longitude": -77.036574 } - }, - "monitors": { + }, + "monitors": { "trafficmonitor": { - "fqdn": "trafficmonitor.infra.ciab.test", - "httpsPort": 443, - "ip": "172.16.239.40", - "ip6": "fc01:9400:1000:8::40", - "location": "CDN_in_a_Box_Edge", - "port": 80, - "profile": "RASCAL-Traffic_Monitor", - "status": "ONLINE" + "fqdn": "trafficmonitor.infra.ciab.test", + "httpsPort": 443, + "ip": "172.26.0.14", + "ip6": "", + "location": "CDN_in_a_Box_Edge", + "port": 80, + "profile": "RASCAL-Traffic_Monitor", + "status": "ONLINE" } - }, - "stats": { + }, + "stats": { "CDN_name": "CDN-in-a-Box", - "date": 1544635937, - "tm_host": "trafficops.infra.ciab.test", - "tm_path": "/tools/write_crconfig/CDN-in-a-Box", + "date": 1590600715, + "tm_host": "trafficops.infra.ciab.test:443", + "tm_path": "/api/3.0/snapshot", "tm_user": "admin", - "tm_version": "traffic_ops-3.0.0-9813.8ad7bd8e.el7" + "tm_version": "development" + }, + "topologies": { + "my-topology": { + "nodes": [ + "CDN_in_a_Box_Edge" + ] + } + }, + "trafficRouterLocations": { + "CDN_in_a_Box_Edge": { + "backupLocations": { + "fallbackToClosest": "false" + }, + "latitude": 38.897663, + "localizationMethods": [ + "GEO", + "CZ", + "DEEP_CZ" + ], + "longitude": -77.036574 + } + } } - }} + } + .. [#httpOnly] These only apply to HTTP-:ref:`routed ` :term:`Delivery Services` diff --git a/docs/source/api/v3/cdns_name_snapshot_new.rst b/docs/source/api/v3/cdns_name_snapshot_new.rst index 0920421c44..049cbde585 100644 --- a/docs/source/api/v3/cdns_name_snapshot_new.rst +++ b/docs/source/api/v3/cdns_name_snapshot_new.rst @@ -41,9 +41,10 @@ Request Structure :caption: Request Example GET /api/3.0/cdns/CDN-in-a-Box/snapshot/new HTTP/1.1 - Host: trafficops.infra.ciab.test - User-Agent: curl/7.47.0 + User-Agent: python-requests/2.23.0 + Accept-Encoding: gzip, deflate Accept: */* + Connection: keep-alive Cookie: mojolicious=... Response Structure @@ -118,6 +119,7 @@ Response Structure :contentServers: An object containing keys which are the (short) hostnames of the :term:`Edge-tier cache servers` in the CDN; the values corresponding to those keys are routing information for said servers :cacheGroup: A string that is the :ref:`cache-group-name` of the :term:`Cache Group` to which the server belongs + :capabilities: An array of this :ref:`Cache Server`'s :term:`Server Capabilities`. If the Cache Server has no Server Capabilities, this field is omitted. :deliveryServices: An object containing keys which are the names of :term:`Delivery Services` to which this :term:`cache server` is assigned; the values corresponding to those keys are arrays of :abbr:`FQDNs (Fully Qualified Domain Names)` that resolve to this :term:`cache server` .. note:: Only :term:`Edge-tier cache servers` can be assigned to a :term:`Delivery Service`, and therefore this field will only be present when ``type`` is ``"EDGE"``. @@ -246,6 +248,7 @@ Response Structure .. seealso:: :ref:`regionalgeo-qht` + :requiredCapabilities: An array of this Delivery Service's :term:`required capabilities `. If there are no required capabilities, this field is omitted. :routingName: A string that is this :ref:`Delivery Service's Routing Name ` :soa: An object defining the :abbr:`SOA (Start of Authority)` record for the :term:`Delivery Service`'s :abbr:`TLDs (Top-Level Domains)` (defined in ``domains``) @@ -271,6 +274,7 @@ Response Structure .. seealso:: :ref:`ds-protocol` + :topology: The name of the :term:`Topology` that this :term:`Delivery Service` is assigned to. If the Delivery Service is not assigned to a topology, this field is omitted. :ttls: An object that contains keys which are types of DNS records that have values which are strings containing integers that specify the time for which a response to the specific type of record request should remain valid .. note:: This overrides ``config.ttls``. @@ -317,6 +321,10 @@ Response Structure :tm_user: The username of the currently logged-in user :tm_version: The full version number of the Traffic Ops server, including release number, git commit hash, and supported Enterprise Linux version +:topologies: An array of :term:`Topologies` where each key is the name of that Topology. + + :nodes: An array of the names of the :term:`Edge-Tier` :term:`Cache Groups` in this :term:`Topology`. :term:`Mid-Tier` Cache Groups in the topology are not included. + :trafficRouterLocations: An object containing keys which are the :ref:`names of Cache Groups ` within the CDN which contain Traffic Routers :backupLocations: An object that describes this :ref:`Cache Group's Fallbacks ` @@ -341,205 +349,224 @@ Response Structure Access-Control-Allow-Headers: Origin, X-Requested-With, Content-Type, Accept, Set-Cookie, Cookie Access-Control-Allow-Methods: POST,GET,OPTIONS,PUT,DELETE Access-Control-Allow-Origin: * + Content-Encoding: gzip Content-Type: application/json - Set-Cookie: mojolicious=...; Path=/; Expires=Mon, 18 Nov 2019 17:40:54 GMT; Max-Age=3600; HttpOnly - Whole-Content-Sha512: MWzgAYngmU1IEIxRa0C6VfY+MMuu7T9OCiIj1Aul58pA7J7DiS6r8wjVRVVW8W2Eu2V9BC7OEacR1fQyuIsRWg== + Set-Cookie: mojolicious=...; Path=/; Expires=Wed, 27 May 2020 20:31:13 GMT; Max-Age=3600; HttpOnly + Vary: Accept-Encoding + Whole-Content-Sha512: M6uhE2oPpjpTUR7gALsPOnM2CepD+VCAjp4dj5Xnppo0G5zL31PQgiteD23q67r7/bq/JJpMvIvdaENVYFtrqQ== X-Server-Name: traffic_ops_golang/ - Date: Wed, 12 Dec 2018 21:41:48 GMT - Transfer-Encoding: chunked - - { "response": { - "config": { - "api.cache-control.max-age": "10", - "certificates.polling.interval": "300000", - "consistent.dns.routing": "true", - "coveragezone.polling.interval": "3600000", - "coveragezone.polling.url": "https://trafficops.infra.ciab.test:443/coverage-zone.json", - "dnssec.dynamic.response.expiration": "300s", - "dnssec.enabled": "false", - "domain_name": "mycdn.ciab.test", - "federationmapping.polling.interval": "60000", - "federationmapping.polling.url": "https://${toHostname}/api/3.0/federations", - "geolocation.polling.interval": "86400000", - "geolocation.polling.url": "https://trafficops.infra.ciab.test:443/GeoLite2-City.mmdb.gz", - "keystore.maintenance.interval": "300", - "neustar.polling.interval": "86400000", - "neustar.polling.url": "https://trafficops.infra.ciab.test:443/neustar.tar.gz", - "soa": { - "admin": "twelve_monkeys", - "expire": "604800", - "minimum": "30", - "refresh": "28800", - "retry": "7200" - }, - "steeringmapping.polling.interval": "60000", - "ttls": { - "A": "3600", - "AAAA": "3600", - "DNSKEY": "30", - "DS": "30", - "NS": "3600", - "SOA": "86400" - }, - "zonemanager.cache.maintenance.interval": "300", - "zonemanager.threadpool.scale": "0.50" - }, - "contentServers": { - "edge": { - "cacheGroup": "CDN_in_a_Box_Edge", - "fqdn": "edge.infra.ciab.test", - "hashCount": 999, - "hashId": "edge", - "httpsPort": 443, - "interfaceName": "eth0", - "ip": "172.16.239.100", - "ip6": "fc01:9400:1000:8::100", - "locationId": "CDN_in_a_Box_Edge", - "port": 80, - "profile": "ATS_EDGE_TIER_CACHE", - "status": "REPORTED", - "type": "EDGE", - "deliveryServices": { - "demo1": [ - "edge.demo1.mycdn.ciab.test" - ] - }, - "routingDisabled": 0 - }, - "mid": { - "cacheGroup": "CDN_in_a_Box_Mid", - "fqdn": "mid.infra.ciab.test", - "hashCount": 999, - "hashId": "mid", - "httpsPort": 443, - "interfaceName": "eth0", - "ip": "172.16.239.120", - "ip6": "fc01:9400:1000:8::120", - "locationId": "CDN_in_a_Box_Mid", - "port": 80, - "profile": "ATS_MID_TIER_CACHE", - "status": "REPORTED", - "type": "MID", - "routingDisabled": 0 - } - }, - "contentRouters": { - "trafficrouter": { - "api.port": "3333", - "secure.api.port": "3443", - "fqdn": "trafficrouter.infra.ciab.test", - "httpsPort": 443, - "ip": "172.16.239.60", - "ip6": "fc01:9400:1000:8::60", - "location": "CDN_in_a_Box_Edge", - "port": 80, - "profile": "CCR_CIAB", - "status": "ONLINE" - } - }, - "deliveryServices": { - "demo1": { - "anonymousBlockingEnabled": "false", - "coverageZoneOnly": "false", - "dispersion": { - "limit": 1, - "shuffled": "true" - }, - "domains": [ - "demo1.mycdn.ciab.test" - ], - "geolocationProvider": "maxmindGeolocationService", - "matchsets": [ - { - "protocol": "HTTP", - "matchlist": [ - { - "regex": ".*\\.demo1\\..*", - "match-type": "HOST" - } - ] - } - ], - "missLocation": { - "lat": 42, - "long": -88 - }, - "protocol": { - "acceptHttps": "false", - "redirectToHttps": "false" - }, - "regionalGeoBlocking": "false", + Date: Wed, 27 May 2020 19:31:13 GMT + Content-Length: 1374 + + { + "response": { + "config": { + "api.cache-control.max-age": "10", + "certificates.polling.interval": "300000", + "consistent.dns.routing": "true", + "coveragezone.polling.interval": "3600000", + "coveragezone.polling.url": "https://trafficops.infra.ciab.test:443/coverage-zone.json", + "dnssec.dynamic.response.expiration": "300s", + "dnssec.enabled": "false", + "domain_name": "mycdn.ciab.test", + "federationmapping.polling.interval": "60000", + "federationmapping.polling.url": "https://${toHostname}/api/2.0/federations/all", + "geolocation.polling.interval": "86400000", + "geolocation.polling.url": "https://trafficops.infra.ciab.test:443/GeoLite2-City.mmdb.gz", + "keystore.maintenance.interval": "300", + "neustar.polling.interval": "86400000", + "neustar.polling.url": "https://trafficops.infra.ciab.test:443/neustar.tar.gz", "soa": { - "admin": "traffic_ops", + "admin": "twelve_monkeys", "expire": "604800", "minimum": "30", "refresh": "28800", "retry": "7200" }, - "sslEnabled": "false", + "steeringmapping.polling.interval": "60000", "ttls": { - "A": "", - "AAAA": "", + "A": "3600", + "AAAA": "3600", + "DNSKEY": "30", + "DS": "30", "NS": "3600", "SOA": "86400" }, - "ip6RoutingEnabled": "true", - "ecsEnabled": "false", - "routingName": "video", - "deepCachingType": "NEVER" - } - }, - "edgeLocations": { - "CDN_in_a_Box_Edge": { - "latitude": 38.897663, - "longitude": -77.036574, - "backupLocations": { - "fallbackToClosest": "true", - "list": [ - "test" - ] - }, - "localizationMethods": [ - "GEO", - "CZ", - "DEEP_CZ" - ] - } - }, - "trafficRouterLocations": { - "CDN_in_a_Box_Edge": { - "latitude": 38.897663, - "longitude": -77.036574, - "backupLocations": { - "fallbackToClosest": "false" + "zonemanager.cache.maintenance.interval": "300", + "zonemanager.threadpool.scale": "0.50" + }, + "contentServers": { + "edge": { + "cacheGroup": "CDN_in_a_Box_Edge", + "capabilities": [ + "RAM_DISK_STORAGE" + ], + "fqdn": "edge.infra.ciab.test", + "hashCount": 999, + "hashId": "edge", + "httpsPort": 443, + "interfaceName": "eth0", + "ip": "172.26.0.3", + "ip6": "", + "locationId": "CDN_in_a_Box_Edge", + "port": 80, + "profile": "ATS_EDGE_TIER_CACHE", + "status": "REPORTED", + "type": "EDGE", + "routingDisabled": 0 }, - "localizationMethods": [ - "GEO", - "CZ", - "DEEP_CZ" - ] - } - }, - "monitors": { - "trafficmonitor": { - "fqdn": "trafficmonitor.infra.ciab.test", - "httpsPort": 443, - "ip": "172.16.239.40", - "ip6": "fc01:9400:1000:8::40", - "location": "CDN_in_a_Box_Edge", - "port": 80, - "profile": "RASCAL-Traffic_Monitor", - "status": "ONLINE" + "mid": { + "cacheGroup": "CDN_in_a_Box_Mid", + "capabilities": [ + "RAM_DISK_STORAGE" + ], + "fqdn": "mid.infra.ciab.test", + "hashCount": 999, + "hashId": "mid", + "httpsPort": 443, + "interfaceName": "eth0", + "ip": "172.26.0.4", + "ip6": "", + "locationId": "CDN_in_a_Box_Mid", + "port": 80, + "profile": "ATS_MID_TIER_CACHE", + "status": "REPORTED", + "type": "MID", + "routingDisabled": 0 + } + }, + "contentRouters": { + "trafficrouter": { + "api.port": "3333", + "fqdn": "trafficrouter.infra.ciab.test", + "httpsPort": 443, + "ip": "172.26.0.15", + "ip6": "", + "location": "CDN_in_a_Box_Edge", + "port": 80, + "profile": "CCR_CIAB", + "secure.api.port": "3443", + "status": "ONLINE" + } + }, + "deliveryServices": { + "demo1": { + "anonymousBlockingEnabled": "false", + "consistentHashQueryParams": [ + "abc", + "pdq", + "xxx", + "zyx" + ], + "coverageZoneOnly": "false", + "deepCachingType": "NEVER", + "dispersion": { + "limit": 1, + "shuffled": "true" + }, + "domains": [ + "demo1.mycdn.ciab.test" + ], + "ecsEnabled": "false", + "geolocationProvider": "maxmindGeolocationService", + "ip6RoutingEnabled": "true", + "matchsets": [ + { + "protocol": "HTTP", + "matchlist": [ + { + "regex": ".*\\.demo1\\..*", + "match-type": "HOST" + } + ] + } + ], + "missLocation": { + "lat": 42, + "long": -88 + }, + "protocol": { + "acceptHttps": "true", + "redirectToHttps": "false" + }, + "regionalGeoBlocking": "false", + "requiredCapabilities": [ + "RAM_DISK_STORAGE" + ], + "routingName": "video", + "soa": { + "admin": "traffic_ops", + "expire": "604800", + "minimum": "30", + "refresh": "28800", + "retry": "7200" + }, + "sslEnabled": "true", + "topology": "my-topology", + "ttls": { + "A": "", + "AAAA": "", + "NS": "3600", + "SOA": "86400" + } + } + }, + "edgeLocations": { + "CDN_in_a_Box_Edge": { + "latitude": 38.897663, + "longitude": -77.036574, + "backupLocations": { + "fallbackToClosest": "true" + }, + "localizationMethods": [ + "GEO", + "CZ", + "DEEP_CZ" + ] + } + }, + "trafficRouterLocations": { + "CDN_in_a_Box_Edge": { + "latitude": 38.897663, + "longitude": -77.036574, + "backupLocations": { + "fallbackToClosest": "false" + }, + "localizationMethods": [ + "GEO", + "CZ", + "DEEP_CZ" + ] + } + }, + "monitors": { + "trafficmonitor": { + "fqdn": "trafficmonitor.infra.ciab.test", + "httpsPort": 443, + "ip": "172.26.0.14", + "ip6": "", + "location": "CDN_in_a_Box_Edge", + "port": 80, + "profile": "RASCAL-Traffic_Monitor", + "status": "ONLINE" + } + }, + "stats": { + "CDN_name": "CDN-in-a-Box", + "date": 1590607873, + "tm_host": "trafficops.infra.ciab.test:443", + "tm_path": "/api/3.0/cdns/CDN-in-a-Box/snapshot/new", + "tm_user": "admin", + "tm_version": "development" + }, + "topologies": { + "my-topology": { + "nodes": [ + "CDN_in_a_Box_Edge" + ] + } } - }, - "stats": { - "CDN_name": "CDN-in-a-Box", - "date": 1544650908, - "tm_host": "ipcdn-cache-51.cdnlab.comcast.net:6443", - "tm_path": "/tools/write_crconfig/CDN-in-a-Box", - "tm_user": "admin", - "tm_version": "traffic_ops-3.0.0-9813.8ad7bd8e.el7" } - }} + } .. [#httpOnly] These only apply to HTTP-:ref:`routed ` :term:`Delivery Services` diff --git a/docs/source/glossary.rst b/docs/source/glossary.rst index 1c2553b78c..f330d7ad63 100644 --- a/docs/source/glossary.rst +++ b/docs/source/glossary.rst @@ -28,6 +28,7 @@ Glossary astats (stats_over_http) An :abbr:`ATS (Apache Traffic Server)` plugin that allows you to monitor vitals of the :abbr:`ATS (Apache Traffic Server)` server. See :ref:`astats`. + Cache Server cache server cache servers The main function of a CDN is to proxy requests from clients to :term:`origin servers` and cache the results. To proxy, in the CDN context, is to obtain content using HTTP from an :term:`origin server` on behalf of a client. To cache is to store the results so they can be reused when other clients are requesting the same content. There are three types of proxies in use on the Internet today: @@ -123,6 +124,7 @@ Glossary Edge Edge-tier + Edge-Tier Edge-tier cache Edge-tier caches Edge-tier cache server @@ -230,6 +232,7 @@ Glossary Mid Mid-tier + Mid-Tier Mid-tier cache Mid-tier caches Mid-tier cache server diff --git a/lib/go-tc/crconfig.go b/lib/go-tc/crconfig.go index 20547fb43a..652be79d5a 100644 --- a/lib/go-tc/crconfig.go +++ b/lib/go-tc/crconfig.go @@ -30,6 +30,7 @@ type CRConfig struct { RouterLocations map[string]CRConfigLatitudeLongitude `json:"trafficRouterLocations,omitempty"` Monitors map[string]CRConfigMonitor `json:"monitors,omitempty"` Stats CRConfigStats `json:"stats,omitempty"` + Topologies map[string]CRConfigTopology `json:"topologies,omitempty"` } // CRConfigConfig used to be the type of CRConfig's Config field, though @@ -86,6 +87,7 @@ type CRConfigServerStatus string type CRConfigTrafficOpsServer struct { CacheGroup *string `json:"cacheGroup,omitempty"` + Capabilities []string `json:"capabilities,omitempty"` Fqdn *string `json:"fqdn,omitempty"` HashCount *int `json:"hashCount,omitempty"` HashId *string `json:"hashId,omitempty"` @@ -105,31 +107,37 @@ type CRConfigTrafficOpsServer struct { //TODO: drichardson - reconcile this with the DeliveryService struct in deliveryservices.go type CRConfigDeliveryService struct { AnonymousBlockingEnabled *string `json:"anonymousBlockingEnabled,omitempty"` + BypassDestination map[string]*CRConfigBypassDestination `json:"bypassDestination,omitempty"` ConsistentHashQueryParams []string `json:"consistentHashQueryParams,omitempty"` ConsistentHashRegex *string `json:"consistentHashRegex,omitempty"` CoverageZoneOnly bool `json:"coverageZoneOnly,string"` + DeepCachingType *DeepCachingType `json:"deepCachingType"` Dispersion *CRConfigDispersion `json:"dispersion,omitempty"` Domains []string `json:"domains,omitempty"` + EcsEnabled *bool `json:"ecsEnabled,string,omitempty"` + GeoEnabled []CRConfigGeoEnabled `json:"geoEnabled,omitempty"` + GeoLimitRedirectURL *string `json:"geoLimitRedirectURL,omitempty"` GeoLocationProvider *string `json:"geolocationProvider,omitempty"` + IP6RoutingEnabled *bool `json:"ip6RoutingEnabled,string,omitempty"` MatchSets []*MatchSet `json:"matchsets,omitempty"` + MaxDNSIPsForLocation *int `json:"maxDnsIpsForLocation,omitempty"` MissLocation *CRConfigLatitudeLongitudeShort `json:"missLocation,omitempty"` Protocol *CRConfigDeliveryServiceProtocol `json:"protocol,omitempty"` RegionalGeoBlocking *string `json:"regionalGeoBlocking,omitempty"` - ResponseHeaders map[string]string `json:"responseHeaders,omitempty"` RequestHeaders []string `json:"requestHeaders,omitempty"` + RequiredCapabilities []string `json:"requiredCapabilities,omitempty"` + ResponseHeaders map[string]string `json:"responseHeaders,omitempty"` + RoutingName *string `json:"routingName,omitempty"` Soa *SOA `json:"soa,omitempty"` SSLEnabled bool `json:"sslEnabled,string"` + StaticDNSEntries []CRConfigStaticDNSEntry `json:"staticDnsEntries,omitempty"` + Topology *string `json:"topology,omitempty"` TTL *int `json:"ttl,omitempty"` TTLs *CRConfigTTL `json:"ttls,omitempty"` - MaxDNSIPsForLocation *int `json:"maxDnsIpsForLocation,omitempty"` - IP6RoutingEnabled *bool `json:"ip6RoutingEnabled,string,omitempty"` - EcsEnabled *bool `json:"ecsEnabled,string,omitempty"` - RoutingName *string `json:"routingName,omitempty"` - BypassDestination map[string]*CRConfigBypassDestination `json:"bypassDestination,omitempty"` - DeepCachingType *DeepCachingType `json:"deepCachingType"` - GeoEnabled []CRConfigGeoEnabled `json:"geoEnabled,omitempty"` - GeoLimitRedirectURL *string `json:"geoLimitRedirectURL,omitempty"` - StaticDNSEntries []CRConfigStaticDNSEntry `json:"staticDnsEntries,omitempty"` +} + +type CRConfigTopology struct { + Nodes []string `json:"nodes"` } type CRConfigGeoEnabled struct { diff --git a/traffic_ops/traffic_ops_golang/crconfig/crconfig.go b/traffic_ops/traffic_ops_golang/crconfig/crconfig.go index 55d3d217ba..c89f93b599 100644 --- a/traffic_ops/traffic_ops_golang/crconfig/crconfig.go +++ b/traffic_ops/traffic_ops_golang/crconfig/crconfig.go @@ -52,6 +52,9 @@ func Make(tx *sql.Tx, cdn, user, toHost, reqPath, toVersion string, useClientReq if crc.DeliveryServices, err = makeDSes(cdn, cdnDomain, tx); err != nil { return nil, errors.New("Error getting Delivery Services: " + err.Error()) } + if crc.Topologies, err = makeTopologies(tx); err != nil { + return nil, errors.New("Error getting Topologies: " + err.Error()) + } if !useClientReqHost { paramTMURL, ok, err := getGlobalParam(tx, "tm.url") diff --git a/traffic_ops/traffic_ops_golang/crconfig/deliveryservice.go b/traffic_ops/traffic_ops_golang/crconfig/deliveryservice.go index 6e766a226a..96fd2771ea 100644 --- a/traffic_ops/traffic_ops_golang/crconfig/deliveryservice.go +++ b/traffic_ops/traffic_ops_golang/crconfig/deliveryservice.go @@ -92,37 +92,41 @@ func makeDSes(cdn string, domain string, tx *sql.Tx) (map[string]tc.CRConfigDeli } q := ` -SELECT d.xml_id, - d.miss_lat, - d.miss_long, - d.protocol, - d.ccr_dns_ttl AS ttl, - d.routing_name, - d.geo_provider, - t.name AS type, - d.geo_limit, - d.geo_limit_countries, - d.geolimit_redirect_url, +SELECT d.anonymous_blocking_enabled, + d.consistent_hash_regex, + d.deep_caching_type, d.initial_dispersion, - d.regional_geo_blocking, - d.tr_response_headers, - d.max_dns_answers, - p.name AS profile, + d.dns_bypass_cname, d.dns_bypass_ip, d.dns_bypass_ip6, d.dns_bypass_ttl, - d.dns_bypass_cname, + (SELECT ARRAY_AGG(name ORDER BY name) + FROM deliveryservice_consistent_hash_query_param + WHERE deliveryservice_id = d.id) AS query_keys, + d.routing_name, + d.ccr_dns_ttl AS ttl, + d.ecs_enabled, + d.regional_geo_blocking, + d.geo_limit, + d.geo_limit_countries, + d.geolimit_redirect_url, + d.geo_provider, d.http_bypass_fqdn, d.ipv6_routing_enabled, - d.ecs_enabled, - d.deep_caching_type, + d.max_dns_answers, + d.miss_lat, + d.miss_long, + p.name AS profile, + d.protocol, + (SELECT ARRAY_AGG(required_capability ORDER BY required_capability) + FROM deliveryservices_required_capability + WHERE deliveryservice_id = d.id) AS required_capabilities, + d.topology, d.tr_request_headers, d.tr_response_headers, - d.anonymous_blocking_enabled, - d.consistent_hash_regex, - (SELECT ARRAY_AGG(name ORDER BY name) - FROM deliveryservice_consistent_hash_query_param - WHERE deliveryservice_id = d.id) AS query_keys + d.tr_response_headers, + t.name AS type, + d.xml_id FROM deliveryservice AS d INNER JOIN type AS t ON t.id = d.type LEFT OUTER JOIN profile AS p ON p.id = d.profile @@ -173,35 +177,37 @@ AND d.active = true anonymousBlocking := false consistentHashRegex := sql.NullString{} err := rows.Scan( - &xmlID, - &missLat, - &missLon, - &protocol, - &ds.TTL, - &ds.RoutingName, - &geoProvider, - &ttype, - &geoLimit, - &geoLimitCountries, - &geoLimitRedirectURL, + &anonymousBlocking, + &consistentHashRegex, + &deepCachingType, &dispersion, - &geoBlocking, - &trRespHdrsStr, - &maxDNSAnswers, - &profile, + &dnsBypassCName, &dnsBypassIP, &dnsBypassIP6, &dnsBypassTTL, - &dnsBypassCName, + pq.Array(&ds.ConsistentHashQueryParams), + &ds.RoutingName, + &ds.TTL, + &ecsEnabled, + &geoBlocking, + &geoLimit, + &geoLimitCountries, + &geoLimitRedirectURL, + &geoProvider, &httpBypassFQDN, &ip6RoutingEnabled, - &ecsEnabled, - &deepCachingType, + &maxDNSAnswers, + &missLat, + &missLon, + &profile, + &protocol, + pq.Array(&ds.RequiredCapabilities), + &ds.Topology, &trRequestHeaders, + &trRespHdrsStr, &trResponseHeaders, - &anonymousBlocking, - &consistentHashRegex, - pq.Array(&ds.ConsistentHashQueryParams), + &ttype, + &xmlID, ) if err != nil { return nil, errors.New("scanning deliveryservice: " + err.Error()) diff --git a/traffic_ops/traffic_ops_golang/crconfig/deliveryservice_test.go b/traffic_ops/traffic_ops_golang/crconfig/deliveryservice_test.go index d272164f84..0940044d13 100644 --- a/traffic_ops/traffic_ops_golang/crconfig/deliveryservice_test.go +++ b/traffic_ops/traffic_ops_golang/crconfig/deliveryservice_test.go @@ -68,9 +68,10 @@ func randDS() tc.CRConfigDeliveryService { AcceptHTTPS: false, RedirectOnHTTPS: false, }, - RegionalGeoBlocking: &falseStrPtr, - ResponseHeaders: nil, - RequestHeaders: nil, + RegionalGeoBlocking: &falseStrPtr, + ResponseHeaders: nil, + RequestHeaders: nil, + RequiredCapabilities: randStrArray(), Soa: &tc.SOA{ Admin: &ttlAdmin, ExpireSeconds: &ttlExpire, @@ -80,6 +81,7 @@ func randDS() tc.CRConfigDeliveryService { }, SSLEnabled: false, EcsEnabled: &ecsEnabled, + Topology: randStr(), TTL: ttl, TTLs: &tc.CRConfigTTL{ ASeconds: &ttlStr, @@ -123,68 +125,72 @@ func ExpectedMakeDSes() map[string]tc.CRConfigDeliveryService { func MockMakeDSes(mock sqlmock.Sqlmock, expected map[string]tc.CRConfigDeliveryService, cdn string) { rows := sqlmock.NewRows([]string{ - "xml_id", - "miss_lat", - "miss_long", - "protocol", - "ttl", - "routing_name", - "geo_provider", - "type", - "geo_limit", - "geo_limit_countries", - "geeo_limit_redirect_url", + "anonymous_blocking_enabled", + "consistent_hash_regex", + "deep_caching_type", "initial_dispersion", - "regional_geo_blocking", - "tr_response_headers", - "max_dns_answers", - "profile", + "dns_bypass_cname", "dns_bypass_ip", "dns_bypass_ip6", "dns_bypass_ttl", - "dns_bypass_cname", + "query_keys", + "routing_name", + "ttl", + "ecs_enabled", + "regional_geo_blocking", + "geo_limit", + "geo_limit_countries", + "geeo_limit_redirect_url", + "geo_provider", "http_bypass_fqdn", "ipv6_routing_enabled", - "ecs_enabled", - "deep_caching_type", + "max_dns_answers", + "miss_lat", + "miss_long", + "profile", + "protocol", + "required_capabilities", + "topology", "tr_request_headers", "tr_response_headers", - "anonymous_blocking_enabled", - "consistent_hash_regex", - "query_keys"}) + "tr_response_headers", + "type", + "xml_id"}) for dsName, ds := range expected { queryParams := "{" + strings.Join(ds.ConsistentHashQueryParams, ",") + "}" rows = rows.AddRow( - dsName, - ds.MissLocation.Lat, - ds.MissLocation.Lon, - 0, - *ds.TTL, - *ds.RoutingName, - 0, - "HTTP", - 0, - "", - "", - 42, false, "", nil, + 42, "", "", "", 0, + queryParams, + *ds.RoutingName, + *ds.TTL, + *ds.EcsEnabled, + false, + 0, + "", "", + 0, *ds.BypassDestination["HTTP"].FQDN, *ds.IP6RoutingEnabled, - *ds.EcsEnabled, nil, + ds.MissLocation.Lat, + ds.MissLocation.Lon, "", + 0, + "{"+strings.Join(ds.RequiredCapabilities, ",")+"}", + ds.Topology, "", - false, "", - queryParams) + "", + "HTTP", + dsName) } mock.ExpectQuery("select").WithArgs(cdn).WillReturnRows(rows) } diff --git a/traffic_ops/traffic_ops_golang/crconfig/servers.go b/traffic_ops/traffic_ops_golang/crconfig/servers.go index 9c7c6447b2..0b0265ac85 100644 --- a/traffic_ops/traffic_ops_golang/crconfig/servers.go +++ b/traffic_ops/traffic_ops_golang/crconfig/servers.go @@ -26,6 +26,8 @@ import ( "strconv" "strings" + "github.com/lib/pq" + "github.com/apache/trafficcontrol/lib/go-log" "github.com/apache/trafficcontrol/lib/go-tc" @@ -102,7 +104,7 @@ type ServerUnion struct { type ServerAndHost struct { Server ServerUnion - Host string + Host string } const DefaultWeightMultiplier = 1000.0 @@ -127,7 +129,10 @@ func getAllServers(cdn string, tx *sql.Tx) (map[string]ServerUnion, error) { p.name AS profile_name, cast(p.routing_disabled AS int), st.name AS status, - t.name AS type + t.name AS type, + (SELECT ARRAY_AGG(server_capability ORDER BY server_capability) + FROM server_server_capability + WHERE server = s.id) AS capabilities FROM server AS s INNER JOIN cachegroup AS cg ON cg.id = s.cachegroup INNER JOIN type AS t on t.id = s.type @@ -153,7 +158,7 @@ func getAllServers(cdn string, tx *sql.Tx) (map[string]ServerUnion, error) { var status string var id int - if err := rows.Scan(&id, &s.Host, &s.Server.CacheGroup, &s.Server.Fqdn, &hashId, &httpsPort, &port, &s.Server.Profile, &s.Server.RoutingDisabled, &status, &s.Server.ServerType); err != nil { + if err := rows.Scan(&id, &s.Host, &s.Server.CacheGroup, &s.Server.Fqdn, &hashId, &httpsPort, &port, &s.Server.Profile, &s.Server.RoutingDisabled, &status, &s.Server.ServerType, pq.Array(&s.Server.Capabilities)); err != nil { return nil, errors.New("Error scanning server: " + err.Error()) } @@ -300,7 +305,8 @@ func getServerDSes(cdn string, tx *sql.Tx, domain string) (map[tc.CacheName]map[ } q := ` -select ds.xml_id as ds, dt.name as ds_type, ds.routing_name, r.pattern as pattern +select ds.xml_id as ds, dt.name as ds_type, ds.routing_name, r.pattern as pattern, +ds.topology IS NOT NULL as has_topology from regex as r inner join type as rt on r.type = rt.id inner join deliveryservice_regex as dsr on dsr.regex = r.id @@ -326,10 +332,15 @@ order by dsr.set_number asc dsType := "" dsPattern := "" dsRoutingName := "" + var hasTopology bool inf := DSRouteInfo{} - if err := rows.Scan(&ds, &dsType, &dsRoutingName, &dsPattern); err != nil { + if err := rows.Scan(&ds, &dsType, &dsRoutingName, &dsPattern, &hasTopology); err != nil { return nil, errors.New("Error scanning server deliveryservices: " + err.Error()) } + // Topology-based delivery services do not use the contentServers.deliveryServices field + if hasTopology { + continue + } inf.IsDNS = strings.HasPrefix(dsType, "DNS") inf.IsRaw = !strings.Contains(dsPattern, `.*`) if !inf.IsRaw { diff --git a/traffic_ops/traffic_ops_golang/crconfig/servers_test.go b/traffic_ops/traffic_ops_golang/crconfig/servers_test.go index a83b4053ac..8579c2bf99 100644 --- a/traffic_ops/traffic_ops_golang/crconfig/servers_test.go +++ b/traffic_ops/traffic_ops_golang/crconfig/servers_test.go @@ -22,9 +22,11 @@ package crconfig import ( "context" "fmt" + "github.com/lib/pq" "math/rand" "net" "reflect" + "strings" "testing" "time" @@ -34,6 +36,8 @@ import ( "gopkg.in/DATA-DOG/go-sqlmock.v1" ) +type StringArray pq.StringArray + func randBool() *bool { b := rand.Int()%2 == 0 return &b @@ -47,6 +51,14 @@ func randStr() *string { } return &s } +func randStrArray() []string { + num := 100 + sArray := make([]string, num) + for i := 0; i < num; i++ { + sArray[i] = *randStr() + } + return sArray +} func randInt() *int { i := rand.Int() return &i @@ -109,6 +121,7 @@ func randServer(ipService bool, ip6Service bool) tc.CRConfigTrafficOpsServer { return tc.CRConfigTrafficOpsServer{ CacheGroup: cachegroup, + Capabilities: randStrArray(), Fqdn: randStr(), HashCount: randInt(), HashId: randStr(), @@ -217,12 +230,13 @@ func ExpectedGetAllServers(params map[string]ServerParams, ipIsService bool, ip6 } func MockGetAllServers(mock sqlmock.Sqlmock, expected map[string]ServerUnion, cdn string, ipIsService bool, ip6IsService bool) { - serverRows := sqlmock.NewRows([]string{"id", "host_name", "cachegroup", "fqdn", "hashid", "https_port", "tcp_port", "profile_name", "routing_disabled", "status", "type"}) + serverRows := sqlmock.NewRows([]string{"id", "host_name", "cachegroup", "fqdn", "hashid", "https_port", "tcp_port", "profile_name", "routing_disabled", "status", "type", "capabilities"}) interfaceRows := sqlmock.NewRows([]string{"max_bandwidth", "monitor", "mtu", "name", "server"}) ipRows := sqlmock.NewRows([]string{"address", "gateway", "service_address", "interface", "server"}) i := 1 for name, s := range expected { - serverRows = serverRows.AddRow(i, name, *s.CacheGroup, *s.Fqdn, *s.HashId, *s.HttpsPort, *s.Port, *s.Profile, s.RoutingDisabled, *s.ServerStatus, *s.ServerType) + capabilities := "{" + strings.Join(s.Capabilities, ",") + "}" + serverRows = serverRows.AddRow(i, name, *s.CacheGroup, *s.Fqdn, *s.HashId, *s.HttpsPort, *s.Port, *s.Profile, s.RoutingDisabled, *s.ServerStatus, *s.ServerType, capabilities) if s.InterfaceName == nil { i++ continue @@ -538,7 +552,7 @@ func ExpectedGetServerDSes(expectedGetServerDSNames map[tc.CacheName][]tc.Delive } func MockGetServerDSes(mock sqlmock.Sqlmock, expected map[tc.CacheName]map[string][]string, cdn string) { - rows := sqlmock.NewRows([]string{"ds", "ds_type", "routing_name", "pattern"}) + rows := sqlmock.NewRows([]string{"ds", "ds_type", "routing_name", "pattern", "hasTopology"}) dsmap := map[string][]string{} for _, dses := range expected { for ds, patterns := range dses { @@ -548,7 +562,7 @@ func MockGetServerDSes(mock sqlmock.Sqlmock, expected map[tc.CacheName]map[strin for ds, patterns := range dsmap { for _, pattern := range patterns { - rows = rows.AddRow(ds, "DNS", "", pattern) + rows = rows.AddRow(ds, "DNS", "", pattern, false) } } mock.ExpectQuery("select").WithArgs(cdn).WillReturnRows(rows) diff --git a/traffic_ops/traffic_ops_golang/crconfig/topologies.go b/traffic_ops/traffic_ops_golang/crconfig/topologies.go new file mode 100644 index 0000000000..d86cdc1fb6 --- /dev/null +++ b/traffic_ops/traffic_ops_golang/crconfig/topologies.go @@ -0,0 +1,64 @@ +package crconfig + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "database/sql" + "errors" + "github.com/apache/trafficcontrol/lib/go-log" + "github.com/apache/trafficcontrol/lib/go-tc" + "github.com/lib/pq" +) + +func makeTopologies(tx *sql.Tx) (map[string]tc.CRConfigTopology, error) { + query := ` +SELECT + t.name, + (SELECT ARRAY_AGG(tc.cachegroup ORDER BY tc.cachegroup) + FROM topology_cachegroup tc + JOIN cachegroup c ON c.name = tc.cachegroup + JOIN type ON type.id = c.type + WHERE t.name = tc.topology + AND type.name = $1 + ) AS nodes +FROM topology t +ORDER BY t.name +` + var rows *sql.Rows + var err error + if rows, err = tx.Query(query, tc.CacheGroupEdgeTypeName); err != nil { + return nil, errors.New("querying topologies: " + err.Error()) + } + defer log.Close(rows, "unable to close DB connection") + + topologies := map[string]tc.CRConfigTopology{} + for rows.Next() { + topology := tc.CRConfigTopology{} + var name string + if err = rows.Scan( + &name, + pq.Array(&topology.Nodes), + ); err != nil { + return nil, errors.New("scanning topology: " + err.Error()) + } + topologies[name] = topology + } + return topologies, nil +} diff --git a/traffic_ops/traffic_ops_golang/crconfig/topologies_test.go b/traffic_ops/traffic_ops_golang/crconfig/topologies_test.go new file mode 100644 index 0000000000..9c9b637110 --- /dev/null +++ b/traffic_ops/traffic_ops_golang/crconfig/topologies_test.go @@ -0,0 +1,102 @@ +package crconfig + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "context" + "encoding/json" + "github.com/apache/trafficcontrol/lib/go-tc" + "gopkg.in/DATA-DOG/go-sqlmock.v1" + "reflect" + "strings" + "testing" + "time" +) + +func randTopology() tc.CRConfigTopology { + return tc.CRConfigTopology{ + Nodes: randStrArray(), + } +} + +func ExpectedMakeTops() map[string]tc.CRConfigTopology { + return map[string]tc.CRConfigTopology{ + "top1": randTopology(), + "top2": randTopology(), + } +} + +func MockMakeTops(mock sqlmock.Sqlmock, expected map[string]tc.CRConfigTopology) { + rows := sqlmock.NewRows([]string{ + "name", + "nodes"}) + + for topName, top := range expected { + nodes := "{" + strings.Join(top.Nodes, ",") + "}" + rows = rows.AddRow( + topName, + nodes) + } + mock.ExpectQuery("SELECT").WillReturnRows(rows) +} + +func TestMakeTops(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) + } + + expected := ExpectedMakeTops() + mock.ExpectBegin() + MockMakeTops(mock, expected) + mock.ExpectCommit() + + dbCtx, _ := context.WithTimeout(context.TODO(), time.Duration(10)*time.Second) + tx, err := db.BeginTx(dbCtx, nil) + if err != nil { + t.Fatal("creating transaction: ", err) + } + + actual, err := makeTopologies(tx) + if err != nil { + t.Fatal("makeTopologies expected: nil error, actual: ", err) + } + + if err = db.Close(); err != nil { + t.Fatal("closing db: ", err) + } + + if len(actual) != len(expected) { + t.Fatalf("makeTopologies len expected: %v, actual: %v", len(expected), len(actual)) + } + + for topName, top := range expected { + actualTop, ok := actual[topName] + if !ok { + t.Errorf("makeTopologies expected: %v, actual: missing", topName) + continue + } + expectedBts, _ := json.MarshalIndent(top, " ", " ") + actualBts, _ := json.MarshalIndent(actualTop, " ", " ") + if !reflect.DeepEqual(expectedBts, actualBts) { + t.Errorf("makeDSes ds %+v expected: %+v\n\nactual: %+v\n\n\n", topName, string(expectedBts), string(actualBts)) + } + } +} diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java index 7cc3ee94d5..d1a213608d 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java @@ -25,12 +25,14 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.TreeSet; import java.util.Iterator; import java.util.concurrent.BlockingQueue; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; +import java.util.stream.Stream; import com.comcast.cdn.traffic_control.traffic_router.core.ds.LetsEncryptDnsChallengeWatcher; import com.comcast.cdn.traffic_control.traffic_router.core.ds.SteeringWatcher; @@ -76,6 +78,7 @@ public class ConfigHandler { private static long lastSnapshotTimestamp = 0; private static Object configSync = new Object(); public static String deliveryServicesKey = "deliveryServices"; + public static String topologiesKey = "topologies"; private TrafficRouterManager trafficRouterManager; private GeolocationDatabaseUpdater geolocationDatabaseUpdater; @@ -227,6 +230,9 @@ public boolean processConfig(final String jsonStr) throws JsonUtilsException, IO parseEdgeTrafficRouterLocations(jo, cacheRegister); parseCacheConfig(JsonUtils.getJsonNode(jo, "contentServers"), cacheRegister); + if (jo.has(topologiesKey)) { + parseTopologyConfig(JsonUtils.getJsonNode(jo, topologiesKey), deliveryServiceMap, cacheRegister); + } parseMonitorConfig(JsonUtils.getJsonNode(jo, "monitors")); federationsWatcher.configure(config); @@ -300,7 +306,7 @@ public void setRegionalGeoUpdater(final RegionalGeoUpdater regionalGeoUpdater) { public void setAnonymousIpConfigUpdater(final AnonymousIpConfigUpdater anonymousIpConfigUpdater) { this.anonymousIpConfigUpdater = anonymousIpConfigUpdater; } - + public void setAnonymousIpDatabaseUpdater(final AnonymousIpDatabaseUpdater anonymousIpDatabaseUpdater) { this.anonymousIpDatabaseUpdater = anonymousIpDatabaseUpdater; } @@ -357,6 +363,22 @@ private void parseCacheConfig(final JsonNode contentServers, final CacheRegister cache.setFqdn(JsonUtils.getString(jo, "fqdn")); cache.setPort(JsonUtils.getInt(jo, "port")); + if (jo.has("capabilities")) { + final Set capabilities = new HashSet<>(); + final JsonNode capabilitiesNode = jo.get("capabilities"); + if (!capabilitiesNode.isArray()) { + LOGGER.error("Server '" + hashId + "' has malformed capabilities. Disregarding."); + } else { + capabilitiesNode.forEach((capabilityNode) -> { + final String capability = capabilityNode.asText(); + if (!capability.isEmpty()) { + capabilities.add(capability); + } + }); + } + cache.addCapabilities(capabilities); + } + final String ip = JsonUtils.getString(jo, "ip"); final String ip6 = JsonUtils.optString(jo, "ip6"); @@ -394,17 +416,9 @@ private void parseCacheConfig(final JsonNode contentServers, final CacheRegister } final String tld = JsonUtils.optString(cacheRegister.getConfig(), "domain_name"); - - if (name.endsWith(tld)) { - final String reName = name.replaceAll("^.*?\\.", ""); - - if (!dsNames.contains(reName)) { - dsNames.add(reName); - } - } else { - if (!dsNames.contains(name)) { - dsNames.add(name); - } + final String dsName = getDsName( name, tld); + if (!dsNames.contains(dsName)) { + dsNames.add(dsName); } i++; @@ -459,6 +473,59 @@ private Map parseDeliveryServiceConfig(final JsonNode a return deliveryServiceMap; } + private String getDsName(final String name, final String tld) { + return name.endsWith(tld) + ? name.replaceAll("^.*?\\.", "") + : name; + } + + private void parseTopologyConfig(final JsonNode allTopologies, final Map deliveryServiceMap, final CacheRegister cacheRegister) { + final Map> topologyMap = new HashMap<>(); + final Map> statMap = new HashMap<>(); + final String tld = JsonUtils.optString(cacheRegister.getConfig(), "domain_name"); + allTopologies.fieldNames().forEachRemaining((String topologyName) -> { + final List nodes = new ArrayList<>(); + allTopologies.get(topologyName).get("nodes").forEach((JsonNode cache) -> nodes.add(cache.textValue())); + topologyMap.put(topologyName, nodes); + }); + + deliveryServiceMap.forEach((xmlId, ds) -> { + final List dsReferences = new ArrayList<>(); + final List dsNames = new ArrayList<>(); // for stats + Stream.of(ds.getTopology()) + .filter(topologyName -> !Objects.isNull(topologyName) && topologyMap.containsKey(topologyName)) + .flatMap(topologyName -> { + statMap.put(ds.getId(), dsNames); + return topologyMap.get(topologyName).stream(); + }) + .flatMap(node -> cacheRegister.getCacheLocation(node).getCaches().stream()) + .filter(cache -> ds.hasRequiredCapabilities(cache.getCapabilities())) + .forEach(cache -> { + cacheRegister.getDeliveryServiceMatchers(ds).stream() + .flatMap(deliveryServiceMatcher -> deliveryServiceMatcher.getRequestMatchers().stream()) + .map(requestMatcher -> requestMatcher.getPattern().pattern()) + .forEach(pattern -> { + final String remap = ds.getRemap(pattern); + final String fqdn = pattern.contains(".*") && !ds.isDns() + ? cache.getId() + "." + remap + : remap; + dsNames.add(getDsName(fqdn, tld)); + if (!remap.equals(ds.isDns() ? ds.getRoutingName() + "." + ds.getDomain() : ds.getDomain())) { + return; + } + try { + dsReferences.add(new DeliveryServiceReference(ds.getId(), fqdn)); + } catch (ParseException e) { + LOGGER.error("Unable to create a DeliveryServiceReference from DeliveryService '" + ds.getId() + "'", e); + } + }); + cache.setDeliveryServices(dsReferences); + }); + + }); + statTracker.initialize(statMap, cacheRegister); + } + private void parseDeliveryServiceMatchSets(final JsonNode allDeliveryServices, final Map deliveryServiceMap, final CacheRegister cacheRegister) throws JsonUtilsException { final TreeSet deliveryServiceMatchers = new TreeSet<>(); final JsonNode config = cacheRegister.getConfig(); @@ -541,7 +608,7 @@ private void initGeoFailedRedirect(final Map dsMap, fin /** * Parses the geolocation database configuration and updates the database if the URL has * changed. - * + * * @param config * the {@link TrafficRouterConfiguration} * @throws JsonUtilsException @@ -593,7 +660,7 @@ private void parseAnonymousIpConfig(final JsonNode jo) throws JsonUtilsException AnonymousIp.getCurrentConfig().enabled = false; return; } - + if (databaseUrl == null) { LOGGER.info(anonymousPollingUrl + " not configured; stopping service updater and disabling feature"); getAnonymousIpDatabaseUpdater().stopServiceUpdater(); diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java index 4a5a8cb9cb..dc155fa004 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryService.java @@ -36,6 +36,7 @@ import java.util.TreeSet; import java.util.Iterator; import java.util.concurrent.atomic.AtomicInteger; +import java.util.regex.Pattern; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.JsonNode; @@ -76,6 +77,12 @@ public class DeliveryService { @JsonIgnore private final String domain; @JsonIgnore + private final String tld; + @JsonIgnore + // Matches the beginning of a HOST_REGEXP pattern with or without confighandler.regex.superhack.enabled. + // ^\(\.\*\\\.\|\^\)|^\.\*\\\.|\\\.\.\* + private final Pattern wildcardPattern = Pattern.compile("^\\(\\.\\*\\\\\\.\\|\\^\\)|^\\.\\*\\\\\\.|\\\\\\.\\.\\*"); + @JsonIgnore private final JsonNode bypassDestination; @JsonIgnore private final JsonNode soa; @@ -83,6 +90,8 @@ public class DeliveryService { private final JsonNode props; private boolean isDns; private final String routingName; + private String topology; + private final Set requiredCapabilities; private final boolean shouldAppendQueryString; private final Geolocation missLocation; private final Dispersion dispersion; @@ -129,10 +138,31 @@ public DeliveryService(final String id, final JsonNode dsJo) throws JsonUtilsExc this.bypassDestination = dsJo.get("bypassDestination"); this.routingName = JsonUtils.getString(dsJo, "routingName").toLowerCase(); this.domain = getDomainFromJson(dsJo.get("domains")); + this.tld = this.domain != null + ? this.domain.replaceAll("^.*?\\.", "") + : null; this.soa = dsJo.get("soa"); this.shouldAppendQueryString = JsonUtils.optBoolean(dsJo, "appendQueryString", true); this.ecsEnabled = JsonUtils.optBoolean(dsJo, "ecsEnabled"); + if (dsJo.has("topology")) { + this.topology = JsonUtils.optString(dsJo, "topology"); + } + this.requiredCapabilities = new HashSet<>(); + if (dsJo.has("requiredCapabilities")) { + final JsonNode requiredCapabilitiesNode = dsJo.get("requiredCapabilities"); + if (!requiredCapabilitiesNode.isArray()) { + LOGGER.error("Delivery Service '" + id + "' has malformed requiredCapabilities. Disregarding."); + } else { + requiredCapabilitiesNode.forEach((requiredCapabilityNode) -> { + final String requiredCapability = requiredCapabilityNode.asText(); + if (!requiredCapability.isEmpty()) { + this.requiredCapabilities.add(requiredCapability); + } + }); + } + } + this.consistentHashQueryParams = new HashSet(); if (dsJo.has("consistentHashQueryParams")) { final JsonNode cqpNode = dsJo.get("consistentHashQueryParams"); @@ -358,6 +388,16 @@ public String createURIString(final HTTPRequest request, final String alternateP return uri.toString(); } + public String getRemap(final String dsPattern) { + if (!dsPattern.contains(".*")) { + return dsPattern; + } + final String host = wildcardPattern.matcher(dsPattern).replaceAll("") + "." + tld; + return this.isDns() + ? this.routingName + "." + host + : host; + } + private String getFQDN(final Cache cache) { for (final DeliveryServiceReference dsRef : cache.getDeliveryServices()) { if (dsRef.getDeliveryServiceId().equals(this.getId())) { @@ -591,6 +631,14 @@ public String getRoutingName() { return routingName; } + public String getTopology() { + return topology; + } + + public boolean hasRequiredCapabilities(final Set serverCapabilities) { + return serverCapabilities.containsAll(requiredCapabilities); + } + public Dispersion getDispersion() { return dispersion; } diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryServiceMatcher.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryServiceMatcher.java index 716fb5ae85..c0253ee489 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryServiceMatcher.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryServiceMatcher.java @@ -48,6 +48,10 @@ public void addMatch(final Type type, final String string, final String target) requestMatchers.add(new RequestMatcher(type, string, target)); } + public List getRequestMatchers() { + return new ArrayList<>(this.requestMatchers); + } + public boolean matches(final Request request) { for (final RequestMatcher matcher : requestMatchers) { if (!matcher.matches(request)) { diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/edge/CacheRegister.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/edge/CacheRegister.java index c802a84c9b..3e243a4369 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/edge/CacheRegister.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/edge/CacheRegister.java @@ -16,6 +16,7 @@ package com.comcast.cdn.traffic_control.traffic_router.core.edge; import java.util.*; +import java.util.stream.Collectors; import com.fasterxml.jackson.databind.JsonNode; @@ -129,6 +130,12 @@ public Map getCacheMap() { return allCaches; } + public Set getDeliveryServiceMatchers(final DeliveryService deliveryService) { + return this.deliveryServiceMatchers.stream() + .filter(deliveryServiceMatcher -> deliveryServiceMatcher.getDeliveryService().getId().equals(deliveryService.getId())) + .collect(Collectors.toCollection(TreeSet::new)); + } + public void setDeliveryServiceMatchers(final TreeSet matchers) { this.deliveryServiceMatchers = matchers; } diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/edge/Node.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/edge/Node.java index ea4158f58a..b40c1a9619 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/edge/Node.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/edge/Node.java @@ -19,7 +19,12 @@ import java.net.InetAddress; import java.net.UnknownHostException; import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; import com.comcast.cdn.traffic_control.traffic_router.core.util.JsonUtils; import com.fasterxml.jackson.databind.JsonNode; @@ -41,6 +46,8 @@ public enum IpVersions { private List ipAddresses; private List unavailableIpAddresses; private int port; + private final Map deliveryServices = new HashMap<>(); + private final Set capabilities = new HashSet<>(); private int httpsPort = 443; public Node(final String id) { @@ -119,6 +126,24 @@ public int hashCode() { .toHashCode(); } + public void addCapabilities(final Set capabilities) { + this.capabilities.addAll(capabilities); + } + + public Set getCapabilities() { + return this.capabilities; + } + + public void setDeliveryServices(final Collection deliveryServices) { + for (final Cache.DeliveryServiceReference deliveryServiceReference : deliveryServices) { + this.deliveryServices.put(deliveryServiceReference.getDeliveryServiceId(), deliveryServiceReference); + } + } + + public boolean hasDeliveryService(final String deliveryServiceId) { + return deliveryServices.containsKey(deliveryServiceId); + } + public void setFqdn(final String fqdn) { this.fqdn = fqdn; } diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/request/RequestMatcher.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/request/RequestMatcher.java index 14fca52eeb..d3bd57952e 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/request/RequestMatcher.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/request/RequestMatcher.java @@ -59,6 +59,14 @@ public boolean matches(final Request request) { return pattern.matcher(target).matches(); } + public Type getType() { + return type; + } + + public Pattern getPattern() { + return pattern; + } + private String getTarget(final Request request) { if (type == Type.HOST) { return request.getHostname(); diff --git a/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandlerTest.java b/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandlerTest.java index 9e9e000dd3..bdb9c9947a 100644 --- a/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandlerTest.java +++ b/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandlerTest.java @@ -16,11 +16,18 @@ package com.comcast.cdn.traffic_control.traffic_router.core.config; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.TreeSet; +import com.comcast.cdn.traffic_control.traffic_router.core.ds.DeliveryServiceMatcher; +import com.comcast.cdn.traffic_control.traffic_router.core.edge.Cache; +import com.comcast.cdn.traffic_control.traffic_router.core.edge.CacheLocation; +import com.comcast.cdn.traffic_control.traffic_router.core.router.StatTracker; +import com.comcast.cdn.traffic_control.traffic_router.geolocation.Geolocation; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.comcast.cdn.traffic_control.traffic_router.core.edge.CacheLocation.LocalizationMethod; @@ -30,7 +37,6 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; -import static org.mockito.Matchers.anyBoolean; import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; @@ -248,6 +254,61 @@ public Void answer(InvocationOnMock invocation) { assertThat(typeUrl[0], equalTo(path)); } + @Test + public void itParsesTheTopologiesConfig() throws Exception { + /* Make the CacheLocation, add a Cache, and add the CacheLocation to the CacheRegister */ + final String cacheId = "edge"; + final Cache cache = new Cache(cacheId, cacheId, 0); + final String location = "CDN_in_a_Box_Edge"; + final CacheLocation cacheLocation = new CacheLocation(location, new Geolocation(38.897663, 38.897663)); + cacheLocation.addCache(cache); + final Set locations = new HashSet<>(); + locations.add(cacheLocation); + final CacheRegister register = new CacheRegister(); + register.setConfiguredLocations(locations); + + /* Add a capability to the Cache */ + final String capability = "a-capability"; + final Set capabilities = new HashSet<>(); + capabilities.add(capability); + cache.addCapabilities(capabilities); + + /* Mock a DeliveryService and add it to our DeliveryService Map */ + final String dsId = "top-ds"; + final String routingName = "cdn"; + final String domain = "ds.site.com"; + final String topology = "foo"; + final String superHackedRegexp = "(.*\\.|^)" + dsId + "\\..*"; + final DeliveryService ds = mock(DeliveryService.class); + when(ds.getId()).thenReturn(dsId); + when(ds.getDomain()).thenReturn(domain); + when(ds.getRemap(superHackedRegexp)).thenReturn(domain); + when(ds.getRoutingName()).thenReturn(routingName); + when(ds.getTopology()).thenReturn(topology); + when(ds.hasRequiredCapabilities(capabilities)).thenReturn(true); + when(ds.isDns()).thenReturn(false); + final Map dsMap = new HashMap<>(); + dsMap.put(dsId, ds); + + final DeliveryServiceMatcher dsMatcher = new DeliveryServiceMatcher(ds); + dsMatcher.addMatch(DeliveryServiceMatcher.Type.HOST, superHackedRegexp, ""); + final TreeSet dsMatchers = new TreeSet<>(); + dsMatchers.add(dsMatcher); + register.setDeliveryServiceMap(dsMap); + register.setDeliveryServiceMatchers(dsMatchers); + + /* Parse the Topologies config JSON */ + final ObjectMapper mapper = new ObjectMapper(); + final JsonNode allTopologiesJson = mapper.readTree("{\"" + topology + "\":{\"nodes\":[\"" + location + "\"]}}"); + Whitebox.setInternalState(handler, "statTracker", new StatTracker()); + Whitebox.invokeMethod(handler, "parseTopologyConfig", allTopologiesJson, dsMap, register); + + /* Assert that the DeliveryService was assigned to the Cache */ + Collection dsReferences = cache.getDeliveryServices(); + assertThat(dsReferences.size(), equalTo(1)); + assertThat(dsReferences.iterator().next().getDeliveryServiceId(), equalTo(dsId)); + } + @Test public void testParseLocalizationMethods() throws Exception { LocalizationMethod[] allMethods = new LocalizationMethod[] { diff --git a/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryServiceTest.java b/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryServiceTest.java index 59da1c97e1..2306fcdd69 100644 --- a/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryServiceTest.java +++ b/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/ds/DeliveryServiceTest.java @@ -19,6 +19,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.Test; +import org.powermock.reflect.Whitebox; + +import java.util.HashSet; +import java.util.Set; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; @@ -59,7 +63,7 @@ public void itHandlesDuplicatesInConsistentHashQueryParams() throws Exception { public void itExtractsQueryParams() throws Exception { final JsonNode json = (new ObjectMapper()).readTree("{\"routingName\":\"edge\",\"coverageZoneOnly\":false,\"consistentHashQueryParams\":[\"test\", \"quest\"]}"); final HTTPRequest r = new HTTPRequest(); - r.setPath("/path1234/some_stream_name1234/some_other_info.m3u8"); + r.setPath("/path1234/some_stream_name1234/some_other_info.m3u8"); r.setQueryString("test=value&foo=fizz&quest=oth%20ervalue&bar=buzz"); assert (new DeliveryService("test", json)).extractSignificantQueryParams(r).equals("quest=oth ervaluetest=value"); } @@ -75,4 +79,12 @@ public void itConfiguresRequestHeadersFromJSON() throws Exception { assertThat(deliveryService.getRequestHeaders(), containsInAnyOrder("Cache-Control", "Cookie", "Content-Type", "If-Modified-Since")); } + @Test + public void itAddsRequiredCapabilities() throws Exception { + final ObjectMapper mapper = new ObjectMapper(); + final JsonNode jsonConfiguration = mapper.readTree("{\"requiredCapabilities\":[\"all-read\",\"all-write\",\"cdn-read\"],\"routingName\":\"edge\",\"coverageZoneOnly\":false}"); + final DeliveryService deliveryService = new DeliveryService("has-required-capabilities", jsonConfiguration); + + assertThat(Whitebox.getInternalState(deliveryService, "requiredCapabilities"), containsInAnyOrder("all-read", "all-write", "cdn-read")); + } }