Changes for Backup Edge Cache Group

CHANGELOG.md

@@ -31,8 +31,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
   - /api/1.3/statuses (R)
   - /api/1.3/system/info
   - /api/1.3/types (R)
+- Backup Edge Cache group: Backup Edge group for a particular cache group can be configured using coverage zone files. With this, users will be able control traffic within portions of their network, thereby avoiding choosing fallback cache groups using geo coordinates(getClosestAvailableCachegroup). This can be controlled using "backupZones" which contains configuration for backup cache groups parsed from the Coverage Zone File as explained [here](http://traffic-control-cdn.readthedocs.io/en/latest/admin/traffic_ops/using.html#the-coverage-zone-file-and-asn-table)
 
 ### Changed
 - Reformatted this CHANGELOG file to the keep-a-changelog format
 
-[Unreleased]: https://github.com/apache/incubator-trafficcontrol/compare/RELEASE-2.1.0...HEAD
+[Unreleased]: https://github.com/apache/incubator-trafficcontrol/compare/RELEASE-2.1.0...HEAD 
+

docs/source/admin/traffic_ops/using.rst

@@ -873,6 +873,24 @@ The Coverage Zone File (CZF) should contain a cachegroup name to network prefix
           "192.168.4.0/24",
           "192.168.5.0/24"
         ]
+      },
+      "cache-group-03": {
+        "backupZones":{
+          "list": ["cache-group-02"],
+          "fallbackToClosestGroup": true
+        },
+        "coordinates": {
+          "latitude":  3.3,
+          "longitude": 4.4
+        },
+        "network6": [
+          "1234:566a::/64",
+          "1234:566b::/64"
+        ],
+        "network": [
+          "192.168.2.0/24",
+          "192.168.3.0/24"
+        ]
       }
     }
   }
@@ -881,7 +899,7 @@ The CZF is an input to the Traffic Control CDN, and as such does not get generat
 
 The script that generates the CZF file is not part of Traffic Control, since it is different for each situation.
 
-.. note:: The ``"coordinates"`` section is optional and may be used by Traffic Router for localization in the case of a CZF "hit" where the zone name does not map to a Cache Group name in Traffic Ops (i.e. Traffic Router will route to the closest Cache Group(s) geographically).
+.. note:: The ``"coordinates"`` section is optional and may be used by Traffic Router for localization in the case of a CZF "hit" where the zone name does not map to a Cache Group name in Traffic Ops (i.e. Traffic Router will route to the closest Cache Group(s) geographically). The ``"backupZones"`` section is optional and is used by Traffic Router for localization in the case of a CZF "hit" when there are no caches available for that DS in the matched cache group.  This backup "list" contains an ordered list of backup cache groups to choose from if the matched cache group has no caches available for a requested DS. If an available cache cannot be found in any of the backup groups either, the "fallbackToClosestGroup" flag determines the Traffic Router's following behavior. If true, Traffic Router will find the next closest cache group with available caches. If false (the default), Traffic Router will bypass the request (if configured) or reject it.
 
 .. _rl-deep-czf:
 

docs/source/admin/traffic_router.rst

@@ -246,6 +246,8 @@ Fields Always Present
 +--------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+
 |GEO_NO_CACHE_FOUND        |Traffic Router could not find a resource via geolocation data based on the requesting client's geolocation                                  |
 +--------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+
+|DS_CZ_BACKUP_CG           |Traffic Router found a cache from backup cache groups configured via backupZones / coordinates in Coverage zone data                        |
++--------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+
 
 ---------------
 

traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/loc/NetworkNode.java

@@ -49,6 +49,8 @@ public class NetworkNode implements Comparable<NetworkNode> {
     private String loc;
     private CacheLocation cacheLocation = null;
     private Geolocation geolocation = null;
+    private List<String> backupCacheGroups = null;
+    private boolean useClosestOnBackupFailure = false;
     protected Map<NetworkNode,NetworkNode> children;
     private Set<String> deepCacheNames;
 
@@ -106,15 +108,29 @@ public static NetworkNode generateTree(final JsonNode json, final boolean verify
                 final String loc = czIter.next();
                 final JsonNode locData = JsonUtils.getJsonNode(coverageZones, loc);
                 final JsonNode coordinates = locData.get("coordinates");
+                final JsonNode backupConfigJson = locData.get("backupZones");
+                boolean useClosestOnBackupFailure = true;
+
                 Geolocation geolocation = null;
+                List<String> backupCacheGroups = null;
 
                 if (coordinates != null && coordinates.has("latitude") && coordinates.has("longitude")) {
                     final double latitude = coordinates.get("latitude").asDouble();
                     final double longitude = coordinates.get("longitude").asDouble();
                     geolocation = new Geolocation(latitude, longitude);
                 }
 
-                if (!addNetworkNodesToRoot(root, loc, locData, geolocation, useDeep)) {
+                if (backupConfigJson != null) {
+                    if (backupConfigJson.has("list")) {
+                        backupCacheGroups = new ArrayList<>();
+                        for (final JsonNode cacheGroup : JsonUtils.getJsonNode(backupConfigJson, "list")) {
+                            backupCacheGroups.add(cacheGroup.asText());
+                        }
+                    }
+                    useClosestOnBackupFailure = JsonUtils.optBoolean(backupConfigJson, "fallbackToClosestGroup", false);
+                }
+
+                if (!addNetworkNodesToRoot(root, loc, locData, geolocation, backupCacheGroups, useDeep, useClosestOnBackupFailure)) {
                     return null;
                 }
             }
@@ -137,18 +153,19 @@ public static NetworkNode generateTree(final JsonNode json, final boolean verify
         return null;
     }
 
+
+    @SuppressWarnings({"PMD.CyclomaticComplexity", "PMD.NPathComplexity"})
     private static boolean addNetworkNodesToRoot(final SuperNode root, final String loc, final JsonNode locData,
-                                                 final Geolocation geolocation, final boolean useDeep) {
+                                                 final Geolocation geolocation, final List<String> backupCacheGroups, final boolean useDeep, final boolean useClosestOnBackupFailure) {
         final CacheLocation deepLoc = new CacheLocation( "deep." + loc, geolocation != null ? geolocation : new Geolocation(0.0, 0.0));  // TODO JvD
         final Set<String> cacheNames = parseDeepCacheNames(locData);
 
         for (final String key : new String[]{"network6", "network"}) {
             try {
                 for (final JsonNode network : JsonUtils.getJsonNode(locData, key)) {
                     final String ip = network.asText();
-
                     try {
-                        final NetworkNode nn = new NetworkNode(ip, loc, geolocation);
+                        final NetworkNode nn = new NetworkNode(ip, loc, geolocation, backupCacheGroups, useClosestOnBackupFailure);
                         if (useDeep) {
                             // For a deep NetworkNode, we set the CacheLocation here without any Caches.
                             // The deep Caches will be lazily loaded in getCoverageZoneCacheLocation() where we have
@@ -198,12 +215,14 @@ public NetworkNode(final String str) throws NetworkNodeException {
     }
 
     public NetworkNode(final String str, final String loc) throws NetworkNodeException {
-        this(str, loc, null);
+        this(str, loc, null, null, false);
     }
 
-    public NetworkNode(final String str, final String loc, final Geolocation geolocation) throws NetworkNodeException {
+    public NetworkNode(final String str, final String loc, final Geolocation geolocation, final List<String> backupCacheGroups, final boolean useClosestOnBackupFailure) throws NetworkNodeException {
         this.loc = loc;
         this.geolocation = geolocation;
+        this.backupCacheGroups = backupCacheGroups;
+        this.useClosestOnBackupFailure = useClosestOnBackupFailure;
         cidrAddress = CidrAddress.fromString(str);
     }
 
@@ -281,6 +300,14 @@ public Geolocation getGeolocation() {
         return geolocation;
     }
 
+    public List<String> getBackupCacheGroups() {
+        return backupCacheGroups;
+    }
+
+    public boolean isUseClosest() {
+        return useClosestOnBackupFailure;
+    }
+
     public CacheLocation getCacheLocation() {
         return cacheLocation;
     }

traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/router/StatTracker.java

@@ -116,7 +116,7 @@ public static enum ResultType {
 
 		public enum ResultDetails {
 			NO_DETAILS, DS_NOT_FOUND, DS_TLS_MISMATCH, DS_NO_BYPASS, DS_BYPASS, DS_CZ_ONLY, DS_CLIENT_GEO_UNSUPPORTED, GEO_NO_CACHE_FOUND,
-			REGIONAL_GEO_NO_RULE, REGIONAL_GEO_ALTERNATE_WITHOUT_CACHE, REGIONAL_GEO_ALTERNATE_WITH_CACHE
+			REGIONAL_GEO_NO_RULE, REGIONAL_GEO_ALTERNATE_WITHOUT_CACHE, REGIONAL_GEO_ALTERNATE_WITH_CACHE, DS_CZ_BACKUP_CG
 		}
 
 		long time;
@@ -130,6 +130,8 @@ public enum ResultDetails {
 		boolean isClientGeolocationQueried;
 
 		RegionalGeoResult regionalGeoResult;
+		boolean fromBackupCzGroup;
+		boolean useNextClosest = true;
 
 		public Track() {
 			start();
@@ -185,6 +187,22 @@ public RegionalGeoResult getRegionalGeoResult() {
 			return regionalGeoResult;
 		}
 
+		public void setFromBackupCzGroup(final boolean fromBackupCzGroup) {
+			this.fromBackupCzGroup = fromBackupCzGroup;
+		}
+
+		public boolean isFromBackupCzGroup() {
+			return fromBackupCzGroup;
+		}
+
+		public void setUseNextClosest(final boolean useNextClosest) {
+			this.useNextClosest = useNextClosest;
+		}
+
+		public boolean isUseNextClosest() {
+			return useNextClosest;
+		}
+
 		public final void start() {
 			time = System.currentTimeMillis();
 		}

traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/router/TrafficRouter.java

@@ -265,8 +265,9 @@ public List<Cache> getCachesByGeo(final DeliveryService ds, final Geolocation cl
 	protected List<Cache> selectCaches(final HTTPRequest request, final DeliveryService ds, final Track track) throws GeolocationException {
 		CacheLocation cacheLocation;
 		ResultType result = ResultType.CZ;
+		final boolean useDeep = (ds.getDeepCache() == DeliveryService.DeepCachingType.ALWAYS);
 
-		if (ds.getDeepCache() == DeliveryService.DeepCachingType.ALWAYS) {
+		if (useDeep) {
 			// Deep caching is enabled. See if there are deep caches available
 			cacheLocation = getDeepCoverageZoneCacheLocation(request.getClientIP(), ds);
 			if (cacheLocation != null && cacheLocation.getCaches().size() != 0) {
@@ -278,7 +279,7 @@ protected List<Cache> selectCaches(final HTTPRequest request, final DeliveryServ
 			}
 		} else {
 			// Deep caching not enabled for this Delivery Service; use the regular CZ
-			cacheLocation = getCoverageZoneCacheLocation(request.getClientIP(), ds);
+			cacheLocation = getCoverageZoneCacheLocation(request.getClientIP(), ds, track, useDeep);
 		}
 
 		List<Cache>caches = selectCachesByCZ(ds, cacheLocation, track, result);
@@ -295,15 +296,15 @@ protected List<Cache> selectCaches(final HTTPRequest request, final DeliveryServ
 				track.setResult(ResultType.MISS);
 				track.setResultDetails(ResultDetails.DS_CZ_ONLY);
 			}
-		} else {
+		} else if (track.isUseNextClosest()) {
+			//Even with Geo limit none, TR wont do geo look-up, if fallback is disabled via backupZones configuration
 			caches = selectCachesByGeo(request.getClientIP(), ds, cacheLocation, track);
 		}
 
 		return caches;
 	}
 
 	public List<Cache> selectCachesByGeo(final String clientIp, final DeliveryService deliveryService, final CacheLocation cacheLocation, final Track track) throws GeolocationException {
-
 		Geolocation clientLocation = null;
 
 		try {
@@ -365,7 +366,7 @@ public DNSRouteResult route(final DNSRequest request, final Track track) throws
 			return result;
 		}
 
-		final CacheLocation cacheLocation = getCoverageZoneCacheLocation(request.getClientIP(), ds);
+		final CacheLocation cacheLocation = getCoverageZoneCacheLocation(request.getClientIP(), ds, track, false);
 		List<Cache> caches = selectCachesByCZ(ds, cacheLocation, track);
 
 		if (caches != null) {
@@ -394,7 +395,9 @@ public DNSRouteResult route(final DNSRequest request, final Track track) throws
 			LOGGER.error("Bad client address: '" + request.getClientIP() + "'");
 		}
 
-		caches = selectCachesByGeo(request.getClientIP(), ds, cacheLocation, track);
+		if (track.isUseNextClosest()) {
+			caches = selectCachesByGeo(request.getClientIP(), ds, cacheLocation, track);
+		}
 
 		if (caches != null) {
 			track.setResult(ResultType.GEO);
@@ -486,6 +489,9 @@ private List<Cache> selectCachesByCZ(final DeliveryService ds, final CacheLocati
 
 		if (caches != null && track != null) {
 			track.setResult(result);
+			if (track.isFromBackupCzGroup()) {
+				track.setResultDetails(ResultDetails.DS_CZ_BACKUP_CG);
+			}
 			track.setResultLocation(cacheLocation.getGeolocation());
 		}
 
@@ -655,11 +661,11 @@ protected NetworkNode getNetworkNode(final String ip) {
 	}
 
 	public CacheLocation getCoverageZoneCacheLocation(final String ip, final String deliveryServiceId) {
-		return getCoverageZoneCacheLocation(ip, deliveryServiceId, false); // default is not deep
+		return getCoverageZoneCacheLocation(ip, deliveryServiceId, null, false); // default is not deep
 	}
 
 	@SuppressWarnings({"PMD.CyclomaticComplexity", "PMD.NPathComplexity"})
-	public CacheLocation getCoverageZoneCacheLocation(final String ip, final String deliveryServiceId, final boolean useDeep) {
+	public CacheLocation getCoverageZoneCacheLocation(final String ip, final String deliveryServiceId, final Track track, final boolean useDeep) {
 		final NetworkNode networkNode = useDeep ? getDeepNetworkNode(ip) : getNetworkNode(ip);
 
 		if (networkNode == null) {
@@ -695,17 +701,48 @@ public CacheLocation getCoverageZoneCacheLocation(final String ip, final String
 			return cacheLocation;
 		}
 
-		// We had a hit in the CZF but the name does not match a known cache location.
-		// Check whether the CZF entry has a geolocation and use it if so.
-		return getClosestCacheLocation(cacheRegister.filterAvailableLocations(deliveryServiceId), networkNode.getGeolocation(), cacheRegister.getDeliveryService(deliveryServiceId));
+		if (networkNode.getBackupCacheGroups() != null) {
+			for (final String cacheGroup : networkNode.getBackupCacheGroups()) {
+				final CacheLocation bkCacheLocation = getCacheRegister().getCacheLocationById(cacheGroup);
+				if (bkCacheLocation != null && !getSupportingCaches(bkCacheLocation.getCaches(), deliveryService).isEmpty()) {
+					LOGGER.debug("Got backup CZ cache group " + bkCacheLocation.getId() + " for " + ip + ", ds " + deliveryServiceId);
+					if (track != null) {
+						track.setFromBackupCzGroup(true);
+					}
+					return bkCacheLocation;
+				}
+			}
+		} 
+
+		CacheLocation closestCacheLocation = null;
+
+		if (networkNode.isUseClosest()) {
+			// We had a hit in the CZF but the name does not match a known cache location.
+			// Check whether the CZF entry has a geolocation and use it if so.
+			closestCacheLocation = getClosestCacheLocation(cacheRegister.filterAvailableLocations(deliveryServiceId), networkNode.getGeolocation(), cacheRegister.getDeliveryService(deliveryServiceId));
+			if (closestCacheLocation != null) {
+				LOGGER.debug("Got closest CZ cache group " + closestCacheLocation.getId() + " for " + ip + ", ds " + deliveryServiceId);
+				if (track != null) {
+					track.setFromBackupCzGroup(true);
+
+				}
+				return closestCacheLocation;
+			}  
+		} else {
+			if (track != null) {
+				track.setUseNextClosest(false);
+			}
+		} 
+
+		return closestCacheLocation;
 	}
 
 	public CacheLocation getDeepCoverageZoneCacheLocation(final String ip, final DeliveryService deliveryService) {
-		return getCoverageZoneCacheLocation(ip, deliveryService, true);
+		return getCoverageZoneCacheLocation(ip, deliveryService, null, true);
 	}
 
-	protected CacheLocation getCoverageZoneCacheLocation(final String ip, final DeliveryService deliveryService, final boolean useDeep) {
-		return getCoverageZoneCacheLocation(ip, deliveryService.getId(), useDeep);
+	protected CacheLocation getCoverageZoneCacheLocation(final String ip, final DeliveryService deliveryService, final Track track, final boolean useDeep) {
+		return getCoverageZoneCacheLocation(ip, deliveryService.getId(), track, useDeep);
 	}
 
 	protected CacheLocation getCoverageZoneCacheLocation(final String ip, final DeliveryService deliveryService) {

traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/loc/CoverageZoneTest.java

@@ -76,7 +76,7 @@ public void before() throws Exception {
 		cacheGroups.add(eastCacheGroup);
 		cacheGroups.add(westCacheGroup);
 
-		NetworkNode eastNetworkNode = new NetworkNode("12.23.34.0/24", "east-cache-group", testLocation);
+		NetworkNode eastNetworkNode = new NetworkNode("12.23.34.0/24", "east-cache-group", testLocation, null, true);
 
 		CacheRegister cacheRegister = mock(CacheRegister.class);
 
@@ -88,7 +88,7 @@ public void before() throws Exception {
 		trafficRouter = PowerMockito.mock(TrafficRouter.class);
 		Whitebox.setInternalState(trafficRouter, "cacheRegister", cacheRegister);
 		when(trafficRouter.getCoverageZoneCacheLocation("12.23.34.45", "delivery-service-1")).thenCallRealMethod();
-		when(trafficRouter.getCoverageZoneCacheLocation("12.23.34.45", "delivery-service-1", false)).thenCallRealMethod();
+		when(trafficRouter.getCoverageZoneCacheLocation("12.23.34.45", "delivery-service-1", null, false)).thenCallRealMethod();
 		when(trafficRouter.getCacheRegister()).thenReturn(cacheRegister);
 		when(trafficRouter.orderCacheLocations(cacheGroups,testLocation)).thenCallRealMethod();
 		when(trafficRouter.getSupportingCaches(anyListOf(Cache.class), eq(deliveryService))).thenCallRealMethod();