diff --git a/CHANGELOG.md b/CHANGELOG.md index 09998ca623..f976a40d0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - /api/1.3/types `(GET,POST,PUT,DELETE)` - Fair Queuing Pacing: Using the FQ Pacing Rate parameter in Delivery Services allows operators to limit the rate of individual sessions to the edge cache. This feature requires a Trafficserver RPM containing the fq_pacing experimental plugin AND setting 'fq' as the default Linux qdisc in sysctl. - Traffic Ops rpm changed to remove world-read permission from configuration files. +- Backup Edge Cache group: If the matched group in the CZF is not available, this list of backup edge cache group configured via Traffic Ops API can be used as backup. In the event of all backup edge cache groups not available, GEO location can be optionally used as further backup. APIs detailed here [here](http://traffic-control-cdn.readthedocs.io/en/latest/development/traffic_ops_api/v12/cachegroup_fallbacks.html) ### Changed - Reformatted this CHANGELOG file to the keep-a-changelog format diff --git a/docs/source/admin/traffic_router.rst b/docs/source/admin/traffic_router.rst index 963fe1516f..b8ed3ba470 100644 --- a/docs/source/admin/traffic_router.rst +++ b/docs/source/admin/traffic_router.rst @@ -200,7 +200,7 @@ Fields Always Present +------+---------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ |rloc |GeoLocation of result |Latitude and Longitude in Decimal Degrees | +------+---------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ -|rdtl |Result Details Associated with unusual conditions |One of DS_NOT_FOUND, DS_NO_BYPASS, DS_BYPASS, DS_CZ_ONLY | +|rdtl |Result Details Associated with unusual conditions |One of DS_NOT_FOUND, DS_NO_BYPASS, DS_BYPASS, DS_CZ_ONLY, DS_CZ_BACKUP_CG | +------+---------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ |rerr |Message about internal Traffic Router Error |String | +------+---------------------------------------------------------------------------------+------------------------------------------------------------------------------------+ @@ -266,8 +266,8 @@ Fields Always Present +---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+ | "-" |The request was not redirected. This is usually a result of a DNS request to the Traffic Router or an explicit denial for that request. | +---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+ - - +|DS_CZ_BACKUP_CG |Traffic Router found a backup cache via fallback (cr-config's edgeLocation) / coordinates (CZF) configuration | ++---------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+ --------------- diff --git a/docs/source/api/v12/cachegroup.rst b/docs/source/api/v12/cachegroup.rst index 1831f49602..2c6d8f9832 100644 --- a/docs/source/api/v12/cachegroup.rst +++ b/docs/source/api/v12/cachegroup.rst @@ -23,7 +23,7 @@ Cache Group /api/1.2/cachegroups ++++++++++++++++++++ -**GET /api/1.1/cachegroups** +**GET /api/1.2/cachegroups** Authentication Required: Yes @@ -66,6 +66,8 @@ Cache Group +-----------------------------------+--------+--------------------------------------------------------------------------+ | ``typeName`` | string | The name of the type of Cache Group entry | +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``fallbackToClosest`` | bool | Behaviour during non-availability/ failure of configured fallbacks | + +-----------------------------------+--------+--------------------------------------------------------------------------+ **Response Example** :: @@ -83,7 +85,8 @@ Cache Group "secondaryParentCachegroupName": null, "shortName": "dcchi", "typeName": "MID_LOC", - "typeId": "4" + "typeId": "4", + "fallbackToClosest":true }, { "id": "22", @@ -97,7 +100,8 @@ Cache Group "secondaryParentCachegroupName": null, "shortName": "dcchi", "typeName": "MID_LOC", - "typeId": "4" + "typeId": "4", + "fallbackToClosest":false } ], } @@ -168,6 +172,8 @@ Cache Group +-----------------------------------+--------+--------------------------------------------------------------------------+ | ``typeName`` | string | The name of the type of Cache Group entry | +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``fallbackToClosest`` | bool | Behaviour during non-availability/ failure of configured fallbacks | + +-----------------------------------+--------+--------------------------------------------------------------------------+ **Response Example** :: @@ -185,7 +191,8 @@ Cache Group "secondaryParentCachegroupName": null, "shortName": "dcchi", "typeName": "MID_LOC", - "typeId": "4" + "typeId": "4", + "fallbackToClosest":true } ], } @@ -448,6 +455,8 @@ Cache Group +---------------------------------+----------+-------------------------------------------------------------------+ | ``typeId`` | yes | The type of Cache Group entry, "EDGE_LOC", "MID_LOC" or "ORG_LOC" | +---------------------------------+----------+-------------------------------------------------------------------+ + | ``fallbackToClosest`` | no | Behaviour on configured fallbacks failure, true / false | + +---------------------------------+----------+-------------------------------------------------------------------+ **Request Example** :: @@ -457,7 +466,8 @@ Cache Group "latitude": 12, "longitude": 45, "parentCachegroup": "cache_group_mid", - "typeId": 6 + "typeId": 6, + "fallbackToClosest":true } **Response Properties** @@ -485,6 +495,8 @@ Cache Group +------------------------------------+--------+-------------------------------------------------------------------+ | ``typeName`` | string | The type of Cache Group entry, "EDGE_LOC", "MID_LOC" or "ORG_LOC" | +------------------------------------+--------+-------------------------------------------------------------------+ + | ``fallbackToClosest`` | bool | Behaviour during non-availability/failure of configured fallbacks | + +------------------------------------+--------+-------------------------------------------------------------------+ | ``lastUpdated`` | string | The Time / Date this entry was last updated | +------------------------------------+--------+-------------------------------------------------------------------+ | ``alerts`` | array | A collection of alert messages. | @@ -514,7 +526,8 @@ Cache Group 'typeName' : 'EDGE_LOC', 'id' : '104', 'parentCachegroupId' : '103', - 'secondaryParentCachegroupId' : null + 'secondaryParentCachegroupId' : null, + 'fallbackToClosest':true } } @@ -555,6 +568,8 @@ Cache Group +---------------------------------+----------+-------------------------------------------------------------------+ | ``typeName`` | yes | The type of Cache Group entry, "EDGE_LOC", "MID_LOC" or "ORG_LOC" | +---------------------------------+----------+-------------------------------------------------------------------+ + | ``fallbackToClosest`` | no | Behaviour on configured fallbacks failure, true / false | + +---------------------------------+----------+-------------------------------------------------------------------+ **Request Example** :: @@ -564,7 +579,8 @@ Cache Group "latitude": 12, "longitude": 45, "parentCachegroup": "cache_group_mid", - "typeName": "EDGE_LOC" + "typeName": "EDGE_LOC", + "fallbackToClosest":true } **Response Properties** @@ -592,6 +608,8 @@ Cache Group +------------------------------------+--------+-------------------------------------------------------------------+ | ``typeName`` | string | The type of Cache Group entry, "EDGE_LOC", "MID_LOC" or "ORG_LOC" | +------------------------------------+--------+-------------------------------------------------------------------+ + | ``fallbackToClosest`` | bool | Behaviour during non-availability/failure of configured fallbacks | + +------------------------------------+--------+-------------------------------------------------------------------+ | ``lastUpdated`` | string | The Time / Date this entry was last updated | +------------------------------------+--------+-------------------------------------------------------------------+ | ``alerts`` | array | A collection of alert messages. | @@ -621,7 +639,8 @@ Cache Group 'typeName' : 'EDGE_LOC', 'id' : '104', 'parentCachegroupId' : '103', - 'secondaryParentCachegroupId' : null + 'secondaryParentCachegroupId' : null, + 'fallbackToClosest':true } } diff --git a/docs/source/api/v12/cachegroup_fallbacks.rst b/docs/source/api/v12/cachegroup_fallbacks.rst new file mode 100644 index 0000000000..cabc4500b8 --- /dev/null +++ b/docs/source/api/v12/cachegroup_fallbacks.rst @@ -0,0 +1,288 @@ +.. +.. +.. Licensed under the Apache License, Version 2.0 (the "License"); +.. you may not use this file except in compliance with the License. +.. You may obtain a copy of the License at +.. +.. http://www.apache.org/licenses/LICENSE-2.0 +.. +.. Unless required by applicable law or agreed to in writing, software +.. distributed under the License is distributed on an "AS IS" BASIS, +.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +.. See the License for the specific language governing permissions and +.. limitations under the License. +.. + +.. _to-api-v12-cachegroupfallbacks: + +Cache Group Fallback +==================== + +.. _to-api-v12-cachegroupfallbacks-route: + +/api/1.2/cachegroup_fallbacks +++++++++++++++++++++++++++++++ + +**GET /api/1.2/cachegroup_fallbacks?cacheGroupId={id}** +**GET /api/1.2/cachegroup_fallbacks?fallbackId={id}** +**GET /api/1.2/cachegroup_fallbacks?cacheGroupId={id}&fallbackId={id}** + + Retrieve fallback related configurations for a cache group. + + Authentication Required: Yes + + Role(s) Required: None + + **Request Query Parameters** + + Query parameter is mandatory. Either one of the parameters must be used. Both can also be used simultaneously. + + +-----------------+---------------------------------------------------------------------------+ + | Name | Description | + +=================+===========================================================================+ + | cacheGroupId | The id of the cache group whose backup configurations has to be retrieved | + +-----------------+---------------------------------------------------------------------------+ + | fallbackId | The id of the fallback cache group associated with a cache group | + +-----------------+---------------------------------------------------------------------------+ + + **Response Properties** + + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | Parameter | Type | Description | + +===================================+========+==========================================================================+ + | | array | parameters array | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>cacheGroupId`` | int | Cache group id | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>fallbackId`` | int | fallback cache group id | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>cacheGroupName`` | string | Cache group name | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>fallbackName`` | string | Fallback cache group name | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>fallbackOrder`` | int | Ordering list in the list of backups | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + + **Response Example** :: + + { + "response": [ + { + "cacheGroupId":1, + "cacheGroupName":"GROUP1", + "fallbackId":2, + "fallbackOrder":10, + "fallbackName":"GROUP2" + } + ] + } + +| + +**POST /api/1.2/cachegroup_fallbacks** + + Creates fallback configuration for the cache group. New fallbacks can be added only via POST. + + Authentication Required: Yes + + Role(s) Required: admin or oper + + **Request Parameters** + The request parameters should be in array format. + + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | Parameter | Type | Description | + +===================================+========+==========================================================================+ + | | array | parameters array | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>cacheGroupId`` | int | Cache group id | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>fallbackId`` | int | Fallback cache group id | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>fallbackOrder`` | int | Ordering list in the list of backups | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + + **Request Example** :: + + [ + { + "cacheGroupId": 1, + "fallbackId": 3, + "fallbackOrder": 10 + } + ] + + **Response Properties** + + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | Parameter | Type | Description | + +===================================+========+==========================================================================+ + | | array | parameters array | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>cacheGroupId`` | int | Cache group id | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>fallbackId`` | int | fallback cache group id | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>cacheGroupName`` | string | Cache group name | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>fallbackName`` | string | Fallback cache group name | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>fallbackOrder`` | int | Ordering list in the list of backups | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``alerts`` | array | A collection of alert messages. | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>level`` | string | Success, info, warning or error. | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>text`` | string | Alert message. | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + + + **Response Example** :: + + { + "alerts": [ + { + "level":"success", + "text":"Backup configuration CREATE for cache group 1 successful." + } + ], + "response": [ + { + "cacheGroupId":1, + "cacheGroupName":"GROUP1", + "fallbackId":3, + "fallbackName":"GROUP2", + "fallbackorder":10, + } + ] + } + +| + +**PUT /api/1.2/cachegroup_fallbacks** + + Updates an existing fallback configuration for the cache group. + + Authentication Required: Yes + + Role(s) Required: admin or oper + + **Request Parameters** + The request parameters should be in array format. + + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | Parameter | Type | Description | + +===================================+========+==========================================================================+ + | | array | parameters array | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>cacheGroupId`` | int | Cache group id | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>fallbackId`` | int | Fallback cache group id | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>fallbackOrder`` | int | Ordering list in the list of backups | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + + **Request Example** :: + + [ + { + "cacheGroupId": 1, + "fallbackId": 3, + "fallbackOrder": 10 + } + ] + + **Response Properties** + + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | Parameter | Type | Description | + +===================================+========+==========================================================================+ + | | array | parameters array | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>cacheGroupId`` | int | Cache group id | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>fallbackId`` | int | fallback cache group id | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>cacheGroupName`` | string | Cache group name | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>fallbackName`` | string | Fallback cache group name | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>fallbackOrder`` | int | Ordering list in the list of backups | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``alerts`` | array | A collection of alert messages. | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>level`` | string | Success, info, warning or error. | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + | ``>text`` | string | Alert message. | + +-----------------------------------+--------+--------------------------------------------------------------------------+ + + + **Response Example** :: + + { + "alerts": [ + { + "level":"success", + "text":"Backup configuration UPDATE for cache group 1 successful." + } + ], + "response": [ + { + "cacheGroupId":1, + "cacheGroupName":"GROUP1", + "fallbackId":3, + "fallbackName":"GROUP2", + "fallbackorder":10, + } + ] + } + +| + +**DELETE /api/1.2/cachegroup_fallbacks?cacheGroupId={id}** +**DELETE /api/1.2/cachegroup_fallbacks?fallbackId={id}** +**DELETE /api/1.2/cachegroup_fallbacks?fallbackId={id}&cacheGroupId={id}** + + Delete fallback list assigned to the cache group. + + Authentication Required: Yes + + Role(s) Required: admin or oper + + **Request Query Parameters** + + Query parameter is mandatory. Either one of the parameters must be used. Both can also be used simultaneously. + + +-----------------+----------+--------------------------------------------------------------------------------------+ + | Name | Required | Description | + +=================+==========+======================================================================================+ + | cacheGroupId | Yes | The id of the cache group whose backup configurations has to be deleted | + +-----------------+----------+--------------------------------------------------------------------------------------+ + | fallbackId | Yes | The id of the fallback cachegroup which has to be deleted from the list of fallbacks | + +-----------------+----------+--------------------------------------------------------------------------------------+ + + **Response Properties** + + +-------------+--------+----------------------------------+ + | Parameter | Type | Description | + +=============+========+==================================+ + | ``alerts`` | array | A collection of alert messages. | + +-------------+--------+----------------------------------+ + | ``>level`` | string | Success, info, warning or error. | + +-------------+--------+----------------------------------+ + | ``>text`` | string | Alert message. | + +-------------+--------+----------------------------------+ + + **Response Example** :: + + { + "alerts": [ + { + "level": "success", + "text": "Backup configuration DELETED" + } + ], + } + +| + diff --git a/docs/source/api/v12/index.rst b/docs/source/api/v12/index.rst index 15e839c882..b60943d975 100644 --- a/docs/source/api/v12/index.rst +++ b/docs/source/api/v12/index.rst @@ -26,6 +26,7 @@ Traffic Ops API V1.2 cache cachegroup cachegroup_parameter + cachegroup_fallbacks cache_stats capability cdn diff --git a/lib/go-tc/cachegroupfallback.go b/lib/go-tc/cachegroupfallback.go new file mode 100644 index 0000000000..ecd1d8c3f4 --- /dev/null +++ b/lib/go-tc/cachegroupfallback.go @@ -0,0 +1,55 @@ +package tc + + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// A List of CacheGroupFallbacks Response +// swagger:response CacheGroupFallbacksResponse +// in: body +type CacheGroupFallbacksResponse struct { + // in: body + Response []CacheGroupFallback `json:"response"` +} + +// A Single CacheGroupFallback Response for Update and Create to depict what changed +// swagger:response CacheGroupFallbackResponse +// in: body +type CacheGroupFallbackResponse struct { + // in: body + Response CacheGroupFallback `json:"response"` +} + +// CacheGroupFallback ... +type CacheGroupFallback struct { + + PrimaryCgId int `json:"primaryId" db:"primary_cg"` + BackupCgId int `json:"backupId" db:"backup_cg"` + SetOrder int `json:"setOrder" db:"set_order"` + +} + +// CacheGroupFallbackNullable ... +type CacheGroupFallbackNullable struct { + + PrimaryCgId *int `json:"primaryId" db:"primary_cg"` + BackupCgId *int `json:"backupId" db:"backup_cg"` + SetOrder *int `json:"setOrder" db:"set_order"` +} + diff --git a/lib/go-tc/crconfig.go b/lib/go-tc/crconfig.go index 66832714d2..18133bf0c4 100644 --- a/lib/go-tc/crconfig.go +++ b/lib/go-tc/crconfig.go @@ -151,9 +151,18 @@ type CRConfigDispersion struct { Shuffled bool `json:"shuffled,string"` } + +type CRConfigBackupLocations struct { + FallbackToClosest bool `json:"fallbackToClosest,string"` + List []string `json:"list,omitempty"` + +} + type CRConfigLatitudeLongitude struct { - Lat float64 `json:"latitude"` - Lon float64 `json:"longitude"` + Lat float64 `json:"latitude"` + Lon float64 `json:"longitude"` + BackupLocations CRConfigBackupLocations `json:"backupLocations,omitempty"` + } type CRConfigLatitudeLongitudeShort struct { diff --git a/lib/go-tc/v13/cachegroups.go b/lib/go-tc/v13/cachegroups.go index ee203fcd5c..1c0f550185 100644 --- a/lib/go-tc/v13/cachegroups.go +++ b/lib/go-tc/v13/cachegroups.go @@ -37,6 +37,7 @@ type CacheGroup struct { ParentCachegroupID int `json:"parentCachegroupId" db:"parent_cachegroup_id"` SecondaryParentName string `json:"secondaryParentCachegroupName"` SecondaryParentCachegroupID int `json:"secondaryParentCachegroupId" db:"secondary_parent_cachegroup_id"` + FallbackToClosest bool `json:"fallbackToClosest" db:"fallback_to_closest` Type string `json:"typeName" db:"type_name"` // aliased to type_name to disambiguate struct scans due to join on 'type' table TypeID int `json:"typeId" db:"type_id"` // aliased to type_id to disambiguate struct scans due join on 'type' table LastUpdated tc.TimeNoMod `json:"lastUpdated" db:"last_updated"` @@ -52,6 +53,7 @@ type CacheGroupNullable struct { ParentCachegroupID *int `json:"parentCachegroupId" db:"parent_cachegroup_id"` SecondaryParentName *string `json:"secondaryParentCachegroupName"` SecondaryParentCachegroupID *int `json:"secondaryParentCachegroupId" db:"secondary_parent_cachegroup_id"` + FallbackToClosest *bool `json:"fallbackToClosest" db:"fallback_to_closest"` Type *string `json:"typeName" db:"type_name"` // aliased to type_name to disambiguate struct scans due to join on 'type' table TypeID *int `json:"typeId" db:"type_id"` // aliased to type_id to disambiguate struct scans due join on 'type' table LastUpdated *tc.TimeNoMod `json:"lastUpdated" db:"last_updated"` diff --git a/traffic_ops/app/db/migrations/20180528000000_cache_group_fallback.sql b/traffic_ops/app/db/migrations/20180528000000_cache_group_fallback.sql new file mode 100644 index 0000000000..dbce434efd --- /dev/null +++ b/traffic_ops/app/db/migrations/20180528000000_cache_group_fallback.sql @@ -0,0 +1,38 @@ +/* + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +-- +goose Up +-- SQL in section 'Up' is executed when this migration is applied + + +-- cachegroup_fallbacks +CREATE TABLE cachegroup_fallbacks ( + primary_cg bigint NOT NULL, + backup_cg bigint CHECK (primary_cg != backup_cg) NOT NULL, + set_order bigint NOT NULL, + CONSTRAINT fk_primary_cg FOREIGN KEY (primary_cg) REFERENCES cachegroup(id) ON DELETE CASCADE, + CONSTRAINT fk_backup_cg FOREIGN KEY (backup_cg) REFERENCES cachegroup(id) ON DELETE CASCADE, + UNIQUE (primary_cg, backup_cg), + UNIQUE (primary_cg, set_order) +); + +ALTER TABLE cachegroup ADD COLUMN fallback_to_closest BOOLEAN DEFAULT TRUE; + +-- +goose Down +-- SQL section 'Down' is executed when this migration is rolled back + +ALTER TABLE cachegroup DROP COLUMN fallback_to_closest; + +DROP TABLE cachegroup_fallbacks; diff --git a/traffic_ops/app/lib/API/Cachegroup.pm b/traffic_ops/app/lib/API/Cachegroup.pm index 8d0f4f03cf..db430b2f80 100644 --- a/traffic_ops/app/lib/API/Cachegroup.pm +++ b/traffic_ops/app/lib/API/Cachegroup.pm @@ -57,6 +57,7 @@ sub index { "lastUpdated" => $row->last_updated, "parentCachegroupId" => $row->parent_cachegroup_id, "parentCachegroupName" => ( defined $row->parent_cachegroup_id ) ? $idnames{ $row->parent_cachegroup_id } : undef, + "fallbackToClosest" => \$row->fallback_to_closest, "secondaryParentCachegroupId" => $row->secondary_parent_cachegroup_id, "secondaryParentCachegroupName" => ( defined $row->secondary_parent_cachegroup_id ) ? $idnames{ $row->secondary_parent_cachegroup_id } @@ -110,6 +111,7 @@ sub show { "lastUpdated" => $row->last_updated, "parentCachegroupId" => $row->parent_cachegroup_id, "parentCachegroupName" => ( defined $row->parent_cachegroup_id ) ? $idnames{ $row->parent_cachegroup_id } : undef, + "fallbackToClosest" => \$row->fallback_to_closest, "secondaryParentCachegroupId" => $row->secondary_parent_cachegroup_id, "secondaryParentCachegroupName" => ( defined $row->secondary_parent_cachegroup_id ) ? $idnames{ $row->secondary_parent_cachegroup_id } @@ -158,12 +160,18 @@ sub update { } } + my $fallback_to_closest = $params->{fallbackToClosest}; + if ( !defined ($fallback_to_closest) ) { + $fallback_to_closest = $cachegroup->fallback_to_closest; + } + my $values = { name => $params->{name}, short_name => $params->{shortName}, latitude => $params->{latitude}, longitude => $params->{longitude}, parent_cachegroup_id => $params->{parentCachegroupId}, + fallback_to_closest => $fallback_to_closest, secondary_parent_cachegroup_id => $params->{secondaryParentCachegroupId}, type => $params->{typeId} }; @@ -189,6 +197,7 @@ sub update { ( defined $rs->parent_cachegroup_id ) ? $idnames{ $rs->parent_cachegroup_id } : undef; + $response->{fallbackToClosest} = $rs->fallback_to_closest; $response->{secondaryParentCachegroupId} = $rs->secondary_parent_cachegroup_id; $response->{secondaryParentCachegroupName} = ( defined $rs->secondary_parent_cachegroup_id ) @@ -239,6 +248,7 @@ sub create { latitude => $params->{latitude}, longitude => $params->{longitude}, parent_cachegroup_id => $params->{parentCachegroupId}, + fallback_to_closest => exists ($params->{fallbackToClosest}) ? $params->{fallbackToClosest} : 1,# defaults to true secondary_parent_cachegroup_id => $params->{secondaryParentCachegroupId}, type => $params->{typeId} }; @@ -265,6 +275,7 @@ sub create { ( defined $rs->parent_cachegroup_id ) ? $idnames{ $rs->parent_cachegroup_id } : undef; + $response->{fallbackToClosest} = $rs->fallback_to_closest; $response->{secondaryParentCachegroupId} = $rs->secondary_parent_cachegroup_id; $response->{secondaryParentCachegroupName} = ( defined $rs->secondary_parent_cachegroup_id ) diff --git a/traffic_ops/app/lib/API/CachegroupFallback.pm b/traffic_ops/app/lib/API/CachegroupFallback.pm new file mode 100644 index 0000000000..a9114dac57 --- /dev/null +++ b/traffic_ops/app/lib/API/CachegroupFallback.pm @@ -0,0 +1,282 @@ +package API::CachegroupFallback; +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# +# a note about locations and cachegroups. This used to be "Location", before we had physical locations in 12M. Very confusing. +# What used to be called a location is now called a "cache group" and location is now a physical address, not a group of caches working together. +# + +# JvD Note: you always want to put Utils as the first use. Sh*t don't work if it's after the Mojo lines. +use UI::Utils; +use Mojo::Base 'Mojolicious::Controller'; +use Data::Dumper; +use JSON; +use MojoPlugins::Response; +use Validate::Tiny ':all'; + +sub delete { + my $self = shift; + my $cache_id = $self->param('cacheGroupId'); + my $fallback_id = $self->param('fallbackId'); + my $params = $self->req->json; + my $rs_backups = undef; + my $result = ""; + + if ( !&is_oper($self) ) { + return $self->forbidden(); + } + + if ( defined ($cache_id) && defined($fallback_id) ) { + $rs_backups = $self->db->resultset('CachegroupFallback')->search( { primary_cg => $cache_id , backup_cg => $fallback_id} ); + $result = "Backup Cachegroup $fallback_id DELETED from cachegroup $cache_id fallback list"; + } elsif (defined ($cache_id)) { + $rs_backups = $self->db->resultset('CachegroupFallback')->search( { primary_cg => $cache_id} ); + $result = "Fallback list for Cachegroup $cache_id DELETED"; + } elsif (defined ($fallback_id)) { + $result = "Cachegroup $fallback_id DELETED from all the configured fallback lists"; + $rs_backups = $self->db->resultset('CachegroupFallback')->search( { backup_cg => $fallback_id} ); + } else { + return $self->alert("Invalid input"); + } + + if ( ($rs_backups->count > 0) ) { + my $del_records = $rs_backups->delete(); + if ($del_records) { + &log( $self, $result, "APICHANGE"); + return $self->success( $result ); + } else { + return $self->alert( "Backup configuration DELETE Failed!." ); + } + } else { + $self->app->log->error( "No backup Cachegroups found" ); + return $self->not_found(); + } +} + +sub show { + my $self = shift; + my $cache_id = $self->param("cacheGroupId"); + my $fallback_id = $self->param("fallbackId"); + my $id = $cache_id ? $cache_id : $fallback_id; + + my ( $is_valid, $result ) = $self->is_valid_cachegroup_fallback(undef, $cache_id); + + if ( !$is_valid ) { + return $self->alert($result); + } + + my $rs_backups = undef; + + if ( defined ($cache_id) && defined ($fallback_id)) { + $rs_backups = $self->db->resultset('CachegroupFallback')->search({ primary_cg => $cache_id, backup_cg => $fallback_id}, {order_by => 'set_order'}); + } elsif ( defined ($cache_id) ) { + $rs_backups = $self->db->resultset('CachegroupFallback')->search({ primary_cg => $cache_id}, {order_by => 'set_order'}); + } elsif ( defined ($fallback_id) ) { + $rs_backups = $self->db->resultset('CachegroupFallback')->search({ backup_cg => $fallback_id}, {order_by => 'set_order'}); + } + + if ( defined ($rs_backups) && ($rs_backups->count > 0) ) { + my $response; + my $backup_cnt = 0; + while ( my $row = $rs_backups->next ) { + $response->[$backup_cnt]{"cacheGroupId"} = $row->primary_cg->id; + $response->[$backup_cnt]{"cacheGroupName"} = $row->primary_cg->name; + $response->[$backup_cnt]{"fallbackName"} = $row->backup_cg->name; + $response->[$backup_cnt]{"fallbackId"} = $row->backup_cg->id; + $response->[$backup_cnt]{"fallbackOrder"} = $row->set_order; + $backup_cnt++; + } + return $self->success( $response ); + } else { + $self->app->log->error("No backup Cachegroups"); + return $self->success([]); + } +} + +sub create { + my $self = shift; + my $cache_id = $self->param('cacheGroupId'); + my $params = $self->req->json; + + if ( !&is_oper($self) ) { + return $self->forbidden(); + } + + if ( !defined($params) ) { + return $self->alert("parameters must be in JSON format, please check!"); + } + + if ( !defined($cache_id)) { + my @param_array = @{$params}; + $cache_id = $param_array[0]{cacheGroupId}; + } + + my ( $is_valid, $result ) = $self->is_valid_cachegroup_fallback($params, $cache_id); + + if ( !$is_valid ) { + return $self->alert($result); + } + + foreach my $config (@{ $params }) { + my $rs_backup = $self->db->resultset('Cachegroup')->search( { id => $config->{fallbackId} } )->single(); + if ( !defined($rs_backup) ) { + $self->app->log->error("ERROR Backup config: No such Cache Group $config->{fallbackId}"); + next; + } + + if ( ($rs_backup->type->name ne "EDGE_LOC") ) { + $self->app->log->error("ERROR Backup config: $config->{name} is not EDGE_LOC"); + next; + } + + my $existing_row = $self->db->resultset('CachegroupFallback')->search( { primary_cg => $cache_id, backup_cg => $config->{fallbackId} } ); + if ( defined ($existing_row->next) ) { + next;#Skip existing rows + } + + my $values = { + primary_cg => $cache_id , + backup_cg => $config->{fallbackId}, + set_order => $config->{fallbackOrder} + }; + + my $rs_data = $self->db->resultset('CachegroupFallback')->create($values)->insert(); + if ( !defined($rs_data)) { + $self->app->log->error("Database operation for backup configuration for cache group $cache_id failed."); + } + } + + my $rs_backups = $self->db->resultset('CachegroupFallback')->search({ primary_cg => $cache_id}, {order_by => 'set_order'}); + my $response; + my $backup_cnt = 0; + if ( ($rs_backups->count > 0) ) { + while ( my $row = $rs_backups->next ) { + $response->[$backup_cnt]{"cacheGroupId"} = $cache_id; + $response->[$backup_cnt]{"cacheGroupName"} = $row->primary_cg->name; + $response->[$backup_cnt]{"fallbackName"} = $row->backup_cg->name; + $response->[$backup_cnt]{"fallbackId"} = $row->backup_cg->id; + $response->[$backup_cnt]{"fallbackOrder"} = $row->set_order; + $backup_cnt++; + } + &log( $self, "Backup configuration UPDATED for $cache_id", "APICHANGE"); + return $self->success( $response, "Backup configuration CREATE for cache group $cache_id successful." ); + } else { + return $self->alert("Backup configuration CREATE for cache group $cache_id Failed." ); + } +} + + +sub update { + my $self = shift; + my $cache_id = $self->param('cacheGroupId'); + my $params = $self->req->json; + + if ( !&is_oper($self) ) { + return $self->forbidden(); + } + + if ( !defined($params) ) { + return $self->alert("parameters must be in JSON format, please check!"); + } + + if ( !defined($cache_id)) { + my @param_array = @{$params}; + $cache_id = $param_array[0]{cacheGroupId}; + } + + my $rs_backups = $self->db->resultset('CachegroupFallback')->search( { primary_cg => $cache_id } ); + if ( !defined ($rs_backups->next) ) { + return $self->alert( "Backup list not configured for $cache_id, create and update" ); + } + + my ( $is_valid, $result ) = $self->is_valid_cachegroup_fallback($params, $cache_id); + + if ( !$is_valid ) { + return $self->alert($result); + } + + foreach my $config (@{ $params }) { + my $rs_backup = $self->db->resultset('Cachegroup')->search( { id => $config->{fallbackId} } )->single(); + if ( !defined($rs_backup) ) { + $self->app->log->error("ERROR Backup config: No such Cache Group $config->{fallbackId}"); + next; + } + + if ( ($rs_backup->type->name ne "EDGE_LOC") ) { + $self->app->log->error("ERROR Backup config: $config->{name} is not EDGE_LOC"); + next; + } + + my $values = { + primary_cg => $cache_id , + backup_cg => $config->{fallbackId}, + set_order => $config->{fallbackOrder} + }; + + my $existing_row = $self->db->resultset('CachegroupFallback')->search( { primary_cg => $cache_id, backup_cg => $config->{fallbackId} } ); + #New row creation disabled for PUT.Only existing rows can be updated + if ( defined ($existing_row->next) ) { + $existing_row->update($values); + } + } + + my $rs_backups = $self->db->resultset('CachegroupFallback')->search({ primary_cg => $cache_id}, {order_by => 'set_order'}); + my $response; + my $backup_cnt = 0; + if ( ($rs_backups->count > 0) ) { + while ( my $row = $rs_backups->next ) { + $response->[$backup_cnt]{"cacheGroupId"} = $cache_id; + $response->[$backup_cnt]{"cacheGroupName"} = $row->primary_cg->name; + $response->[$backup_cnt]{"fallbackName"} = $row->backup_cg->name; + $response->[$backup_cnt]{"fallbackId"} = $row->backup_cg->id; + $response->[$backup_cnt]{"fallbackOrder"} = $row->set_order; + $backup_cnt++; + } + &log( $self, "Backup configuration UPDATED for $cache_id", "APICHANGE"); + return $self->success( $response, "Backup configuration UPDATE for cache group $cache_id successful." ); + } else { + return $self->alert("Backup configuration UPDATE for cache group $cache_id Failed." ); + } +} + +sub is_valid_cachegroup_fallback { + my $self = shift; + my $params = shift; + my $cache_id = shift; + + if ( $cache_id !~ /^\d+?$/ ) { + return ( 0, "Invalid cachegroup id, should be an integer" ); + } + + my $cachegroup = $self->db->resultset('Cachegroup')->search( { id => $cache_id } )->single(); + if ( !defined($cachegroup) ) { + return ( 0, "Invalid cachegroup id" ); + } + + if ( ($cachegroup->type->name ne "EDGE_LOC") ) { + return ( 0, "cachegroup is not of type EDGE_LOC" ); + } + + foreach my $config (@{ $params }) { + if ( $config->{fallbackId} !~ /^\d+?$/ ) { + return ( 0, "Invalid cachegroup specified as fallback, should be an integer" ); + } + } + + return ( 1, "success" ); +} + +1; diff --git a/traffic_ops/app/lib/Schema/Result/Cachegroup.pm b/traffic_ops/app/lib/Schema/Result/Cachegroup.pm index fea22b3ee2..a93c039906 100644 --- a/traffic_ops/app/lib/Schema/Result/Cachegroup.pm +++ b/traffic_ops/app/lib/Schema/Result/Cachegroup.pm @@ -75,6 +75,12 @@ __PACKAGE__->table("cachegroup"); is_nullable: 1 original: {default_value => \"now()"} +=head2 fallback_to_closest + + data_type: 'boolean' + default_value: true + is_nullable: 1 + =cut __PACKAGE__->add_columns( @@ -106,6 +112,8 @@ __PACKAGE__->add_columns( is_nullable => 1, original => { default_value => \"now()" }, }, + "fallback_to_closest", + { data_type => "boolean", default_value => \"true", is_nullable => 1 }, ); =head1 PRIMARY KEY @@ -177,6 +185,36 @@ __PACKAGE__->has_many( { cascade_copy => 0, cascade_delete => 0 }, ); +=head2 cachegroup_fallbacks_backup_cgs + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "cachegroup_fallbacks_backup_cgs", + "Schema::Result::CachegroupFallback", + { "foreign.backup_cg" => "self.id" }, + { cascade_copy => 0, cascade_delete => 0 }, +); + +=head2 cachegroup_fallbacks_primary_cgs + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "cachegroup_fallbacks_primary_cgs", + "Schema::Result::CachegroupFallback", + { "foreign.primary_cg" => "self.id" }, + { cascade_copy => 0, cascade_delete => 0 }, +); + =head2 cachegroup_parameters Type: has_many diff --git a/traffic_ops/app/lib/Schema/Result/CachegroupFallback.pm b/traffic_ops/app/lib/Schema/Result/CachegroupFallback.pm new file mode 100644 index 0000000000..58046b3a88 --- /dev/null +++ b/traffic_ops/app/lib/Schema/Result/CachegroupFallback.pm @@ -0,0 +1,138 @@ +use utf8; +package Schema::Result::CachegroupFallback; + +# Created by DBIx::Class::Schema::Loader +# DO NOT MODIFY THE FIRST PART OF THIS FILE + +=head1 NAME + +Schema::Result::CachegroupFallback + +=cut + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + +=head1 TABLE: C + +=cut + +__PACKAGE__->table("cachegroup_fallbacks"); + +=head1 ACCESSORS + +=head2 primary_cg + + data_type: 'bigint' + is_foreign_key: 1 + is_nullable: 1 + +=head2 backup_cg + + data_type: 'bigint' + is_foreign_key: 1 + is_nullable: 1 + +=head2 set_order + + data_type: 'bigint' + is_nullable: 0 + +=cut + +__PACKAGE__->add_columns( + "primary_cg", + { data_type => "bigint", is_foreign_key => 1, is_nullable => 1 }, + "backup_cg", + { data_type => "bigint", is_foreign_key => 1, is_nullable => 1 }, + "set_order", + { data_type => "bigint", is_nullable => 0 }, +); + +=head1 UNIQUE CONSTRAINTS + +=head2 C + +=over 4 + +=item * L + +=item * L + +=back + +=cut + +__PACKAGE__->add_unique_constraint( + "cachegroup_fallbacks_primary_cg_backup_cg_key", + ["primary_cg", "backup_cg"], +); + +=head2 C + +=over 4 + +=item * L + +=item * L + +=back + +=cut + +__PACKAGE__->add_unique_constraint( + "cachegroup_fallbacks_primary_cg_set_order_key", + ["primary_cg", "set_order"], +); + +=head1 RELATIONS + +=head2 backup_cg + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "backup_cg", + "Schema::Result::Cachegroup", + { id => "backup_cg" }, + { + is_deferrable => 0, + join_type => "LEFT", + on_delete => "CASCADE", + on_update => "NO ACTION", + }, +); + +=head2 primary_cg + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "primary_cg", + "Schema::Result::Cachegroup", + { id => "primary_cg" }, + { + is_deferrable => 0, + join_type => "LEFT", + on_delete => "CASCADE", + on_update => "NO ACTION", + }, +); + + +# Created by DBIx::Class::Schema::Loader v0.07048 @ 2018-03-20 04:15:32 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:9bJ/JA5FNpy0LYu1KRdQqA + + +# You can replace this text with custom code or comments, and it will be preserved on regeneration +1; diff --git a/traffic_ops/app/lib/TrafficOpsRoutes.pm b/traffic_ops/app/lib/TrafficOpsRoutes.pm index 5c944b957f..a371442c1d 100644 --- a/traffic_ops/app/lib/TrafficOpsRoutes.pm +++ b/traffic_ops/app/lib/TrafficOpsRoutes.pm @@ -419,6 +419,13 @@ sub api_routes { $r->put("/api/$version/cachegroups/:id" => [ id => qr/\d+/ ] )->over( authenticated => 1, not_ldap => 1 )->to( 'Cachegroup#update', namespace => $namespace ); $r->delete("/api/$version/cachegroups/:id" => [ id => qr/\d+/ ] )->over( authenticated => 1, not_ldap => 1 )->to( 'Cachegroup#delete', namespace => $namespace ); + + # -- CACHEGROUP-Fallbacks: CRUD + $r->get("/api/$version/cachegroup_fallbacks")->over( authenticated => 1, not_ldap => 1 )->to( 'CachegroupFallback#show', namespace => $namespace ); + $r->post("/api/$version/cachegroup_fallbacks")->over( authenticated => 1, not_ldap => 1 )->to( 'CachegroupFallback#create', namespace => $namespace ); + $r->put("/api/$version/cachegroup_fallbacks")->over( authenticated => 1, not_ldap => 1 )->to( 'CachegroupFallback#update', namespace => $namespace ); + $r->delete("/api/$version/cachegroup_fallbacks")->over( authenticated => 1, not_ldap => 1 )->to( 'CachegroupFallback#delete', namespace => $namespace ); + # -- CACHEGROUPS: ASSIGN DELIVERYSERVICES $r->post("/api/$version/cachegroups/:id/deliveryservices" => [ id => qr/\d+/ ] )->over( authenticated => 1, not_ldap => 1 ) ->to( 'DeliveryServiceServer#assign_ds_to_cachegroup', namespace => $namespace ); diff --git a/traffic_ops/app/lib/UI/Topology.pm b/traffic_ops/app/lib/UI/Topology.pm index 2107baacdb..c87f8f0cd4 100644 --- a/traffic_ops/app/lib/UI/Topology.pm +++ b/traffic_ops/app/lib/UI/Topology.pm @@ -250,6 +250,15 @@ sub gen_crconfig_json { if ( $row->type->name =~ m/^EDGE/ ) { $data_obj->{'edgeLocations'}->{ $row->cachegroup->name }->{'latitude'} = $row->cachegroup->latitude + 0; $data_obj->{'edgeLocations'}->{ $row->cachegroup->name }->{'longitude'} = $row->cachegroup->longitude + 0; + $data_obj->{'edgeLocations'}->{ $row->cachegroup->name }->{'backupLocations'}->{'fallbackToClosest'} = $row->cachegroup->fallback_to_closest ? "true" : "false"; + + my $rs_backups = $self->db->resultset('CachegroupFallback')->search({ primary_cg => $row->cachegroup->id}, {order_by => 'set_order'}); + my $backup_cnt = 0; + + while ( my $backup_row = $rs_backups->next ) { + $data_obj->{'edgeLocations'}->{ $row->cachegroup->name }->{'backupLocations'}->{'list'}[$backup_cnt] = $backup_row->backup_cg->name; + $backup_cnt++; + } } if ( !exists $cache_tracker{ $row->id } ) { diff --git a/traffic_ops/traffic_ops_golang/crconfig/edgelocations.go b/traffic_ops/traffic_ops_golang/crconfig/edgelocations.go index 2d43fd6547..7f9cb96420 100644 --- a/traffic_ops/traffic_ops_golang/crconfig/edgelocations.go +++ b/traffic_ops/traffic_ops_golang/crconfig/edgelocations.go @@ -32,7 +32,7 @@ func makeLocations(cdn string, db *sql.DB) (map[string]tc.CRConfigLatitudeLongit // TODO test whether it's faster to do a single query, joining lat/lon into servers q := ` -select cg.name, t.name as type, cg.latitude, cg.longitude from cachegroup as cg +select cg.name, cg.id, t.name as type, cg.latitude, cg.longitude, cg.fallback_to_closest from cachegroup as cg inner join server as s on s.cachegroup = cg.id inner join type as t on t.id = s.type inner join status as st ON st.id = s.status @@ -49,14 +49,49 @@ and (st.name = 'REPORTED' or st.name = 'ONLINE' or st.name = 'ADMIN_DOWN') for rows.Next() { cachegroup := "" + primaryCacheID := 0 ttype := "" + var fallbackToClosest *bool latlon := tc.CRConfigLatitudeLongitude{} - if err := rows.Scan(&cachegroup, &ttype, &latlon.Lat, &latlon.Lon); err != nil { + if err := rows.Scan(&cachegroup, &primaryCacheID, &ttype, &latlon.Lat, &latlon.Lon, &fallbackToClosest); err != nil { return nil, nil, errors.New("Error scanning cachegroup: " + err.Error()) } if ttype == RouterTypeName { routerLocs[cachegroup] = latlon } else { + q := `select cachegroup.name from cachegroup_fallbacks +join cachegroup on cachegroup_fallbacks.backup_cg = cachegroup.id +and cachegroup_fallbacks.primary_cg = $1 order by cachegroup_fallbacks.set_order +` + dbRows, err := db.Query(q, primaryCacheID) + + if err != nil { + return nil, nil, errors.New("Error retrieving from cachegroup_fallbacks: " + err.Error()) + } + defer dbRows.Close() + + + if fallbackToClosest == nil { + fallbackToClosest = new(bool) + *fallbackToClosest = true + + } + latlon.BackupLocations.FallbackToClosest = *fallbackToClosest + + index := 0 + for dbRows.Next() { + backupName := "" + if err := dbRows.Scan(&backupName); err != nil { + return nil, nil, errors.New("Error while scanning from cachegroup_fallbacks: " + err.Error()) + } else { + latlon.BackupLocations.List = append(latlon.BackupLocations.List, backupName) + index++ + } + } + + if err := dbRows.Err(); err != nil { + return nil, nil, errors.New("Error iterating cachegroup_fallbacks rows: " + err.Error()) + } edgeLocs[cachegroup] = latlon } } diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/api/controllers/DeepCoverageZoneController.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/api/controllers/DeepCoverageZoneController.java index 6503bcb9c8..56d5ab06cc 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/api/controllers/DeepCoverageZoneController.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/api/controllers/DeepCoverageZoneController.java @@ -35,7 +35,7 @@ public class DeepCoverageZoneController { public @ResponseBody ResponseEntity getCacheLocationForIp(@RequestParam(name = "ip") final String ip, @RequestParam(name = "deliveryServiceId") final String deliveryServiceId) { - final CacheLocation cacheLocation = trafficRouterManager.getTrafficRouter().getCoverageZoneCacheLocation(ip, deliveryServiceId, true); + final CacheLocation cacheLocation = trafficRouterManager.getTrafficRouter().getCoverageZoneCacheLocation(ip, deliveryServiceId, true, null); if (cacheLocation == null) { return ResponseEntity.status(HttpStatus.NOT_FOUND).body(null); } diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/cache/CacheLocation.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/cache/CacheLocation.java index 67159c1162..9c2b93d8f5 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/cache/CacheLocation.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/cache/CacheLocation.java @@ -38,6 +38,8 @@ public class CacheLocation { private final Geolocation geolocation; private final Map caches; + private List backupCacheGroups = null; + private boolean useClosestGeoOnBackupFailure = true; /** * Creates a CacheLocation with the specified ID at the specified location. @@ -48,8 +50,28 @@ public class CacheLocation { * the coordinates of this location */ public CacheLocation(final String id, final Geolocation geolocation) { + this(id, geolocation, null, true); + } + + /** + * Creates a CacheLocation with the specified ID at the specified location. + * + * @param id + * the id of the location + * @param geolocation + * the coordinates of this location + * + * @param backupCacheGroups + * the backup cache groups for this id + * + * @param useClosestGeoOnBackupFailure + * the backup fallback setting for this id + */ + public CacheLocation(final String id, final Geolocation geolocation, final List backupCacheGroups, final boolean useClosestGeoOnBackupFailure) { this.id = id; this.geolocation = geolocation; + this.backupCacheGroups = backupCacheGroups; + this.useClosestGeoOnBackupFailure = useClosestGeoOnBackupFailure; caches = new HashMap(); } @@ -128,6 +150,24 @@ public Geolocation getGeolocation() { return geolocation; } + /** + * Gets backupCacheGroups. + * + * @return the backupCacheGroups + */ + public List getBackupCacheGroups() { + return backupCacheGroups; + } + + /** + * Tests useClosestGeoOnBackupFailure. + * + * @return useClosestGeoOnBackupFailure + */ + public boolean isUseClosestGeoLoc() { + return useClosestGeoOnBackupFailure; + } + /** * Gets id. * diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java index 2c33409e1a..6ab8fa1fdd 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/config/ConfigHandler.java @@ -672,8 +672,22 @@ private void parseLocationConfig(final JsonNode locationsJo, final CacheRegister while (locIter.hasNext()) { final String loc = locIter.next(); final JsonNode jo = JsonUtils.getJsonNode(locationsJo, loc); + List backupCacheGroups = null; + boolean useClosestOnBackupFailure = true; + + if (jo != null && jo.has("backupLocations")) { + final JsonNode backupConfigJson = JsonUtils.getJsonNode(jo, "backupLocations"); + backupCacheGroups = new ArrayList<>(); + if (backupConfigJson.has("list")) { + for (final JsonNode cacheGroup : JsonUtils.getJsonNode(backupConfigJson, "list")) { + backupCacheGroups.add(cacheGroup.asText()); + } + useClosestOnBackupFailure = JsonUtils.optBoolean(backupConfigJson, "fallbackToClosest", false); + } + + } try { - locations.add(new CacheLocation(loc, new Geolocation(JsonUtils.getDouble(jo, "latitude"), JsonUtils.getDouble(jo, "longitude")))); + locations.add(new CacheLocation(loc, new Geolocation(JsonUtils.getDouble(jo, "latitude"), JsonUtils.getDouble(jo, "longitude")), backupCacheGroups, useClosestOnBackupFailure)); } catch (JsonUtilsException e) { LOGGER.warn(e,e); } diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/router/StatTracker.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/router/StatTracker.java index ccc675cb97..b4cb3f7130 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/router/StatTracker.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/router/StatTracker.java @@ -116,7 +116,7 @@ public static enum ResultType { public enum ResultDetails { NO_DETAILS, DS_NOT_FOUND, DS_TLS_MISMATCH, DS_NO_BYPASS, DS_BYPASS, DS_CZ_ONLY, DS_CLIENT_GEO_UNSUPPORTED, GEO_NO_CACHE_FOUND, - REGIONAL_GEO_NO_RULE, REGIONAL_GEO_ALTERNATE_WITHOUT_CACHE, REGIONAL_GEO_ALTERNATE_WITH_CACHE + REGIONAL_GEO_NO_RULE, REGIONAL_GEO_ALTERNATE_WITHOUT_CACHE, REGIONAL_GEO_ALTERNATE_WITH_CACHE, DS_CZ_BACKUP_CG } long time; @@ -130,6 +130,11 @@ public enum ResultDetails { boolean isClientGeolocationQueried; RegionalGeoResult regionalGeoResult; + boolean fromBackupCzGroup; + // in memory switch to track if need to continue geo based + // defaulting to true, changes the false by router at runtime when primary cache group is configured using fallbackToClosedGeoLoc + // to false and backup group list is configured and failing + boolean continueGeo = true; public Track() { start(); @@ -185,6 +190,14 @@ public RegionalGeoResult getRegionalGeoResult() { return regionalGeoResult; } + public void setFromBackupCzGroup(final boolean fromBackupCzGroup) { + this.fromBackupCzGroup = fromBackupCzGroup; + } + + public boolean isFromBackupCzGroup() { + return fromBackupCzGroup; + } + public final void start() { time = System.currentTimeMillis(); } diff --git a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/router/TrafficRouter.java b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/router/TrafficRouter.java index 78ec478a85..caaf4e8cb0 100644 --- a/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/router/TrafficRouter.java +++ b/traffic_router/core/src/main/java/com/comcast/cdn/traffic_control/traffic_router/core/router/TrafficRouter.java @@ -274,8 +274,9 @@ public List getCachesByGeo(final DeliveryService ds, final Geolocation cl protected List selectCaches(final HTTPRequest request, final DeliveryService ds, final Track track) throws GeolocationException { CacheLocation cacheLocation; ResultType result = ResultType.CZ; + final boolean useDeep = (ds.getDeepCache() == DeliveryService.DeepCachingType.ALWAYS); - if (ds.getDeepCache() == DeliveryService.DeepCachingType.ALWAYS) { + if (useDeep) { // Deep caching is enabled. See if there are deep caches available cacheLocation = getDeepCoverageZoneCacheLocation(request.getClientIP(), ds); if (cacheLocation != null && cacheLocation.getCaches().size() != 0) { @@ -287,7 +288,7 @@ protected List selectCaches(final HTTPRequest request, final DeliveryServ } } else { // Deep caching not enabled for this Delivery Service; use the regular CZ - cacheLocation = getCoverageZoneCacheLocation(request.getClientIP(), ds); + cacheLocation = getCoverageZoneCacheLocation(request.getClientIP(), ds, useDeep, track); } Listcaches = selectCachesByCZ(ds, cacheLocation, track, result); @@ -304,7 +305,8 @@ protected List selectCaches(final HTTPRequest request, final DeliveryServ track.setResult(ResultType.MISS); track.setResultDetails(ResultDetails.DS_CZ_ONLY); } - } else { + } else if (track.continueGeo) { + // continue Geo can be disabled when backup group is used -- ended up an empty cache list if reach here caches = selectCachesByGeo(request.getClientIP(), ds, cacheLocation, track); } @@ -312,7 +314,6 @@ protected List selectCaches(final HTTPRequest request, final DeliveryServ } public List selectCachesByGeo(final String clientIp, final DeliveryService deliveryService, final CacheLocation cacheLocation, final Track track) throws GeolocationException { - Geolocation clientLocation = null; try { @@ -374,7 +375,7 @@ public DNSRouteResult route(final DNSRequest request, final Track track) throws return result; } - final CacheLocation cacheLocation = getCoverageZoneCacheLocation(request.getClientIP(), ds); + final CacheLocation cacheLocation = getCoverageZoneCacheLocation(request.getClientIP(), ds, false, track); List caches = selectCachesByCZ(ds, cacheLocation, track); if (caches != null) { @@ -403,7 +404,9 @@ public DNSRouteResult route(final DNSRequest request, final Track track) throws LOGGER.error("Bad client address: '" + request.getClientIP() + "'"); } - caches = selectCachesByGeo(request.getClientIP(), ds, cacheLocation, track); + if (track.continueGeo) { + caches = selectCachesByGeo(request.getClientIP(), ds, cacheLocation, track); + } if (caches != null) { track.setResult(ResultType.GEO); @@ -495,6 +498,9 @@ private List selectCachesByCZ(final DeliveryService ds, final CacheLocati if (caches != null && track != null) { track.setResult(result); + if (track.isFromBackupCzGroup()) { + track.setResultDetails(ResultDetails.DS_CZ_BACKUP_CG); + } track.setResultLocation(cacheLocation.getGeolocation()); } @@ -676,11 +682,11 @@ protected NetworkNode getNetworkNode(final String ip) { } public CacheLocation getCoverageZoneCacheLocation(final String ip, final String deliveryServiceId) { - return getCoverageZoneCacheLocation(ip, deliveryServiceId, false); // default is not deep + return getCoverageZoneCacheLocation(ip, deliveryServiceId, false, null); // default is not deep } @SuppressWarnings({"PMD.CyclomaticComplexity", "PMD.NPathComplexity"}) - public CacheLocation getCoverageZoneCacheLocation(final String ip, final String deliveryServiceId, final boolean useDeep) { + public CacheLocation getCoverageZoneCacheLocation(final String ip, final String deliveryServiceId, final boolean useDeep, final Track track) { final NetworkNode networkNode = useDeep ? getDeepNetworkNode(ip) : getNetworkNode(ip); if (networkNode == null) { @@ -716,17 +722,44 @@ public CacheLocation getCoverageZoneCacheLocation(final String ip, final String return cacheLocation; } + if (cacheLocation != null && cacheLocation.getBackupCacheGroups() != null) { + for (final String cacheGroup : cacheLocation.getBackupCacheGroups()) { + final CacheLocation bkCacheLocation = getCacheRegister().getCacheLocationById(cacheGroup); + if (bkCacheLocation != null && !getSupportingCaches(bkCacheLocation.getCaches(), deliveryService).isEmpty()) { + LOGGER.debug("Got backup CZ cache group " + bkCacheLocation.getId() + " for " + ip + ", ds " + deliveryServiceId); + if (track != null) { + track.setFromBackupCzGroup(true); + } + return bkCacheLocation; + } + } + // track.continueGeo + // will become to false only when backups are configured and (primary group's) fallbackToClosedGeo is configured (non-empty list) to false + // False signals subsequent cacheSelection routine to stop geo based selection. + if (!cacheLocation.isUseClosestGeoLoc()) { + track.continueGeo = false; + return null; + } + } + // We had a hit in the CZF but the name does not match a known cache location. // Check whether the CZF entry has a geolocation and use it if so. - return getClosestCacheLocation(cacheRegister.filterAvailableLocations(deliveryServiceId), networkNode.getGeolocation(), cacheRegister.getDeliveryService(deliveryServiceId)); + final CacheLocation closestCacheLocation = getClosestCacheLocation(cacheRegister.filterAvailableLocations(deliveryServiceId), networkNode.getGeolocation(), cacheRegister.getDeliveryService(deliveryServiceId)); + if (closestCacheLocation != null) { + LOGGER.debug("Got closest CZ cache group " + closestCacheLocation.getId() + " for " + ip + ", ds " + deliveryServiceId); + if (track != null) { + track.setFromBackupCzGroup(true); + } + } + return closestCacheLocation; } public CacheLocation getDeepCoverageZoneCacheLocation(final String ip, final DeliveryService deliveryService) { - return getCoverageZoneCacheLocation(ip, deliveryService, true); + return getCoverageZoneCacheLocation(ip, deliveryService, true, null); } - protected CacheLocation getCoverageZoneCacheLocation(final String ip, final DeliveryService deliveryService, final boolean useDeep) { - return getCoverageZoneCacheLocation(ip, deliveryService.getId(), useDeep); + protected CacheLocation getCoverageZoneCacheLocation(final String ip, final DeliveryService deliveryService, final boolean useDeep, final Track track) { + return getCoverageZoneCacheLocation(ip, deliveryService.getId(), useDeep, track); } protected CacheLocation getCoverageZoneCacheLocation(final String ip, final DeliveryService deliveryService) { @@ -744,7 +777,7 @@ public Cache consistentHashForCoverageZone(final String ip, final String deliver return null; } - final CacheLocation coverageZoneCacheLocation = getCoverageZoneCacheLocation(ip, deliveryService, useDeep); + final CacheLocation coverageZoneCacheLocation = getCoverageZoneCacheLocation(ip, deliveryService, useDeep, null); final List caches = selectCachesByCZ(deliveryService, coverageZoneCacheLocation); if (caches == null || caches.isEmpty()) { diff --git a/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/loc/CoverageZoneTest.java b/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/loc/CoverageZoneTest.java index 0387c89c29..f340b2efb7 100644 --- a/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/loc/CoverageZoneTest.java +++ b/traffic_router/core/src/test/java/com/comcast/cdn/traffic_control/traffic_router/core/loc/CoverageZoneTest.java @@ -88,7 +88,7 @@ public void before() throws Exception { trafficRouter = PowerMockito.mock(TrafficRouter.class); Whitebox.setInternalState(trafficRouter, "cacheRegister", cacheRegister); when(trafficRouter.getCoverageZoneCacheLocation("12.23.34.45", "delivery-service-1")).thenCallRealMethod(); - when(trafficRouter.getCoverageZoneCacheLocation("12.23.34.45", "delivery-service-1", false)).thenCallRealMethod(); + when(trafficRouter.getCoverageZoneCacheLocation("12.23.34.45", "delivery-service-1", false, null)).thenCallRealMethod(); when(trafficRouter.getCacheRegister()).thenReturn(cacheRegister); when(trafficRouter.orderCacheLocations(cacheGroups,testLocation)).thenCallRealMethod(); when(trafficRouter.getSupportingCaches(anyListOf(Cache.class), eq(deliveryService))).thenCallRealMethod();