From ad4c170eef26bc16877284d798cba57701c69a1b Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Sun, 19 Jul 2020 16:26:58 -0600 Subject: [PATCH 01/24] Reorder objects to be consistent with other tests --- traffic_ops/testing/api/v3/serverupdatestatus_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traffic_ops/testing/api/v3/serverupdatestatus_test.go b/traffic_ops/testing/api/v3/serverupdatestatus_test.go index 2842fc7679..3c4f21d999 100644 --- a/traffic_ops/testing/api/v3/serverupdatestatus_test.go +++ b/traffic_ops/testing/api/v3/serverupdatestatus_test.go @@ -196,7 +196,7 @@ func TestServerUpdateStatus(t *testing.T) { } func TestServerQueueUpdate(t *testing.T) { - WithObjs(t, []TCObj{Divisions, Regions, PhysLocations, Statuses, Types, CacheGroups, CDNs, Profiles, Servers}, func() { + WithObjs(t, []TCObj{CDNs, Types, Profiles, Statuses, Divisions, Regions, PhysLocations, CacheGroups, Servers}, func() { // TODO: DON'T hard-code server hostnames! const serverName = "atlanta-edge-01" From 851ffe06d668783c3b4f93582d306b70c99bc965 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Mon, 20 Jul 2020 19:02:07 -0600 Subject: [PATCH 02/24] When the status of a server changes, queue updates on servers within descendant cachegroups of that server's cachegroup --- CHANGELOG.md | 1 + docs/source/api/v3/servers_id_status.rst | 2 +- traffic_ops/client/server.go | 21 ++ .../testing/api/v3/serverupdatestatus_test.go | 84 +++++++ traffic_ops/testing/api/v3/tc-fixtures.json | 221 ++++++++++++++++++ .../traffic_ops_golang/server/put_status.go | 60 ++++- 6 files changed, 382 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd3eda934f..029956b02f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Traffic Ops: Added support for `topology` query parameter to `GET /api/3.0/deliveryservices` to return all delivery services that employ a given topology. - Traffic Ops: Added new topology-based delivery service fields for header rewrites: `firstHeaderRewrite`, `innerHeaderRewrite`, `lastHeaderRewrite` - Traffic Ops: Added validation to prohibit assigning caches to topology-based delivery services + - Traffic Ops: Consider Topologies parentage when queueing server updates - Traffic Portal: Added the ability to create, read, update and delete flexible topologies. - Traffic Portal: Added the ability to assign topologies to delivery services. - Traffic Portal: Added the ability to view all delivery services and cache groups associated with a topology. diff --git a/docs/source/api/v3/servers_id_status.rst b/docs/source/api/v3/servers_id_status.rst index a0de988afd..78c57db400 100644 --- a/docs/source/api/v3/servers_id_status.rst +++ b/docs/source/api/v3/servers_id_status.rst @@ -21,7 +21,7 @@ ``PUT`` ======= -Updates server status and queues updates on all child caches if server type is EDGE or MID. Also, captures offline reason if status is set to ADMIN_DOWN or OFFLINE and prepends offline reason with the user that initiated the status change. +Updates server status and queues updates on all descendant :ref:`Topology` nodes or child caches if server type is EDGE or MID. Also, captures offline reason if status is set to ADMIN_DOWN or OFFLINE and prepends offline reason with the user that initiated the status change. :Auth. Required: Yes :Roles Required: "admin" or "operations" diff --git a/traffic_ops/client/server.go b/traffic_ops/client/server.go index 38e8229f17..25c519dcf9 100644 --- a/traffic_ops/client/server.go +++ b/traffic_ops/client/server.go @@ -178,6 +178,27 @@ func (to *Session) GetServers(params *url.Values, header http.Header) (tc.Server return data, reqInf, err } +// GetFirstServer returns the first server in a servers GET response. +// If no servers match, an error is returned. +// The 'params' parameter can be used to optionally pass URL "query string +// parameters" in the request. +// It returns, in order, the API response that Traffic Ops returned, a request +// info object, and any error that occurred. +func (to *Session) GetFirstServer(params *url.Values, header http.Header) (tc.ServerNullable, ReqInf, error) { + serversResponse, reqInf, err := to.GetServers(params, header) + var firstServer tc.ServerNullable + if err != nil || reqInf.StatusCode == http.StatusNotModified { + return firstServer, reqInf, err + } + for _, firstServer = range serversResponse.Response { + return firstServer, reqInf, err + } + + err = fmt.Errorf("unable to find server matching params %v", *params) + return firstServer, reqInf, err +} + + // GetServerDetailsByHostName GETs Servers by the Server hostname. func (to *Session) GetServerDetailsByHostName(hostName string, header http.Header) ([]tc.ServerDetailV30, ReqInf, error) { v := url.Values{} diff --git a/traffic_ops/testing/api/v3/serverupdatestatus_test.go b/traffic_ops/testing/api/v3/serverupdatestatus_test.go index 3c4f21d999..a3b870b604 100644 --- a/traffic_ops/testing/api/v3/serverupdatestatus_test.go +++ b/traffic_ops/testing/api/v3/serverupdatestatus_test.go @@ -20,6 +20,7 @@ import ( "fmt" "net/http" "net/url" + "strconv" "testing" "github.com/apache/trafficcontrol/lib/go-tc" @@ -377,3 +378,86 @@ func TestSetServerUpdateStatuses(t *testing.T) { } }) } + +func TestSetTopologiesServerUpdateStatuses(t *testing.T) { + WithObjs(t, []TCObj{CDNs, Types, Parameters, Profiles, Statuses, Divisions, Regions, PhysLocations, CacheGroups, Servers, Topologies}, func() { + const ( + topologyName = "forked-topology" + edgeCacheGroup = "topology-edge-cg-01" + otherEdgeCacheGroup = "topology-edge-cg-02" + midCacheGroup = "topology-mid-cg-04" + ) + cacheGroupNames := []string{edgeCacheGroup, otherEdgeCacheGroup, midCacheGroup} + cachesByCacheGroup := map[string]tc.ServerNullable{} + + forkedTopology, _, err := TOSession.GetTopology(topologyName, nil) + if err != nil { + t.Fatalf("topology %s was not found", topologyName) + } + for _, cacheGroupName := range cacheGroupNames { + foundNode := false + for _, node := range forkedTopology.Nodes { + if node.Cachegroup == cacheGroupName { + foundNode = true + break + } + } + if !foundNode { + t.Fatalf("unable to find topology node with cachegroup %s", cacheGroupName) + } + + cacheGroups, _, err := TOSession.GetCacheGroupNullableByName(cacheGroupName, nil) + if err != nil { + t.Fatalf("unable to get cachegroup %s: %s", cacheGroupName, err.Error()) + } + if len(cacheGroups) != 1 { + t.Fatalf("incorrect number of cachegroups. expected: %d actual: %d", 1, len(cacheGroups)) + } + cacheGroup := cacheGroups[0] + + params := url.Values{"cachegroup": []string{strconv.Itoa(*cacheGroup.ID)}} + cachesByCacheGroup[cacheGroupName], _, err = TOSession.GetFirstServer(¶ms, nil) + if err != nil { + t.Fatalf("unable to get a server from cachegroup %s: %s", cacheGroupName, err.Error()) + } + } + + // update status of MID server to OFFLINE + _, _, err = TOSession.UpdateServerStatus(*cachesByCacheGroup[midCacheGroup].ID, tc.ServerPutStatus{ + Status: util.JSONNameOrIDStr{Name: util.StrPtr("OFFLINE")}, + OfflineReason: util.StrPtr("testing")}) + if err != nil { + t.Fatalf("cannot update server status: %s", err.Error()) + } + + for _, cacheGroupName := range cacheGroupNames { + params := url.Values{"cachegroup": []string{strconv.Itoa(*cachesByCacheGroup[cacheGroupName].CachegroupID)}} + cachesByCacheGroup[cacheGroupName], _, err = TOSession.GetFirstServer(¶ms, nil) + if err != nil { + t.Fatalf("unable to get a server from cachegroup %s: %s", cacheGroupName, err.Error()) + } + } + for _, cacheGroupName := range cacheGroupNames { + if err != nil { + t.Fatalf("unable to get a server from cachegroup %s: %s", cacheGroupName, err.Error()) + } + } + // updating the server status does not queue updates within the same cachegroup + if *cachesByCacheGroup[midCacheGroup].UpdPending { + t.Fatalf("expected UpdPending: %t, actual: %t", false, *cachesByCacheGroup[midCacheGroup].UpdPending) + } + // edgeCacheGroup is a descendant of midCacheGroup + if !*cachesByCacheGroup[edgeCacheGroup].UpdPending { + t.Fatalf("expected UpdPending: %t, actual: %t", true, *cachesByCacheGroup[edgeCacheGroup].UpdPending) + } + // otherEdgeCacheGroup is not a descendant of midCacheGroup but is still in the same topology + if *cachesByCacheGroup[otherEdgeCacheGroup].UpdPending { + t.Fatalf("expected UpdPending: %t, actual: %t", false, *cachesByCacheGroup[otherEdgeCacheGroup].UpdPending) + } + + _, _, err = TOSession.SetServerQueueUpdate(*cachesByCacheGroup[midCacheGroup].ID, true) + if err != nil { + t.Fatalf("cannot update server status on %s: %s", *cachesByCacheGroup[midCacheGroup].HostName, err.Error()) + } + }) +} diff --git a/traffic_ops/testing/api/v3/tc-fixtures.json b/traffic_ops/testing/api/v3/tc-fixtures.json index 59e76a64b0..c9480cc6c6 100644 --- a/traffic_ops/testing/api/v3/tc-fixtures.json +++ b/traffic_ops/testing/api/v3/tc-fixtures.json @@ -138,6 +138,69 @@ "parentCachegroupName": "edge-parent1", "shortName": "hep1", "typeName": "EDGE_LOC" + }, + { + "latitude": 0, + "longitude": 0, + "name": "topology-edge-cg-01", + "shortName": "te1", + "typeName": "EDGE_LOC" + }, + { + "latitude": 0, + "longitude": 0, + "name": "topology-edge-cg-02", + "shortName": "te2", + "typeName": "EDGE_LOC" + }, + { + "latitude": 0, + "longitude": 0, + "name": "topology-mid-cg-01", + "shortName": "tm1", + "typeName": "MID_LOC" + }, + { + "latitude": 0, + "longitude": 0, + "name": "topology-mid-cg-02", + "shortName": "tm2", + "typeName": "MID_LOC" + }, + { + "latitude": 0, + "longitude": 0, + "name": "topology-mid-cg-03", + "shortName": "tm3", + "typeName": "MID_LOC" + }, + { + "latitude": 0, + "longitude": 0, + "name": "topology-mid-cg-04", + "shortName": "tm4", + "typeName": "MID_LOC" + }, + { + "latitude": 0, + "longitude": 0, + "name": "topology-mid-cg-05", + "shortName": "tm5", + "typeName": "MID_LOC" + }, + { + "latitude": 0, + "longitude": 0, + "name": "topology-mid-cg-06", + "shortName": "tm6", + "typeName": "MID_LOC" + }, + { + "latitude": 0, + "longitude": 0, + "name": "topology-mid-cg-07", + "shortName": "tm7", + "typeName": "MID_LOC" } ], "cdns": [ @@ -2477,6 +2540,108 @@ "updPending": false, "xmppId": "", "xmppPasswd": "X" + }, + { + "cachegroup": "topology-edge-cg-01", + "cdnName": "cdn1", + "domainName": "edge-01.forked-topology.kabletown.net", + "hostName": "topology-edge-01", + "httpsPort": 443, + "interfaces": [ + { + "ipAddresses": [ + { + "address": "2345:1234:12:2::4/64", + "gateway": "2345:1234:12:2::4", + "serviceAddress": false + }, + { + "address": "127.0.0.13/30", + "gateway": "127.0.0.1", + "serviceAddress": true + } + ], + "monitor": true, + "mtu": 9000, + "name": "bond0" + } + ], + "physLocation": "Denver", + "profile": "EDGE1", + "rack": "RR 119.02", + "revalPending": false, + "status": "REPORTED", + "tcpPort": 80, + "type": "EDGE", + "updPending": false + }, + { + "cachegroup": "topology-edge-cg-02", + "cdnName": "cdn1", + "domainName": "edge-02.forked-topology.kabletown.net", + "hostName": "topology-edge-02", + "httpsPort": 443, + "interfaces": [ + { + "ipAddresses": [ + { + "address": "2345:1234:12:2::4/64", + "gateway": "2345:1234:12:2::4", + "serviceAddress": false + }, + { + "address": "127.0.0.13/30", + "gateway": "127.0.0.1", + "serviceAddress": true + } + ], + "monitor": true, + "mtu": 9000, + "name": "bond0" + } + ], + "physLocation": "Denver", + "profile": "EDGE1", + "rack": "RR 119.02", + "revalPending": false, + "status": "REPORTED", + "tcpPort": 80, + "type": "EDGE", + "updPending": false + }, + { + "cachegroup": "topology-mid-cg-04", + "cdnName": "cdn1", + "domainName": "mid-04.forked-topology.kabletown.net", + "hostName": "topology-mid-04", + "httpsPort": 443, + "interfaces": [ + { + "ipAddresses": [ + { + "address": "2345:1234:12:2::4/64", + "gateway": "2345:1234:12:2::4", + "serviceAddress": false + }, + { + "address": "127.0.0.13/30", + "gateway": "127.0.0.1", + "serviceAddress": true + } + ], + "monitor": true, + "mtu": 9000, + "name": "bond0" + } + ], + "physLocation": "Denver", + "profile": "MID1", + "rack": "RR 119.02", + "revalPending": false, + "status": "REPORTED", + "tcpPort": 80, + "type": "MID", + "updPending": false } ], "serverCapabilities": [ @@ -2698,6 +2863,62 @@ ] } ] + }, + { + "name": "forked-topology", + "description": "This topology stems from 2 anscestors", + "nodes": [ + { + "cachegroup": "topology-edge-cg-01", + "parents": [ + 2 + ] + }, + { + "cachegroup": "topology-edge-cg-02", + "parents": [ + 6 + ] + }, + { + "cachegroup": "topology-mid-cg-01", + "parents": [ + 3 + ] + }, + { + "cachegroup": "topology-mid-cg-02", + "parents": [ + 4 + ] + }, + { + "cachegroup": "topology-mid-cg-03", + "parents": [ + 5 + ] + }, + { + "cachegroup": "topology-mid-cg-04", + "parents": [] + }, + { + "cachegroup": "topology-mid-cg-05", + "parents": [ + 7 + ] + }, + { + "cachegroup": "topology-mid-cg-06", + "parents": [ + 8 + ] + }, + { + "cachegroup": "topology-mid-cg-07", + "parents": [] + } + ] } ], "types": [ diff --git a/traffic_ops/traffic_ops_golang/server/put_status.go b/traffic_ops/traffic_ops_golang/server/put_status.go index 8c0bcb7696..ac0752ca18 100644 --- a/traffic_ops/traffic_ops_golang/server/put_status.go +++ b/traffic_ops/traffic_ops_golang/server/put_status.go @@ -105,16 +105,64 @@ func UpdateStatusHandler(w http.ResponseWriter, r *http.Request) { api.WriteRespAlert(w, r, tc.SuccessLevel, msg) } +// queueUpdatesOnChildNodes queues updates on child nodes of the topology node corresponding corresponding to the given cachegroup +// and returns an error (if one occurs). +func queueUpdatesOnChildNodes(tx *sql.Tx, serverID int, cachegroupID int) error { + query := ` +WITH RECURSIVE topology AS ( + SELECT tcp.parent child, tc.cachegroup + FROM "server" s + JOIN cachegroup c ON s.cachegroup = c.id + JOIN topology_cachegroup tc ON c."name" = tc.cachegroup + JOIN topology_cachegroup_parents tcp ON tc.id = tcp.parent + WHERE s.id = $1 + UNION ALL + SELECT tcp.child, tc.cachegroup FROM topology t, topology_cachegroup_parents tcp + JOIN topology_cachegroup tc ON tcp.child = tc.id + WHERE t.child = tcp.parent +) +SELET c.id +FROM cachegroup c +JOIN topology t ON c."name" = t.cachegroup +WHERE s.cachegroup = c.id +AND s.cachegroup != $2 +` + if _, err := tx.Exec(query, serverID, cachegroupID); err != nil { + return errors.New("queueing updates on child topology nodes: " + err.Error()) + } + return nil +} + // queueUpdatesOnChildCaches queues updates on child caches of the given cdnID and parentCachegroupID and returns an error (if one occurs). func queueUpdatesOnChildCaches(tx *sql.Tx, cdnID, parentCachegroupID int) error { q := ` +WITH RECURSIVE topology_descendants AS ( + SELECT tcp.parent child, tc.cachegroup + FROM cachegroup c + JOIN topology_cachegroup tc ON c."name" = tc.cachegroup + JOIN topology_cachegroup_parents tcp ON tc.id = tcp.parent + WHERE c.id = $2 +UNION ALL + SELECT tcp.child, tc.cachegroup + FROM topology_descendants td, topology_cachegroup_parents tcp + JOIN topology_cachegroup tc ON tcp.child = tc.id + WHERE td.child = tcp.parent +), server_topology_descendants AS ( +SELECT c.id +FROM cachegroup c +JOIN topology_descendants td ON c."name" = td.cachegroup +WHERE c.id != $2 +) UPDATE server -SET upd_pending = TRUE -WHERE server.cdn_id = $1 - AND server.cachegroup IN (SELECT id - FROM cachegroup - WHERE parent_cachegroup_id = $2 - OR secondary_parent_cachegroup_id = $2) +SET upd_pending = TRUE +WHERE (server.cdn_id = $1 + AND server.cachegroup IN ( + SELECT id + FROM cachegroup + WHERE parent_cachegroup_id = $2 + OR secondary_parent_cachegroup_id = $2 + )) + OR server.cachegroup IN (SELECT stc.id FROM server_topology_descendants stc) ` if _, err := tx.Exec(q, cdnID, parentCachegroupID); err != nil { return errors.New("queueing updates on child caches: " + err.Error()) From 43d3d8a3960fa8a3ea9dc8b0f7fb8032e3dd6d76 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Mon, 6 Jul 2020 06:41:19 -0600 Subject: [PATCH 03/24] Reformat update_status query for readability and add language injection comments for syntax highlighting --- .../server/servers_update_status.go | 49 ++++++++++++++----- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/traffic_ops/traffic_ops_golang/server/servers_update_status.go b/traffic_ops/traffic_ops_golang/server/servers_update_status.go index 17eaa13c38..681a624b9e 100644 --- a/traffic_ops/traffic_ops_golang/server/servers_update_status.go +++ b/traffic_ops/traffic_ops_golang/server/servers_update_status.go @@ -46,18 +46,45 @@ func GetServerUpdateStatusHandler(w http.ResponseWriter, r *http.Request) { } func getServerUpdateStatus(tx *sql.Tx, cfg *config.Config, hostName string) ([]tc.ServerUpdateStatus, error) { - baseSelectStatement := - `WITH parentservers AS (SELECT ps.id, ps.cachegroup, ps.cdn_id, ps.upd_pending, ps.reval_pending FROM server ps - LEFT JOIN status AS pstatus ON pstatus.id = ps.status - WHERE pstatus.name != 'OFFLINE' ), - use_reval_pending AS (SELECT value::boolean FROM parameter WHERE name = 'use_reval_pending' AND config_file = 'global' UNION ALL SELECT FALSE FETCH FIRST 1 ROW ONLY) - SELECT s.id, s.host_name, type.name AS type, (s.reval_pending::boolean) as server_reval_pending, use_reval_pending.value, s.upd_pending, status.name AS status, COALESCE(bool_or(ps.upd_pending), FALSE) AS parent_upd_pending, COALESCE(bool_or(ps.reval_pending), FALSE) AS parent_reval_pending FROM use_reval_pending, server s - LEFT JOIN status ON s.status = status.id - LEFT JOIN cachegroup cg ON s.cachegroup = cg.id - LEFT JOIN type ON type.id = s.type - LEFT JOIN parentservers ps ON ps.cachegroup = cg.parent_cachegroup_id AND ps.cdn_id = s.cdn_id AND type.name = 'EDGE'` //remove the EDGE reference if other server types should have their parents processed + // language=SQL + baseSelectStatement := ` +WITH parentservers AS ( + SELECT ps.id, ps.cachegroup, ps.cdn_id, ps.upd_pending, ps.reval_pending + FROM server ps + LEFT JOIN status AS pstatus ON pstatus.id = ps.status + WHERE pstatus.name != 'OFFLINE' +), use_reval_pending AS ( + SELECT value::BOOLEAN + FROM parameter + WHERE name = 'use_reval_pending' + AND config_file = 'global' + UNION ALL SELECT FALSE FETCH FIRST 1 ROW ONLY +) +SELECT + s.id, + s.host_name, + type.name AS type, + (s.reval_pending::BOOLEAN) AS server_reval_pending, + use_reval_pending.value, + s.upd_pending, + status.name AS status, + COALESCE(BOOL_OR(ps.upd_pending), FALSE) AS parent_upd_pending, + COALESCE(BOOL_OR(ps.reval_pending), FALSE) AS parent_reval_pending + FROM use_reval_pending, + server s +LEFT JOIN status ON s.status = status.id +LEFT JOIN cachegroup cg ON s.cachegroup = cg.id +LEFT JOIN type ON type.id = s.type +LEFT JOIN parentservers ps ON ps.cachegroup = cg.parent_cachegroup_id + AND ps.cdn_id = s.cdn_id + AND type.name = 'EDGE' +` // remove the EDGE reference if other server types should have their parents processed - groupBy := ` GROUP BY s.id, s.host_name, type.name, server_reval_pending, use_reval_pending.value, s.upd_pending, status.name ORDER BY s.id;` + // language=SQL + groupBy := ` +GROUP BY s.id, s.host_name, type.name, server_reval_pending, use_reval_pending.value, s.upd_pending, status.name +ORDER BY s.id +` updateStatuses := []tc.ServerUpdateStatus{} var rows *sql.Rows From bf6782dd65a37b4f06a8e7e96517ff290661e203 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Tue, 21 Jul 2020 10:40:49 -0600 Subject: [PATCH 04/24] Add legacy severs{host_name}/update_status handler to continue deprecated support for servers/all/update_status for API versions below 3.0 --- .../traffic_ops_golang/routing/routes.go | 4 +- .../server/servers_update_status.go | 93 +++++++++++++++++++ 2 files changed, 95 insertions(+), 2 deletions(-) diff --git a/traffic_ops/traffic_ops_golang/routing/routes.go b/traffic_ops/traffic_ops_golang/routing/routes.go index 85224ebc14..ff9205d565 100644 --- a/traffic_ops/traffic_ops_golang/routing/routes.go +++ b/traffic_ops/traffic_ops_golang/routing/routes.go @@ -679,7 +679,7 @@ func Routes(d ServerData) ([]Route, []RawRoute, http.Handler, error) { //Server status {api.Version{2, 0}, http.MethodPut, `servers/{id}/status$`, server.UpdateStatusHandler, auth.PrivLevelOperations, Authenticated, nil, 276663851, noPerlBypass}, {api.Version{2, 0}, http.MethodPost, `servers/{id}/queue_update$`, server.QueueUpdateHandler, auth.PrivLevelOperations, Authenticated, nil, 2189471, noPerlBypass}, - {api.Version{2, 0}, http.MethodGet, `servers/{host_name}/update_status$`, server.GetServerUpdateStatusHandler, auth.PrivLevelReadOnly, Authenticated, nil, 238451599, noPerlBypass}, + {api.Version{2, 0}, http.MethodGet, `servers/{host_name}/update_status$`, server.GetServerUpdateStatusHandlerV2, auth.PrivLevelReadOnly, Authenticated, nil, 238451599, noPerlBypass}, {api.Version{2, 0}, http.MethodPost, `servers/{id-or-name}/update$`, server.UpdateHandler, auth.PrivLevelOperations, Authenticated, nil, 14381323, noPerlBypass}, //Server: CRUD @@ -1077,7 +1077,7 @@ func Routes(d ServerData) ([]Route, []RawRoute, http.Handler, error) { //Server status {api.Version{1, 1}, http.MethodPut, `servers/{id}/status$`, server.UpdateStatusHandler, auth.PrivLevelOperations, Authenticated, nil, 776663851, perlBypass}, {api.Version{1, 1}, http.MethodPost, `servers/{id}/queue_update$`, server.QueueUpdateHandler, auth.PrivLevelOperations, Authenticated, nil, 9189471, perlBypass}, - {api.Version{1, 3}, http.MethodGet, `servers/{host_name}/update_status$`, server.GetServerUpdateStatusHandler, auth.PrivLevelReadOnly, Authenticated, nil, 438451599, noPerlBypass}, + {api.Version{1, 3}, http.MethodGet, `servers/{host_name}/update_status$`, server.GetServerUpdateStatusHandlerV2, auth.PrivLevelReadOnly, Authenticated, nil, 438451599, noPerlBypass}, //Server: CRUD {api.Version{1, 1}, http.MethodGet, `servers/?(\.json)?$`, server.Read, auth.PrivLevelReadOnly, Authenticated, nil, 1720959285, noPerlBypass}, diff --git a/traffic_ops/traffic_ops_golang/server/servers_update_status.go b/traffic_ops/traffic_ops_golang/server/servers_update_status.go index 681a624b9e..265fa2453f 100644 --- a/traffic_ops/traffic_ops_golang/server/servers_update_status.go +++ b/traffic_ops/traffic_ops_golang/server/servers_update_status.go @@ -119,3 +119,96 @@ ORDER BY s.id } return updateStatuses, nil } + +func GetServerUpdateStatusHandlerV2(w http.ResponseWriter, r *http.Request) { + inf, userErr, sysErr, errCode := api.NewInfo(r, []string{"host_name"}, nil) + if userErr != nil || sysErr != nil { + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr) + return + } + defer inf.Close() + + serverUpdateStatus, err := getServerUpdateStatusV2(inf.Tx.Tx, inf.Config, inf.Params["host_name"]) + if err != nil { + api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, err) + return + } + api.WriteRespRaw(w, r, serverUpdateStatus) +} + +// getServerUpdateStatusV2 supports /servers/all/update_status in addition to /servers/{host_name}/update_status +// This special case is believed to be used nowhere. +func getServerUpdateStatusV2(tx *sql.Tx, cfg *config.Config, hostName string) ([]tc.ServerUpdateStatus, error) { + // language=SQL + baseSelectStatement := ` +WITH parentservers AS ( + SELECT ps.id, ps.cachegroup, ps.cdn_id, ps.upd_pending, ps.reval_pending + FROM server ps + LEFT JOIN status AS pstatus ON pstatus.id = ps.status + WHERE pstatus.name != 'OFFLINE' +), use_reval_pending AS ( + SELECT value::BOOLEAN + FROM parameter + WHERE name = 'use_reval_pending' + AND config_file = 'global' + UNION ALL SELECT FALSE FETCH FIRST 1 ROW ONLY +) +SELECT + s.id, + s.host_name, + type.name AS type, + (s.reval_pending::BOOLEAN) AS server_reval_pending, + use_reval_pending.value, + s.upd_pending, + status.name AS status, + COALESCE(BOOL_OR(ps.upd_pending), FALSE) AS parent_upd_pending, + COALESCE(BOOL_OR(ps.reval_pending), FALSE) AS parent_reval_pending + FROM use_reval_pending, + server s +LEFT JOIN status ON s.status = status.id +LEFT JOIN cachegroup cg ON s.cachegroup = cg.id +LEFT JOIN type ON type.id = s.type +LEFT JOIN parentservers ps ON ps.cachegroup = cg.parent_cachegroup_id + AND ps.cdn_id = s.cdn_id + AND type.name = 'EDGE' +` // remove the EDGE reference if other server types should have their parents processed + + // language=SQL + groupBy := ` +GROUP BY s.id, s.host_name, type.name, server_reval_pending, use_reval_pending.value, s.upd_pending, status.name +ORDER BY s.id +` + + updateStatuses := []tc.ServerUpdateStatus{} + var rows *sql.Rows + var err error + if hostName == "all" { + rows, err = tx.Query(baseSelectStatement + groupBy) + if err != nil { + log.Error.Printf("could not execute select server update status query: %s\n", err) + return nil, tc.DBError + } + } else { + rows, err = tx.Query(baseSelectStatement+` WHERE s.host_name = $1`+groupBy, hostName) + if err != nil { + log.Error.Printf("could not execute select server update status by hostname query: %s\n", err) + return nil, tc.DBError + } + } + defer rows.Close() + + for rows.Next() { + var serverUpdateStatus tc.ServerUpdateStatus + var serverType string + if err := rows.Scan(&serverUpdateStatus.HostId, &serverUpdateStatus.HostName, &serverType, &serverUpdateStatus.RevalPending, &serverUpdateStatus.UseRevalPending, &serverUpdateStatus.UpdatePending, &serverUpdateStatus.Status, &serverUpdateStatus.ParentPending, &serverUpdateStatus.ParentRevalPending); err != nil { + log.Error.Printf("could not scan server update status: %s\n", err) + return nil, tc.DBError + } + if hostName == "all" { //if we want to return the parent data for servers when all is used remove this block + serverUpdateStatus.ParentRevalPending = false + serverUpdateStatus.ParentPending = false + } + updateStatuses = append(updateStatuses, serverUpdateStatus) + } + return updateStatuses, nil +} From f7e3b7e0814c9e80f38d08e6a9229d10e7fffc00 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Mon, 20 Jul 2020 19:07:39 -0600 Subject: [PATCH 05/24] When the status of a server in an ancestor topology cachegroup changes. set parent_pending to true --- CHANGELOG.md | 2 +- .../api/v3/servers_hostname_update_status.rst | 4 +- .../testing/api/v3/serverupdatestatus_test.go | 23 ++++++ traffic_ops/testing/api/v3/tc-fixtures.json | 2 +- .../server/servers_update_status.go | 74 +++++++++++-------- 5 files changed, 69 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 029956b02f..4ee5216963 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Traffic Ops: Added support for `topology` query parameter to `GET /api/3.0/deliveryservices` to return all delivery services that employ a given topology. - Traffic Ops: Added new topology-based delivery service fields for header rewrites: `firstHeaderRewrite`, `innerHeaderRewrite`, `lastHeaderRewrite` - Traffic Ops: Added validation to prohibit assigning caches to topology-based delivery services - - Traffic Ops: Consider Topologies parentage when queueing server updates + - Traffic Ops: Consider Topologies parentage when queueing or checking server updates - Traffic Portal: Added the ability to create, read, update and delete flexible topologies. - Traffic Portal: Added the ability to assign topologies to delivery services. - Traffic Portal: Added the ability to view all delivery services and cache groups associated with a topology. diff --git a/docs/source/api/v3/servers_hostname_update_status.rst b/docs/source/api/v3/servers_hostname_update_status.rst index fd4e13268e..767d3f7cab 100644 --- a/docs/source/api/v3/servers_hostname_update_status.rst +++ b/docs/source/api/v3/servers_hostname_update_status.rst @@ -54,8 +54,8 @@ Each object in the returned array\ [1]_ will contain the following fields: :host_id: The integral, unique identifier for the server for which the other fields in this object represent the pending updates and revalidation status :host_name: The (short) hostname of the server for which the other fields in this object represent the pending updates and revalidation status -:parent_pending: A boolean telling whether or not the :term:`parents` of this server have pending updates -:parent_reval_pending: A boolean telling whether or not the :term:`parents` of this server have pending revalidation jobs +:parent_pending: A boolean telling whether or not any :ref:`Topology` ancestor or :term:`parent` of this server has pending updates +:parent_reval_pending: A boolean telling whether or not any :ref:`Topology` ancestor or :term:`parent` of this server has pending revalidation jobs :reval_pending: ``true`` if the server has pending revalidation jobs, ``false`` otherwise :status: The name of the status of this server diff --git a/traffic_ops/testing/api/v3/serverupdatestatus_test.go b/traffic_ops/testing/api/v3/serverupdatestatus_test.go index a3b870b604..8c00de99bb 100644 --- a/traffic_ops/testing/api/v3/serverupdatestatus_test.go +++ b/traffic_ops/testing/api/v3/serverupdatestatus_test.go @@ -389,6 +389,7 @@ func TestSetTopologiesServerUpdateStatuses(t *testing.T) { ) cacheGroupNames := []string{edgeCacheGroup, otherEdgeCacheGroup, midCacheGroup} cachesByCacheGroup := map[string]tc.ServerNullable{} + updateStatusByCacheGroup := map[string]tc.ServerUpdateStatus{} forkedTopology, _, err := TOSession.GetTopology(topologyName, nil) if err != nil { @@ -438,6 +439,7 @@ func TestSetTopologiesServerUpdateStatuses(t *testing.T) { } } for _, cacheGroupName := range cacheGroupNames { + updateStatusByCacheGroup[cacheGroupName], _, err = TOSession.GetServerUpdateStatus(*cachesByCacheGroup[cacheGroupName].HostName, nil) if err != nil { t.Fatalf("unable to get a server from cachegroup %s: %s", cacheGroupName, err.Error()) } @@ -450,14 +452,35 @@ func TestSetTopologiesServerUpdateStatuses(t *testing.T) { if !*cachesByCacheGroup[edgeCacheGroup].UpdPending { t.Fatalf("expected UpdPending: %t, actual: %t", true, *cachesByCacheGroup[edgeCacheGroup].UpdPending) } + if !updateStatusByCacheGroup[edgeCacheGroup].UpdatePending { + t.Fatalf("expected UpdPending: %t, actual: %t", true, updateStatusByCacheGroup[edgeCacheGroup].UpdatePending) + } // otherEdgeCacheGroup is not a descendant of midCacheGroup but is still in the same topology if *cachesByCacheGroup[otherEdgeCacheGroup].UpdPending { t.Fatalf("expected UpdPending: %t, actual: %t", false, *cachesByCacheGroup[otherEdgeCacheGroup].UpdPending) } + if updateStatusByCacheGroup[otherEdgeCacheGroup].UpdatePending { + t.Fatalf("expected UpdPending: %t, actual: %t", false, updateStatusByCacheGroup[otherEdgeCacheGroup].UpdatePending) + } _, _, err = TOSession.SetServerQueueUpdate(*cachesByCacheGroup[midCacheGroup].ID, true) if err != nil { t.Fatalf("cannot update server status on %s: %s", *cachesByCacheGroup[midCacheGroup].HostName, err.Error()) } + for _, cacheGroupName := range cacheGroupNames { + updateStatusByCacheGroup[cacheGroupName], _, err = TOSession.GetServerUpdateStatus(*cachesByCacheGroup[cacheGroupName].HostName, nil) + if err != nil { + t.Fatalf("unable to get a server from cachegroup %s: %s", cacheGroupName, err.Error()) + } + } + + // edgeCacheGroup is a descendant of midCacheGroup + if !updateStatusByCacheGroup[edgeCacheGroup].ParentPending { + t.Fatalf("expected UpdPending: %t, actual: %t", true, updateStatusByCacheGroup[edgeCacheGroup].ParentPending) + } + // otherEdgeCacheGroup is not a descendant of midCacheGroup but is still in the same topology + if updateStatusByCacheGroup[otherEdgeCacheGroup].ParentPending { + t.Fatalf("expected UpdPending: %t, actual: %t", false, updateStatusByCacheGroup[otherEdgeCacheGroup].ParentPending) + } }) } diff --git a/traffic_ops/testing/api/v3/tc-fixtures.json b/traffic_ops/testing/api/v3/tc-fixtures.json index c9480cc6c6..8ee3edd953 100644 --- a/traffic_ops/testing/api/v3/tc-fixtures.json +++ b/traffic_ops/testing/api/v3/tc-fixtures.json @@ -2866,7 +2866,7 @@ }, { "name": "forked-topology", - "description": "This topology stems from 2 anscestors", + "description": "This topology stems from 2 ancestors", "nodes": [ { "cachegroup": "topology-edge-cg-01", diff --git a/traffic_ops/traffic_ops_golang/server/servers_update_status.go b/traffic_ops/traffic_ops_golang/server/servers_update_status.go index 265fa2453f..aae6970469 100644 --- a/traffic_ops/traffic_ops_golang/server/servers_update_status.go +++ b/traffic_ops/traffic_ops_golang/server/servers_update_status.go @@ -46,11 +46,33 @@ func GetServerUpdateStatusHandler(w http.ResponseWriter, r *http.Request) { } func getServerUpdateStatus(tx *sql.Tx, cfg *config.Config, hostName string) ([]tc.ServerUpdateStatus, error) { - // language=SQL - baseSelectStatement := ` -WITH parentservers AS ( - SELECT ps.id, ps.cachegroup, ps.cdn_id, ps.upd_pending, ps.reval_pending - FROM server ps + + updateStatuses := []tc.ServerUpdateStatus{} + var rows *sql.Rows + var err error + + selectQuery := ` +WITH RECURSIVE topology_ancestors AS ( + SELECT tcp.child parent, tc.cachegroup + FROM "server" s + JOIN cachegroup c ON s.cachegroup = c.id + JOIN topology_cachegroup tc ON c."name" = tc.cachegroup + JOIN topology_cachegroup_parents tcp ON tc.id = tcp.child + WHERE s.host_name = $1 +UNION ALL + SELECT tcp.parent, tc.cachegroup + FROM topology_ancestors ta, topology_cachegroup_parents tcp + JOIN topology_cachegroup tc ON tcp.parent = tc.id + WHERE ta.parent = tcp.child +), server_topology_ancestors AS ( +SELECT s.id, s.cachegroup, s.cdn_id, s.upd_pending, s.reval_pending, s.status +FROM server s +JOIN cachegroup c ON s.cachegroup = c.id +JOIN topology_ancestors ta ON c."name" = ta.cachegroup +WHERE s.host_name != $1 +), parentservers AS ( + SELECT ps.id, ps.cachegroup, ps.cdn_id, ps.upd_pending, ps.reval_pending, ps.status + FROM server ps LEFT JOIN status AS pstatus ON pstatus.id = ps.status WHERE pstatus.name != 'OFFLINE' ), use_reval_pending AS ( @@ -68,8 +90,14 @@ SELECT use_reval_pending.value, s.upd_pending, status.name AS status, - COALESCE(BOOL_OR(ps.upd_pending), FALSE) AS parent_upd_pending, - COALESCE(BOOL_OR(ps.reval_pending), FALSE) AS parent_reval_pending + COALESCE(BOOL_OR(ps.upd_pending), + TRUE IN ( + SELECT sta.upd_pending FROM server_topology_ancestors sta) + ) AS parent_upd_pending, + COALESCE(BOOL_OR(ps.reval_pending), + TRUE IN ( + SELECT sta.reval_pending FROM server_topology_ancestors sta) + ) AS parent_reval_pending FROM use_reval_pending, server s LEFT JOIN status ON s.status = status.id @@ -78,31 +106,17 @@ LEFT JOIN type ON type.id = s.type LEFT JOIN parentservers ps ON ps.cachegroup = cg.parent_cachegroup_id AND ps.cdn_id = s.cdn_id AND type.name = 'EDGE' -` // remove the EDGE reference if other server types should have their parents processed - - // language=SQL - groupBy := ` +WHERE s.host_name = $1 GROUP BY s.id, s.host_name, type.name, server_reval_pending, use_reval_pending.value, s.upd_pending, status.name ORDER BY s.id -` +` // remove the type.name = 'EDGE' condition if other server types should have their parents processed - updateStatuses := []tc.ServerUpdateStatus{} - var rows *sql.Rows - var err error - if hostName == "all" { - rows, err = tx.Query(baseSelectStatement + groupBy) - if err != nil { - log.Error.Printf("could not execute select server update status query: %s\n", err) - return nil, tc.DBError - } - } else { - rows, err = tx.Query(baseSelectStatement+` WHERE s.host_name = $1`+groupBy, hostName) - if err != nil { - log.Error.Printf("could not execute select server update status by hostname query: %s\n", err) - return nil, tc.DBError - } + rows, err = tx.Query(selectQuery, hostName) + if err != nil { + log.Error.Printf("could not execute query: %s\n", err) + return nil, tc.DBError } - defer rows.Close() + defer log.Close(rows, "unable to close db connection") for rows.Next() { var serverUpdateStatus tc.ServerUpdateStatus @@ -111,10 +125,6 @@ ORDER BY s.id log.Error.Printf("could not scan server update status: %s\n", err) return nil, tc.DBError } - if hostName == "all" { //if we want to return the parent data for servers when all is used remove this block - serverUpdateStatus.ParentRevalPending = false - serverUpdateStatus.ParentPending = false - } updateStatuses = append(updateStatuses, serverUpdateStatus) } return updateStatuses, nil From 6407610e29bc071a258c2497f9056e018d08f3b4 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Fri, 24 Jul 2020 10:45:24 -0600 Subject: [PATCH 06/24] Separate API v1 and v2 handlers --- traffic_ops/traffic_ops_golang/routing/routes.go | 2 +- .../traffic_ops_golang/server/servers_update_status.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/traffic_ops/traffic_ops_golang/routing/routes.go b/traffic_ops/traffic_ops_golang/routing/routes.go index ff9205d565..e83127f853 100644 --- a/traffic_ops/traffic_ops_golang/routing/routes.go +++ b/traffic_ops/traffic_ops_golang/routing/routes.go @@ -1077,7 +1077,7 @@ func Routes(d ServerData) ([]Route, []RawRoute, http.Handler, error) { //Server status {api.Version{1, 1}, http.MethodPut, `servers/{id}/status$`, server.UpdateStatusHandler, auth.PrivLevelOperations, Authenticated, nil, 776663851, perlBypass}, {api.Version{1, 1}, http.MethodPost, `servers/{id}/queue_update$`, server.QueueUpdateHandler, auth.PrivLevelOperations, Authenticated, nil, 9189471, perlBypass}, - {api.Version{1, 3}, http.MethodGet, `servers/{host_name}/update_status$`, server.GetServerUpdateStatusHandlerV2, auth.PrivLevelReadOnly, Authenticated, nil, 438451599, noPerlBypass}, + {api.Version{1, 3}, http.MethodGet, `servers/{host_name}/update_status$`, server.GetServerUpdateStatusHandlerV1, auth.PrivLevelReadOnly, Authenticated, nil, 438451599, noPerlBypass}, //Server: CRUD {api.Version{1, 1}, http.MethodGet, `servers/?(\.json)?$`, server.Read, auth.PrivLevelReadOnly, Authenticated, nil, 1720959285, noPerlBypass}, diff --git a/traffic_ops/traffic_ops_golang/server/servers_update_status.go b/traffic_ops/traffic_ops_golang/server/servers_update_status.go index aae6970469..7e02246999 100644 --- a/traffic_ops/traffic_ops_golang/server/servers_update_status.go +++ b/traffic_ops/traffic_ops_golang/server/servers_update_status.go @@ -131,6 +131,10 @@ ORDER BY s.id } func GetServerUpdateStatusHandlerV2(w http.ResponseWriter, r *http.Request) { + GetServerUpdateStatusHandlerV1(w, r) +} + +func GetServerUpdateStatusHandlerV1(w http.ResponseWriter, r *http.Request) { inf, userErr, sysErr, errCode := api.NewInfo(r, []string{"host_name"}, nil) if userErr != nil || sysErr != nil { api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr) From e58320368b7c57ff754dbbffaa5cea90b4f89680 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Thu, 30 Jul 2020 09:58:10 -0600 Subject: [PATCH 07/24] Remove unused function queueUpdatesOnChildNodes() --- .../traffic_ops_golang/server/put_status.go | 28 ------------------- 1 file changed, 28 deletions(-) diff --git a/traffic_ops/traffic_ops_golang/server/put_status.go b/traffic_ops/traffic_ops_golang/server/put_status.go index ac0752ca18..3242612ecd 100644 --- a/traffic_ops/traffic_ops_golang/server/put_status.go +++ b/traffic_ops/traffic_ops_golang/server/put_status.go @@ -105,34 +105,6 @@ func UpdateStatusHandler(w http.ResponseWriter, r *http.Request) { api.WriteRespAlert(w, r, tc.SuccessLevel, msg) } -// queueUpdatesOnChildNodes queues updates on child nodes of the topology node corresponding corresponding to the given cachegroup -// and returns an error (if one occurs). -func queueUpdatesOnChildNodes(tx *sql.Tx, serverID int, cachegroupID int) error { - query := ` -WITH RECURSIVE topology AS ( - SELECT tcp.parent child, tc.cachegroup - FROM "server" s - JOIN cachegroup c ON s.cachegroup = c.id - JOIN topology_cachegroup tc ON c."name" = tc.cachegroup - JOIN topology_cachegroup_parents tcp ON tc.id = tcp.parent - WHERE s.id = $1 - UNION ALL - SELECT tcp.child, tc.cachegroup FROM topology t, topology_cachegroup_parents tcp - JOIN topology_cachegroup tc ON tcp.child = tc.id - WHERE t.child = tcp.parent -) -SELET c.id -FROM cachegroup c -JOIN topology t ON c."name" = t.cachegroup -WHERE s.cachegroup = c.id -AND s.cachegroup != $2 -` - if _, err := tx.Exec(query, serverID, cachegroupID); err != nil { - return errors.New("queueing updates on child topology nodes: " + err.Error()) - } - return nil -} - // queueUpdatesOnChildCaches queues updates on child caches of the given cdnID and parentCachegroupID and returns an error (if one occurs). func queueUpdatesOnChildCaches(tx *sql.Tx, cdnID, parentCachegroupID int) error { q := ` From f97f12126703bfc8db00289591753f955319555b Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Thu, 30 Jul 2020 09:59:32 -0600 Subject: [PATCH 08/24] Comment Postgres queries --- traffic_ops/traffic_ops_golang/server/put_status.go | 11 +++++++++++ .../server/servers_update_status.go | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/traffic_ops/traffic_ops_golang/server/put_status.go b/traffic_ops/traffic_ops_golang/server/put_status.go index 3242612ecd..fa8f2a2310 100644 --- a/traffic_ops/traffic_ops_golang/server/put_status.go +++ b/traffic_ops/traffic_ops_golang/server/put_status.go @@ -108,21 +108,32 @@ func UpdateStatusHandler(w http.ResponseWriter, r *http.Request) { // queueUpdatesOnChildCaches queues updates on child caches of the given cdnID and parentCachegroupID and returns an error (if one occurs). func queueUpdatesOnChildCaches(tx *sql.Tx, cdnID, parentCachegroupID int) error { q := ` +/* topology_descendants finds the descendant topology nodes of the topology node + * for the cachegroup containing server $2. + */ WITH RECURSIVE topology_descendants AS ( +/* This is the base case of the recursive CTE, the topology node for the + * cachegroup containing cachegroup $2. + */ SELECT tcp.parent child, tc.cachegroup FROM cachegroup c JOIN topology_cachegroup tc ON c."name" = tc.cachegroup JOIN topology_cachegroup_parents tcp ON tc.id = tcp.parent WHERE c.id = $2 UNION ALL +/* Find all direct topology child nodes tc of a given topology descendant td. */ SELECT tcp.child, tc.cachegroup FROM topology_descendants td, topology_cachegroup_parents tcp JOIN topology_cachegroup tc ON tcp.child = tc.id WHERE td.child = tcp.parent +/* server_topology_descendants is the set of every server whose cachegroup is a + * descendant topology node found by topology_descendants. + */ ), server_topology_descendants AS ( SELECT c.id FROM cachegroup c JOIN topology_descendants td ON c."name" = td.cachegroup +/* Filter out cachegroup id $2 */ WHERE c.id != $2 ) UPDATE server diff --git a/traffic_ops/traffic_ops_golang/server/servers_update_status.go b/traffic_ops/traffic_ops_golang/server/servers_update_status.go index 7e02246999..90bfad9400 100644 --- a/traffic_ops/traffic_ops_golang/server/servers_update_status.go +++ b/traffic_ops/traffic_ops_golang/server/servers_update_status.go @@ -52,7 +52,13 @@ func getServerUpdateStatus(tx *sql.Tx, cfg *config.Config, hostName string) ([]t var err error selectQuery := ` +/* topology_ancestors finds the ancestor topology nodes of the topology node for + * the cachegroup containing server $1. + */ WITH RECURSIVE topology_ancestors AS ( +/* This is the base case of the recursive CTE, the topology node for the + * cachegroup containing server $1. + */ SELECT tcp.child parent, tc.cachegroup FROM "server" s JOIN cachegroup c ON s.cachegroup = c.id @@ -60,15 +66,20 @@ WITH RECURSIVE topology_ancestors AS ( JOIN topology_cachegroup_parents tcp ON tc.id = tcp.child WHERE s.host_name = $1 UNION ALL +/* Find all direct topology parent nodes tc of a given topology ancestor ta. */ SELECT tcp.parent, tc.cachegroup FROM topology_ancestors ta, topology_cachegroup_parents tcp JOIN topology_cachegroup tc ON tcp.parent = tc.id WHERE ta.parent = tcp.child +/* server_topology_ancestors is the set of every server whose cachegroup is an + * ancestor topology node found by topology_ancestors. + */ ), server_topology_ancestors AS ( SELECT s.id, s.cachegroup, s.cdn_id, s.upd_pending, s.reval_pending, s.status FROM server s JOIN cachegroup c ON s.cachegroup = c.id JOIN topology_ancestors ta ON c."name" = ta.cachegroup +/* Filter out host_name $1 */ WHERE s.host_name != $1 ), parentservers AS ( SELECT ps.id, ps.cachegroup, ps.cdn_id, ps.upd_pending, ps.reval_pending, ps.status From 94d4d987b699e6d4d7b06d8291ede560faf2c2ff Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Thu, 30 Jul 2020 10:02:35 -0600 Subject: [PATCH 09/24] Allow mids to be child cachegroups in TO API v3 --- traffic_ops/traffic_ops_golang/server/servers_update_status.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/traffic_ops/traffic_ops_golang/server/servers_update_status.go b/traffic_ops/traffic_ops_golang/server/servers_update_status.go index 90bfad9400..1dc9d995f5 100644 --- a/traffic_ops/traffic_ops_golang/server/servers_update_status.go +++ b/traffic_ops/traffic_ops_golang/server/servers_update_status.go @@ -116,11 +116,10 @@ LEFT JOIN cachegroup cg ON s.cachegroup = cg.id LEFT JOIN type ON type.id = s.type LEFT JOIN parentservers ps ON ps.cachegroup = cg.parent_cachegroup_id AND ps.cdn_id = s.cdn_id - AND type.name = 'EDGE' WHERE s.host_name = $1 GROUP BY s.id, s.host_name, type.name, server_reval_pending, use_reval_pending.value, s.upd_pending, status.name ORDER BY s.id -` // remove the type.name = 'EDGE' condition if other server types should have their parents processed +` rows, err = tx.Query(selectQuery, hostName) if err != nil { From 2ea1793d4a93e11c9cd899614ba450d41bbd311e Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Thu, 30 Jul 2020 10:05:41 -0600 Subject: [PATCH 10/24] Make GoDoc a complete sentence --- traffic_ops/traffic_ops_golang/server/servers_update_status.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/traffic_ops/traffic_ops_golang/server/servers_update_status.go b/traffic_ops/traffic_ops_golang/server/servers_update_status.go index 1dc9d995f5..088b5e4bb5 100644 --- a/traffic_ops/traffic_ops_golang/server/servers_update_status.go +++ b/traffic_ops/traffic_ops_golang/server/servers_update_status.go @@ -160,8 +160,7 @@ func GetServerUpdateStatusHandlerV1(w http.ResponseWriter, r *http.Request) { api.WriteRespRaw(w, r, serverUpdateStatus) } -// getServerUpdateStatusV2 supports /servers/all/update_status in addition to /servers/{host_name}/update_status -// This special case is believed to be used nowhere. +// getServerUpdateStatusV2 supports /servers/all/update_status (believed to be used nowhere) in addition to /servers/{host_name}/update_status. func getServerUpdateStatusV2(tx *sql.Tx, cfg *config.Config, hostName string) ([]tc.ServerUpdateStatus, error) { // language=SQL baseSelectStatement := ` From 1c6326a5e176359bd3591f3b25557a755fccf335 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Thu, 30 Jul 2020 10:09:28 -0600 Subject: [PATCH 11/24] log.Error.Printf() -> log.Errorf() --- traffic_ops/traffic_ops_golang/server/servers.go | 2 +- .../traffic_ops_golang/server/servers_assignment.go | 2 +- .../traffic_ops_golang/server/servers_update_status.go | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/traffic_ops/traffic_ops_golang/server/servers.go b/traffic_ops/traffic_ops_golang/server/servers.go index 33fa6b4693..df541b2481 100644 --- a/traffic_ops/traffic_ops_golang/server/servers.go +++ b/traffic_ops/traffic_ops_golang/server/servers.go @@ -860,7 +860,7 @@ func getMidServers(edgeIDs []int, servers map[int]tc.ServerNullable, tx *sqlx.Tx for rows.Next() { var s tc.ServerNullable if err := rows.StructScan(&s); err != nil { - log.Error.Printf("could not scan mid servers: %s\n", err) + log.Errorf("could not scan mid servers: %s\n", err) return nil, nil, err, http.StatusInternalServerError } if s.ID == nil { diff --git a/traffic_ops/traffic_ops_golang/server/servers_assignment.go b/traffic_ops/traffic_ops_golang/server/servers_assignment.go index 53b1bf908b..7d017265e7 100644 --- a/traffic_ops/traffic_ops_golang/server/servers_assignment.go +++ b/traffic_ops/traffic_ops_golang/server/servers_assignment.go @@ -312,7 +312,7 @@ INSERT INTO parameter (config_file, name, value) for rows.Next() { var ID int64 if err := rows.Scan(&ID); err != nil { - log.Error.Printf("could not scan parameter ID: %s\n", err) + log.Errorf("could not scan parameter ID: %s\n", err) return nil, tc.DBError } parameterIds = append(parameterIds, ID) diff --git a/traffic_ops/traffic_ops_golang/server/servers_update_status.go b/traffic_ops/traffic_ops_golang/server/servers_update_status.go index 088b5e4bb5..49e9f932af 100644 --- a/traffic_ops/traffic_ops_golang/server/servers_update_status.go +++ b/traffic_ops/traffic_ops_golang/server/servers_update_status.go @@ -123,7 +123,7 @@ ORDER BY s.id rows, err = tx.Query(selectQuery, hostName) if err != nil { - log.Error.Printf("could not execute query: %s\n", err) + log.Errorf("could not execute query: %s\n", err) return nil, tc.DBError } defer log.Close(rows, "unable to close db connection") @@ -132,7 +132,7 @@ ORDER BY s.id var serverUpdateStatus tc.ServerUpdateStatus var serverType string if err := rows.Scan(&serverUpdateStatus.HostId, &serverUpdateStatus.HostName, &serverType, &serverUpdateStatus.RevalPending, &serverUpdateStatus.UseRevalPending, &serverUpdateStatus.UpdatePending, &serverUpdateStatus.Status, &serverUpdateStatus.ParentPending, &serverUpdateStatus.ParentRevalPending); err != nil { - log.Error.Printf("could not scan server update status: %s\n", err) + log.Errorf("could not scan server update status: %s\n", err) return nil, tc.DBError } updateStatuses = append(updateStatuses, serverUpdateStatus) @@ -208,13 +208,13 @@ ORDER BY s.id if hostName == "all" { rows, err = tx.Query(baseSelectStatement + groupBy) if err != nil { - log.Error.Printf("could not execute select server update status query: %s\n", err) + log.Errorf("could not execute select server update status query: %s\n", err) return nil, tc.DBError } } else { rows, err = tx.Query(baseSelectStatement+` WHERE s.host_name = $1`+groupBy, hostName) if err != nil { - log.Error.Printf("could not execute select server update status by hostname query: %s\n", err) + log.Errorf("could not execute select server update status by hostname query: %s\n", err) return nil, tc.DBError } } @@ -224,7 +224,7 @@ ORDER BY s.id var serverUpdateStatus tc.ServerUpdateStatus var serverType string if err := rows.Scan(&serverUpdateStatus.HostId, &serverUpdateStatus.HostName, &serverType, &serverUpdateStatus.RevalPending, &serverUpdateStatus.UseRevalPending, &serverUpdateStatus.UpdatePending, &serverUpdateStatus.Status, &serverUpdateStatus.ParentPending, &serverUpdateStatus.ParentRevalPending); err != nil { - log.Error.Printf("could not scan server update status: %s\n", err) + log.Errorf("could not scan server update status: %s\n", err) return nil, tc.DBError } if hostName == "all" { //if we want to return the parent data for servers when all is used remove this block From 73ff6a16bc7362a8c3d9b44209aa0c2c49f57113 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Thu, 30 Jul 2020 10:12:59 -0600 Subject: [PATCH 12/24] Add function name to error message for context --- traffic_ops/traffic_ops_golang/server/servers_update_status.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traffic_ops/traffic_ops_golang/server/servers_update_status.go b/traffic_ops/traffic_ops_golang/server/servers_update_status.go index 49e9f932af..3240922427 100644 --- a/traffic_ops/traffic_ops_golang/server/servers_update_status.go +++ b/traffic_ops/traffic_ops_golang/server/servers_update_status.go @@ -126,7 +126,7 @@ ORDER BY s.id log.Errorf("could not execute query: %s\n", err) return nil, tc.DBError } - defer log.Close(rows, "unable to close db connection") + defer log.Close(rows, "getServerUpdateStatus(): unable to close db connection") for rows.Next() { var serverUpdateStatus tc.ServerUpdateStatus From 29c188bd7494b49b05379d977bf74c59a2f072f3 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Thu, 30 Jul 2020 10:17:53 -0600 Subject: [PATCH 13/24] serverUpdateStatus -> us for query result scanning --- .../server/servers_update_status.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/traffic_ops/traffic_ops_golang/server/servers_update_status.go b/traffic_ops/traffic_ops_golang/server/servers_update_status.go index 3240922427..66a28c2abd 100644 --- a/traffic_ops/traffic_ops_golang/server/servers_update_status.go +++ b/traffic_ops/traffic_ops_golang/server/servers_update_status.go @@ -129,13 +129,13 @@ ORDER BY s.id defer log.Close(rows, "getServerUpdateStatus(): unable to close db connection") for rows.Next() { - var serverUpdateStatus tc.ServerUpdateStatus + var us tc.ServerUpdateStatus var serverType string - if err := rows.Scan(&serverUpdateStatus.HostId, &serverUpdateStatus.HostName, &serverType, &serverUpdateStatus.RevalPending, &serverUpdateStatus.UseRevalPending, &serverUpdateStatus.UpdatePending, &serverUpdateStatus.Status, &serverUpdateStatus.ParentPending, &serverUpdateStatus.ParentRevalPending); err != nil { + if err := rows.Scan(&us.HostId, &us.HostName, &serverType, &us.RevalPending, &us.UseRevalPending, &us.UpdatePending, &us.Status, &us.ParentPending, &us.ParentRevalPending); err != nil { log.Errorf("could not scan server update status: %s\n", err) return nil, tc.DBError } - updateStatuses = append(updateStatuses, serverUpdateStatus) + updateStatuses = append(updateStatuses, us) } return updateStatuses, nil } @@ -221,17 +221,17 @@ ORDER BY s.id defer rows.Close() for rows.Next() { - var serverUpdateStatus tc.ServerUpdateStatus + var us tc.ServerUpdateStatus var serverType string - if err := rows.Scan(&serverUpdateStatus.HostId, &serverUpdateStatus.HostName, &serverType, &serverUpdateStatus.RevalPending, &serverUpdateStatus.UseRevalPending, &serverUpdateStatus.UpdatePending, &serverUpdateStatus.Status, &serverUpdateStatus.ParentPending, &serverUpdateStatus.ParentRevalPending); err != nil { + if err := rows.Scan(&us.HostId, &us.HostName, &serverType, &us.RevalPending, &us.UseRevalPending, &us.UpdatePending, &us.Status, &us.ParentPending, &us.ParentRevalPending); err != nil { log.Errorf("could not scan server update status: %s\n", err) return nil, tc.DBError } if hostName == "all" { //if we want to return the parent data for servers when all is used remove this block - serverUpdateStatus.ParentRevalPending = false - serverUpdateStatus.ParentPending = false + us.ParentRevalPending = false + us.ParentPending = false } - updateStatuses = append(updateStatuses, serverUpdateStatus) + updateStatuses = append(updateStatuses, us) } return updateStatuses, nil } From b8ddc95d5ab4a393bb2923761b1a09512d61dd0a Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Thu, 30 Jul 2020 10:23:04 -0600 Subject: [PATCH 14/24] Bind rows and err at their first usage instead of predeclaring --- .../traffic_ops_golang/server/servers_update_status.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/traffic_ops/traffic_ops_golang/server/servers_update_status.go b/traffic_ops/traffic_ops_golang/server/servers_update_status.go index 66a28c2abd..c5559417d1 100644 --- a/traffic_ops/traffic_ops_golang/server/servers_update_status.go +++ b/traffic_ops/traffic_ops_golang/server/servers_update_status.go @@ -48,8 +48,6 @@ func GetServerUpdateStatusHandler(w http.ResponseWriter, r *http.Request) { func getServerUpdateStatus(tx *sql.Tx, cfg *config.Config, hostName string) ([]tc.ServerUpdateStatus, error) { updateStatuses := []tc.ServerUpdateStatus{} - var rows *sql.Rows - var err error selectQuery := ` /* topology_ancestors finds the ancestor topology nodes of the topology node for @@ -121,7 +119,7 @@ GROUP BY s.id, s.host_name, type.name, server_reval_pending, use_reval_pending.v ORDER BY s.id ` - rows, err = tx.Query(selectQuery, hostName) + rows, err := tx.Query(selectQuery, hostName) if err != nil { log.Errorf("could not execute query: %s\n", err) return nil, tc.DBError From 5ef79b21cbfb48c89cb01f41afff5b423e963c68 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Thu, 30 Jul 2020 10:34:18 -0600 Subject: [PATCH 15/24] Use tc.CacheStatusOffline instead of hard-coded 'OFFLINE' --- .../server/servers_update_status.go | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/traffic_ops/traffic_ops_golang/server/servers_update_status.go b/traffic_ops/traffic_ops_golang/server/servers_update_status.go index c5559417d1..ecd9dd3917 100644 --- a/traffic_ops/traffic_ops_golang/server/servers_update_status.go +++ b/traffic_ops/traffic_ops_golang/server/servers_update_status.go @@ -51,18 +51,18 @@ func getServerUpdateStatus(tx *sql.Tx, cfg *config.Config, hostName string) ([]t selectQuery := ` /* topology_ancestors finds the ancestor topology nodes of the topology node for - * the cachegroup containing server $1. + * the cachegroup containing server $2. */ WITH RECURSIVE topology_ancestors AS ( /* This is the base case of the recursive CTE, the topology node for the - * cachegroup containing server $1. + * cachegroup containing server $2. */ SELECT tcp.child parent, tc.cachegroup FROM "server" s JOIN cachegroup c ON s.cachegroup = c.id JOIN topology_cachegroup tc ON c."name" = tc.cachegroup JOIN topology_cachegroup_parents tcp ON tc.id = tcp.child - WHERE s.host_name = $1 + WHERE s.host_name = $2 UNION ALL /* Find all direct topology parent nodes tc of a given topology ancestor ta. */ SELECT tcp.parent, tc.cachegroup @@ -77,13 +77,13 @@ SELECT s.id, s.cachegroup, s.cdn_id, s.upd_pending, s.reval_pending, s.status FROM server s JOIN cachegroup c ON s.cachegroup = c.id JOIN topology_ancestors ta ON c."name" = ta.cachegroup -/* Filter out host_name $1 */ -WHERE s.host_name != $1 +/* Filter out host_name $2 */ +WHERE s.host_name != $2 ), parentservers AS ( SELECT ps.id, ps.cachegroup, ps.cdn_id, ps.upd_pending, ps.reval_pending, ps.status FROM server ps LEFT JOIN status AS pstatus ON pstatus.id = ps.status - WHERE pstatus.name != 'OFFLINE' + WHERE pstatus.name != $1 ), use_reval_pending AS ( SELECT value::BOOLEAN FROM parameter @@ -114,12 +114,12 @@ LEFT JOIN cachegroup cg ON s.cachegroup = cg.id LEFT JOIN type ON type.id = s.type LEFT JOIN parentservers ps ON ps.cachegroup = cg.parent_cachegroup_id AND ps.cdn_id = s.cdn_id -WHERE s.host_name = $1 +WHERE s.host_name = $2 GROUP BY s.id, s.host_name, type.name, server_reval_pending, use_reval_pending.value, s.upd_pending, status.name ORDER BY s.id ` - rows, err := tx.Query(selectQuery, hostName) + rows, err := tx.Query(selectQuery, tc.CacheStatusOffline, hostName) if err != nil { log.Errorf("could not execute query: %s\n", err) return nil, tc.DBError @@ -166,7 +166,7 @@ WITH parentservers AS ( SELECT ps.id, ps.cachegroup, ps.cdn_id, ps.upd_pending, ps.reval_pending FROM server ps LEFT JOIN status AS pstatus ON pstatus.id = ps.status - WHERE pstatus.name != 'OFFLINE' + WHERE pstatus.name != $1 ), use_reval_pending AS ( SELECT value::BOOLEAN FROM parameter @@ -210,7 +210,7 @@ ORDER BY s.id return nil, tc.DBError } } else { - rows, err = tx.Query(baseSelectStatement+` WHERE s.host_name = $1`+groupBy, hostName) + rows, err = tx.Query(baseSelectStatement+` WHERE s.host_name = $2`+groupBy, tc.CacheStatusOffline, hostName) if err != nil { log.Errorf("could not execute select server update status by hostname query: %s\n", err) return nil, tc.DBError From ccb41948c29eb67094ae9d3a7639fd70085a15d5 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Thu, 30 Jul 2020 10:47:18 -0600 Subject: [PATCH 16/24] Use tc.UseRevalPendingParameterName instead of hard-coded 'use_reval_pending' --- lib/go-tc/enum.go | 6 ++++++ .../invalidationjobs/invalidationjobs.go | 2 +- .../server/servers_update_status.go | 20 +++++++++---------- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/lib/go-tc/enum.go b/lib/go-tc/enum.go index f9614e7f0d..c8ec22aeb0 100644 --- a/lib/go-tc/enum.go +++ b/lib/go-tc/enum.go @@ -84,6 +84,12 @@ const ( const GlobalProfileName = "GLOBAL" +// ParameterName represents the name of a Traffic Ops parameter meant to belong in a Traffic Ops config file. +type ParameterName string + +// UseRevalPendingParameterName is the name of a parameter which tells whether or not Traffic Ops should use pending revalidation jobs. +const UseRevalPendingParameterName = ParameterName("use_reval_pending") + func (c CacheName) String() string { return string(c) } diff --git a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go index 78dfec7f7c..020a1b02b8 100644 --- a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go +++ b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go @@ -634,7 +634,7 @@ func Delete(w http.ResponseWriter, r *http.Request) { func setRevalFlags(d interface{}, tx *sql.Tx) error { var useReval string - row := tx.QueryRow(`SELECT value FROM parameter WHERE name='use_reval_pending' AND config_file='global'`) + row := tx.QueryRow(`SELECT value FROM parameter WHERE name=$1 AND config_file='global'`, tc.UseRevalPendingParameterName) if err := row.Scan(&useReval); err != nil { if err != sql.ErrNoRows { return err diff --git a/traffic_ops/traffic_ops_golang/server/servers_update_status.go b/traffic_ops/traffic_ops_golang/server/servers_update_status.go index ecd9dd3917..7848e41bc2 100644 --- a/traffic_ops/traffic_ops_golang/server/servers_update_status.go +++ b/traffic_ops/traffic_ops_golang/server/servers_update_status.go @@ -51,18 +51,18 @@ func getServerUpdateStatus(tx *sql.Tx, cfg *config.Config, hostName string) ([]t selectQuery := ` /* topology_ancestors finds the ancestor topology nodes of the topology node for - * the cachegroup containing server $2. + * the cachegroup containing server $3. */ WITH RECURSIVE topology_ancestors AS ( /* This is the base case of the recursive CTE, the topology node for the - * cachegroup containing server $2. + * cachegroup containing server $3. */ SELECT tcp.child parent, tc.cachegroup FROM "server" s JOIN cachegroup c ON s.cachegroup = c.id JOIN topology_cachegroup tc ON c."name" = tc.cachegroup JOIN topology_cachegroup_parents tcp ON tc.id = tcp.child - WHERE s.host_name = $2 + WHERE s.host_name = $3 UNION ALL /* Find all direct topology parent nodes tc of a given topology ancestor ta. */ SELECT tcp.parent, tc.cachegroup @@ -77,8 +77,8 @@ SELECT s.id, s.cachegroup, s.cdn_id, s.upd_pending, s.reval_pending, s.status FROM server s JOIN cachegroup c ON s.cachegroup = c.id JOIN topology_ancestors ta ON c."name" = ta.cachegroup -/* Filter out host_name $2 */ -WHERE s.host_name != $2 +/* Filter out host_name $3 */ +WHERE s.host_name != $3 ), parentservers AS ( SELECT ps.id, ps.cachegroup, ps.cdn_id, ps.upd_pending, ps.reval_pending, ps.status FROM server ps @@ -87,7 +87,7 @@ WHERE s.host_name != $2 ), use_reval_pending AS ( SELECT value::BOOLEAN FROM parameter - WHERE name = 'use_reval_pending' + WHERE name = $2 AND config_file = 'global' UNION ALL SELECT FALSE FETCH FIRST 1 ROW ONLY ) @@ -114,12 +114,12 @@ LEFT JOIN cachegroup cg ON s.cachegroup = cg.id LEFT JOIN type ON type.id = s.type LEFT JOIN parentservers ps ON ps.cachegroup = cg.parent_cachegroup_id AND ps.cdn_id = s.cdn_id -WHERE s.host_name = $2 +WHERE s.host_name = $3 GROUP BY s.id, s.host_name, type.name, server_reval_pending, use_reval_pending.value, s.upd_pending, status.name ORDER BY s.id ` - rows, err := tx.Query(selectQuery, tc.CacheStatusOffline, hostName) + rows, err := tx.Query(selectQuery, tc.CacheStatusOffline, tc.UseRevalPendingParameterName, hostName) if err != nil { log.Errorf("could not execute query: %s\n", err) return nil, tc.DBError @@ -170,7 +170,7 @@ WITH parentservers AS ( ), use_reval_pending AS ( SELECT value::BOOLEAN FROM parameter - WHERE name = 'use_reval_pending' + WHERE name = $2 AND config_file = 'global' UNION ALL SELECT FALSE FETCH FIRST 1 ROW ONLY ) @@ -210,7 +210,7 @@ ORDER BY s.id return nil, tc.DBError } } else { - rows, err = tx.Query(baseSelectStatement+` WHERE s.host_name = $2`+groupBy, tc.CacheStatusOffline, hostName) + rows, err = tx.Query(baseSelectStatement+` WHERE s.host_name = $3`+groupBy, tc.CacheStatusOffline, tc.UseRevalPendingParameterName, hostName) if err != nil { log.Errorf("could not execute select server update status by hostname query: %s\n", err) return nil, tc.DBError From b1e912424d61eab1eba24ed745bd7428df943bbf Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Thu, 30 Jul 2020 11:15:06 -0600 Subject: [PATCH 17/24] Use tc.GlobalConfigFileName instead of hard-coded 'global' --- lib/go-tc/enum.go | 6 ++++++ traffic_ops/traffic_ops_golang/ats/db.go | 6 +++--- .../traffic_ops_golang/crconfig/servers.go | 2 +- .../invalidationjobs/invalidationjobs.go | 2 +- traffic_ops/traffic_ops_golang/login/login.go | 4 ++-- .../traffic_ops_golang/login/register.go | 2 +- .../server/servers_update_status.go | 20 +++++++++---------- .../systeminfo/system_info.go | 4 ++-- .../systeminfo/system_info_test.go | 3 ++- 9 files changed, 28 insertions(+), 21 deletions(-) diff --git a/lib/go-tc/enum.go b/lib/go-tc/enum.go index c8ec22aeb0..b0fdee0558 100644 --- a/lib/go-tc/enum.go +++ b/lib/go-tc/enum.go @@ -90,6 +90,12 @@ type ParameterName string // UseRevalPendingParameterName is the name of a parameter which tells whether or not Traffic Ops should use pending revalidation jobs. const UseRevalPendingParameterName = ParameterName("use_reval_pending") +// ConfigFileName represents the name of a Traffic Ops config file. +type ConfigFileName string + +// GlobalConfigFileName is the name of the global Traffic Ops config file. +const GlobalConfigFileName = ConfigFileName("global") + func (c CacheName) String() string { return string(c) } diff --git a/traffic_ops/traffic_ops_golang/ats/db.go b/traffic_ops/traffic_ops_golang/ats/db.go index 6e22b9d9b9..bce348850d 100644 --- a/traffic_ops/traffic_ops_golang/ats/db.go +++ b/traffic_ops/traffic_ops_golang/ats/db.go @@ -687,7 +687,7 @@ func GetATSMajorVersion(tx *sql.Tx, serverProfileID atscfg.ProfileID) (int, erro // GetTMParams returns the global "tm.url" and "tm.rev_proxy.url" parameters, and any error. If either param doesn't exist, an empty string is returned without error. func GetTMParams(tx *sql.Tx) (TMParams, error) { - rows, err := tx.Query(`SELECT name, value from parameter where config_file = 'global' AND (name = 'tm.url' OR name = 'tm.rev_proxy.url')`) + rows, err := tx.Query(`SELECT name, value from parameter where config_file = $1 AND (name = 'tm.url' OR name = 'tm.rev_proxy.url')`, tc.GlobalConfigFileName) if err != nil { return TMParams{}, errors.New("querying: " + err.Error()) } @@ -949,9 +949,9 @@ SELECT FROM parameter p WHERE - (p.name = 'tm.toolname' OR p.name = 'tm.url') AND p.config_file = 'global' + (p.name = 'tm.toolname' OR p.name = 'tm.url') AND p.config_file = $1 ` - rows, err := tx.Query(qry) + rows, err := tx.Query(qry, tc.GlobalConfigFileName) if err != nil { return "", "", errors.New("querying: " + err.Error()) } diff --git a/traffic_ops/traffic_ops_golang/crconfig/servers.go b/traffic_ops/traffic_ops_golang/crconfig/servers.go index 0b0265ac85..ac5951d966 100644 --- a/traffic_ops/traffic_ops_golang/crconfig/servers.go +++ b/traffic_ops/traffic_ops_golang/crconfig/servers.go @@ -464,7 +464,7 @@ func getCDNNameFromID(id int, tx *sql.Tx) (string, bool, error) { // getGlobalParam returns the global parameter with the requested name, whether it existed, and any error func getGlobalParam(tx *sql.Tx, name string) (string, bool, error) { val := "" - if err := tx.QueryRow(`SELECT value FROM parameter WHERE config_file = 'global' and name = $1`, name).Scan(&val); err != nil { + if err := tx.QueryRow(`SELECT value FROM parameter WHERE config_file = $1 and name = $2`, tc.GlobalConfigFileName, name).Scan(&val); err != nil { if err == sql.ErrNoRows { return "", false, nil } diff --git a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go index 020a1b02b8..7210a265fe 100644 --- a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go +++ b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go @@ -634,7 +634,7 @@ func Delete(w http.ResponseWriter, r *http.Request) { func setRevalFlags(d interface{}, tx *sql.Tx) error { var useReval string - row := tx.QueryRow(`SELECT value FROM parameter WHERE name=$1 AND config_file='global'`, tc.UseRevalPendingParameterName) + row := tx.QueryRow(`SELECT value FROM parameter WHERE name=$1 AND config_file=$2`, tc.UseRevalPendingParameterName, tc.GlobalConfigFileName) if err := row.Scan(&useReval); err != nil { if err != sql.ErrNoRows { return err diff --git a/traffic_ops/traffic_ops_golang/login/login.go b/traffic_ops/traffic_ops_golang/login/login.go index bf013164b4..c1c0bc2388 100644 --- a/traffic_ops/traffic_ops_golang/login/login.go +++ b/traffic_ops/traffic_ops_golang/login/login.go @@ -57,7 +57,7 @@ const instanceNameQuery = ` SELECT value FROM parameter WHERE name='tm.instance_name' AND - config_file='global' + config_file=$1 ` const userQueryByEmail = `SELECT EXISTS(SELECT * FROM tm_user WHERE email=$1)` const setTokenQuery = `UPDATE tm_user SET token=$1 WHERE email=$2` @@ -419,7 +419,7 @@ func setToken(addr rfc.EmailAddress, tx *sql.Tx) (string, error) { func createMsg(addr rfc.EmailAddress, t string, db *sqlx.DB, c config.ConfigPortal) ([]byte, error) { var instanceName string - row := db.QueryRow(instanceNameQuery) + row := db.QueryRow(instanceNameQuery, tc.GlobalConfigFileName) if err := row.Scan(&instanceName); err != nil { return nil, err } diff --git a/traffic_ops/traffic_ops_golang/login/register.go b/traffic_ops/traffic_ops_golang/login/register.go index a00df6f46a..dcc7c046c4 100644 --- a/traffic_ops/traffic_ops_golang/login/register.go +++ b/traffic_ops/traffic_ops_golang/login/register.go @@ -129,7 +129,7 @@ Subject: {{.InstanceName}} New User Registration` + "\r\n\r" + ` func createRegistrationMsg(addr rfc.EmailAddress, t string, tx *sql.Tx, c config.ConfigPortal) ([]byte, error) { var instanceName string - if err := tx.QueryRow(instanceNameQuery).Scan(&instanceName); err != nil { + if err := tx.QueryRow(instanceNameQuery, tc.GlobalConfigFileName).Scan(&instanceName); err != nil { return nil, err } diff --git a/traffic_ops/traffic_ops_golang/server/servers_update_status.go b/traffic_ops/traffic_ops_golang/server/servers_update_status.go index 7848e41bc2..5abb7c3686 100644 --- a/traffic_ops/traffic_ops_golang/server/servers_update_status.go +++ b/traffic_ops/traffic_ops_golang/server/servers_update_status.go @@ -51,18 +51,18 @@ func getServerUpdateStatus(tx *sql.Tx, cfg *config.Config, hostName string) ([]t selectQuery := ` /* topology_ancestors finds the ancestor topology nodes of the topology node for - * the cachegroup containing server $3. + * the cachegroup containing server $4. */ WITH RECURSIVE topology_ancestors AS ( /* This is the base case of the recursive CTE, the topology node for the - * cachegroup containing server $3. + * cachegroup containing server $4. */ SELECT tcp.child parent, tc.cachegroup FROM "server" s JOIN cachegroup c ON s.cachegroup = c.id JOIN topology_cachegroup tc ON c."name" = tc.cachegroup JOIN topology_cachegroup_parents tcp ON tc.id = tcp.child - WHERE s.host_name = $3 + WHERE s.host_name = $4 UNION ALL /* Find all direct topology parent nodes tc of a given topology ancestor ta. */ SELECT tcp.parent, tc.cachegroup @@ -77,8 +77,8 @@ SELECT s.id, s.cachegroup, s.cdn_id, s.upd_pending, s.reval_pending, s.status FROM server s JOIN cachegroup c ON s.cachegroup = c.id JOIN topology_ancestors ta ON c."name" = ta.cachegroup -/* Filter out host_name $3 */ -WHERE s.host_name != $3 +/* Filter out host_name $4 */ +WHERE s.host_name != $4 ), parentservers AS ( SELECT ps.id, ps.cachegroup, ps.cdn_id, ps.upd_pending, ps.reval_pending, ps.status FROM server ps @@ -88,7 +88,7 @@ WHERE s.host_name != $3 SELECT value::BOOLEAN FROM parameter WHERE name = $2 - AND config_file = 'global' + AND config_file = $3 UNION ALL SELECT FALSE FETCH FIRST 1 ROW ONLY ) SELECT @@ -114,12 +114,12 @@ LEFT JOIN cachegroup cg ON s.cachegroup = cg.id LEFT JOIN type ON type.id = s.type LEFT JOIN parentservers ps ON ps.cachegroup = cg.parent_cachegroup_id AND ps.cdn_id = s.cdn_id -WHERE s.host_name = $3 +WHERE s.host_name = $4 GROUP BY s.id, s.host_name, type.name, server_reval_pending, use_reval_pending.value, s.upd_pending, status.name ORDER BY s.id ` - rows, err := tx.Query(selectQuery, tc.CacheStatusOffline, tc.UseRevalPendingParameterName, hostName) + rows, err := tx.Query(selectQuery, tc.CacheStatusOffline, tc.UseRevalPendingParameterName, tc.GlobalConfigFileName, hostName) if err != nil { log.Errorf("could not execute query: %s\n", err) return nil, tc.DBError @@ -171,7 +171,7 @@ WITH parentservers AS ( SELECT value::BOOLEAN FROM parameter WHERE name = $2 - AND config_file = 'global' + AND config_file = $3 UNION ALL SELECT FALSE FETCH FIRST 1 ROW ONLY ) SELECT @@ -210,7 +210,7 @@ ORDER BY s.id return nil, tc.DBError } } else { - rows, err = tx.Query(baseSelectStatement+` WHERE s.host_name = $3`+groupBy, tc.CacheStatusOffline, tc.UseRevalPendingParameterName, hostName) + rows, err = tx.Query(baseSelectStatement+` WHERE s.host_name = $4`+groupBy, tc.CacheStatusOffline, tc.UseRevalPendingParameterName, hostName) if err != nil { log.Errorf("could not execute select server update status by hostname query: %s\n", err) return nil, tc.DBError diff --git a/traffic_ops/traffic_ops_golang/systeminfo/system_info.go b/traffic_ops/traffic_ops_golang/systeminfo/system_info.go index b9c14454ba..9831fa781e 100644 --- a/traffic_ops/traffic_ops_golang/systeminfo/system_info.go +++ b/traffic_ops/traffic_ops_golang/systeminfo/system_info.go @@ -51,9 +51,9 @@ SELECT FROM parameter p WHERE - p.config_file = 'global' + p.config_file = $1 ` - rows, err := tx.Queryx(q) + rows, err := tx.Queryx(q, tc.GlobalConfigFileName) if err != nil { return nil, errors.New("querying system info global parameters: " + err.Error()) } diff --git a/traffic_ops/traffic_ops_golang/systeminfo/system_info_test.go b/traffic_ops/traffic_ops_golang/systeminfo/system_info_test.go index de46069355..8c5a66c781 100644 --- a/traffic_ops/traffic_ops_golang/systeminfo/system_info_test.go +++ b/traffic_ops/traffic_ops_golang/systeminfo/system_info_test.go @@ -22,6 +22,7 @@ package systeminfo import ( "context" "encoding/json" + "fmt" "testing" "time" @@ -93,7 +94,7 @@ func TestGetSystemInfo(t *testing.T) { } mock.ExpectBegin() - mock.ExpectQuery("SELECT.*WHERE p.config_file = 'global'").WillReturnRows(rows) + mock.ExpectQuery(fmt.Sprintf("SELECT.*WHERE p.config_file = '%s'", tc.GlobalConfigFileName)).WillReturnRows(rows) dbCtx, _ := context.WithTimeout(context.TODO(), time.Duration(10)*time.Second) tx, err := db.BeginTxx(dbCtx, nil) From fe0fc4e374c7eaeb3e0b1023e354d17cfa076c4d Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Thu, 30 Jul 2020 13:44:45 -0600 Subject: [PATCH 18/24] Update test SQL regex to reference parameter --- traffic_ops/traffic_ops_golang/systeminfo/system_info_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/traffic_ops/traffic_ops_golang/systeminfo/system_info_test.go b/traffic_ops/traffic_ops_golang/systeminfo/system_info_test.go index 8c5a66c781..6c60829c93 100644 --- a/traffic_ops/traffic_ops_golang/systeminfo/system_info_test.go +++ b/traffic_ops/traffic_ops_golang/systeminfo/system_info_test.go @@ -22,7 +22,6 @@ package systeminfo import ( "context" "encoding/json" - "fmt" "testing" "time" @@ -94,7 +93,7 @@ func TestGetSystemInfo(t *testing.T) { } mock.ExpectBegin() - mock.ExpectQuery(fmt.Sprintf("SELECT.*WHERE p.config_file = '%s'", tc.GlobalConfigFileName)).WillReturnRows(rows) + mock.ExpectQuery(`SELECT.*WHERE p.config_file = \$1`).WillReturnRows(rows) dbCtx, _ := context.WithTimeout(context.TODO(), time.Duration(10)*time.Second) tx, err := db.BeginTxx(dbCtx, nil) From 8f9914f5b9afaa7bd76e367ed8bfd12eb68b47f4 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Fri, 31 Jul 2020 12:32:47 -0600 Subject: [PATCH 19/24] Rename function to getServerUpdateStatusV1() since it is called from GetServerUpdateStatusHandlerV1() --- .../traffic_ops_golang/server/servers_update_status.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/traffic_ops/traffic_ops_golang/server/servers_update_status.go b/traffic_ops/traffic_ops_golang/server/servers_update_status.go index 5abb7c3686..de151dff57 100644 --- a/traffic_ops/traffic_ops_golang/server/servers_update_status.go +++ b/traffic_ops/traffic_ops_golang/server/servers_update_status.go @@ -150,7 +150,7 @@ func GetServerUpdateStatusHandlerV1(w http.ResponseWriter, r *http.Request) { } defer inf.Close() - serverUpdateStatus, err := getServerUpdateStatusV2(inf.Tx.Tx, inf.Config, inf.Params["host_name"]) + serverUpdateStatus, err := getServerUpdateStatusV1(inf.Tx.Tx, inf.Config, inf.Params["host_name"]) if err != nil { api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, err) return @@ -158,8 +158,8 @@ func GetServerUpdateStatusHandlerV1(w http.ResponseWriter, r *http.Request) { api.WriteRespRaw(w, r, serverUpdateStatus) } -// getServerUpdateStatusV2 supports /servers/all/update_status (believed to be used nowhere) in addition to /servers/{host_name}/update_status. -func getServerUpdateStatusV2(tx *sql.Tx, cfg *config.Config, hostName string) ([]tc.ServerUpdateStatus, error) { +// getServerUpdateStatusV1 supports /servers/all/update_status (believed to be used nowhere) in addition to /servers/{host_name}/update_status. +func getServerUpdateStatusV1(tx *sql.Tx, cfg *config.Config, hostName string) ([]tc.ServerUpdateStatus, error) { // language=SQL baseSelectStatement := ` WITH parentservers AS ( From c3d4a4ceb59aba192a2804d3cd059532da050e79 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Fri, 31 Jul 2020 12:30:04 -0600 Subject: [PATCH 20/24] Include prepared statement parameters in legacy query commands --- .../traffic_ops_golang/server/servers_update_status.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/traffic_ops/traffic_ops_golang/server/servers_update_status.go b/traffic_ops/traffic_ops_golang/server/servers_update_status.go index de151dff57..6e0f817a6d 100644 --- a/traffic_ops/traffic_ops_golang/server/servers_update_status.go +++ b/traffic_ops/traffic_ops_golang/server/servers_update_status.go @@ -204,13 +204,13 @@ ORDER BY s.id var rows *sql.Rows var err error if hostName == "all" { - rows, err = tx.Query(baseSelectStatement + groupBy) + rows, err = tx.Query(baseSelectStatement + groupBy, tc.CacheStatusOffline, tc.UseRevalPendingParameterName, tc.GlobalConfigFileName) if err != nil { log.Errorf("could not execute select server update status query: %s\n", err) return nil, tc.DBError } } else { - rows, err = tx.Query(baseSelectStatement+` WHERE s.host_name = $4`+groupBy, tc.CacheStatusOffline, tc.UseRevalPendingParameterName, hostName) + rows, err = tx.Query(baseSelectStatement+` WHERE s.host_name = $4`+groupBy, tc.CacheStatusOffline, tc.UseRevalPendingParameterName, tc.GlobalConfigFileName, hostName) if err != nil { log.Errorf("could not execute select server update status by hostname query: %s\n", err) return nil, tc.DBError From f1da3aba821da685ac03550b30c4286765991397 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Fri, 31 Jul 2020 15:41:15 -0600 Subject: [PATCH 21/24] Filter out server's cachegroup, not just the server itself --- .../traffic_ops_golang/server/servers_update_status.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/traffic_ops/traffic_ops_golang/server/servers_update_status.go b/traffic_ops/traffic_ops_golang/server/servers_update_status.go index 6e0f817a6d..b694a4a5aa 100644 --- a/traffic_ops/traffic_ops_golang/server/servers_update_status.go +++ b/traffic_ops/traffic_ops_golang/server/servers_update_status.go @@ -77,8 +77,8 @@ SELECT s.id, s.cachegroup, s.cdn_id, s.upd_pending, s.reval_pending, s.status FROM server s JOIN cachegroup c ON s.cachegroup = c.id JOIN topology_ancestors ta ON c."name" = ta.cachegroup -/* Filter out host_name $4 */ -WHERE s.host_name != $4 +/* Filter out cachegroup of host_name $4 */ +WHERE s.cachegroup != (SELECT s.cachegroup FROM server s WHERE s.host_name = $4) ), parentservers AS ( SELECT ps.id, ps.cachegroup, ps.cdn_id, ps.upd_pending, ps.reval_pending, ps.status FROM server ps From f99605c1aab861911950d07325453f919eb0c5c6 Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Mon, 3 Aug 2020 16:12:46 -0600 Subject: [PATCH 22/24] Fix parent upd_pending to return an accurate value if topology and legacy cachegroup parentage simultaneously exist --- .../traffic_ops_golang/server/servers_update_status.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/traffic_ops/traffic_ops_golang/server/servers_update_status.go b/traffic_ops/traffic_ops_golang/server/servers_update_status.go index b694a4a5aa..b3e5de6f4d 100644 --- a/traffic_ops/traffic_ops_golang/server/servers_update_status.go +++ b/traffic_ops/traffic_ops_golang/server/servers_update_status.go @@ -99,13 +99,15 @@ SELECT use_reval_pending.value, s.upd_pending, status.name AS status, - COALESCE(BOOL_OR(ps.upd_pending), + /* True if the cachegroup parent or any ancestor topology node has pending updates */ TRUE IN ( - SELECT sta.upd_pending FROM server_topology_ancestors sta) + SELECT sta.upd_pending FROM server_topology_ancestors sta + UNION SELECT COALESCE(BOOL_OR(ps.upd_pending), FALSE) ) AS parent_upd_pending, - COALESCE(BOOL_OR(ps.reval_pending), + /* True if the cachegroup parent or any ancestor topology node has pending updates */ TRUE IN ( - SELECT sta.reval_pending FROM server_topology_ancestors sta) + SELECT sta.reval_pending FROM server_topology_ancestors sta + UNION SELECT COALESCE(BOOL_OR(ps.reval_pending), FALSE) ) AS parent_reval_pending FROM use_reval_pending, server s From 7809db694467d18de7d461136a52a36252c653ab Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Mon, 3 Aug 2020 16:14:53 -0600 Subject: [PATCH 23/24] Revalidation, not updates --- .../traffic_ops_golang/server/servers_update_status.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/traffic_ops/traffic_ops_golang/server/servers_update_status.go b/traffic_ops/traffic_ops_golang/server/servers_update_status.go index b3e5de6f4d..2e46b4af09 100644 --- a/traffic_ops/traffic_ops_golang/server/servers_update_status.go +++ b/traffic_ops/traffic_ops_golang/server/servers_update_status.go @@ -99,12 +99,12 @@ SELECT use_reval_pending.value, s.upd_pending, status.name AS status, - /* True if the cachegroup parent or any ancestor topology node has pending updates */ + /* True if the cachegroup parent or any ancestor topology node has pending updates. */ TRUE IN ( SELECT sta.upd_pending FROM server_topology_ancestors sta UNION SELECT COALESCE(BOOL_OR(ps.upd_pending), FALSE) ) AS parent_upd_pending, - /* True if the cachegroup parent or any ancestor topology node has pending updates */ + /* True if the cachegroup parent or any ancestor topology node has pending revalidation. */ TRUE IN ( SELECT sta.reval_pending FROM server_topology_ancestors sta UNION SELECT COALESCE(BOOL_OR(ps.reval_pending), FALSE) From 14f17f486feeb229ce22e60e6fd3061e3621158d Mon Sep 17 00:00:00 2001 From: Zach Hoffman Date: Mon, 3 Aug 2020 16:17:29 -0600 Subject: [PATCH 24/24] whitespace --- .../traffic_ops_golang/server/servers_update_status.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/traffic_ops/traffic_ops_golang/server/servers_update_status.go b/traffic_ops/traffic_ops_golang/server/servers_update_status.go index 2e46b4af09..6372d51340 100644 --- a/traffic_ops/traffic_ops_golang/server/servers_update_status.go +++ b/traffic_ops/traffic_ops_golang/server/servers_update_status.go @@ -101,12 +101,12 @@ SELECT status.name AS status, /* True if the cachegroup parent or any ancestor topology node has pending updates. */ TRUE IN ( - SELECT sta.upd_pending FROM server_topology_ancestors sta + SELECT sta.upd_pending FROM server_topology_ancestors sta UNION SELECT COALESCE(BOOL_OR(ps.upd_pending), FALSE) ) AS parent_upd_pending, /* True if the cachegroup parent or any ancestor topology node has pending revalidation. */ TRUE IN ( - SELECT sta.reval_pending FROM server_topology_ancestors sta + SELECT sta.reval_pending FROM server_topology_ancestors sta UNION SELECT COALESCE(BOOL_OR(ps.reval_pending), FALSE) ) AS parent_reval_pending FROM use_reval_pending,