From 42ce70887160f0408713d1c01e09906c81ac7435 Mon Sep 17 00:00:00 2001 From: Rawlin Peters Date: Wed, 8 Sep 2021 12:18:03 -0600 Subject: [PATCH] Make `t3c` request less unnecessary DSS and jobs data Use the new query parameters supported by Traffic Ops to reduce the amount of unnecessary data requested from the /deliveryserviceserver and /jobs APIs. Closes: #5674 Closes: #6034 --- CHANGELOG.md | 3 ++ cache-config/t3cutil/getdatacfg.go | 52 +++++++++---------- cache-config/t3cutil/toreq/clientfuncs.go | 15 ++++-- cache-config/t3cutil/toreq/clienthlp.go | 4 +- .../cache/traffic_ops_ort.crontab | 2 +- 5 files changed, 43 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d796105d0..eb4d0a52d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Added - [#5674](https://github.com/apache/trafficcontrol/issues/5674) Added new query parameters `cdn` and `maxRevalDurationDays` to the `GET /api/x/jobs` Traffic Ops API to filter by CDN name and within the start_time window defined by the `maxRevalDurationDays` GLOBAL profile parameter, respectively. +### Changed +- Updated `t3c` to request less unnecessary deliveryservice-server assignment and invalidation jobs data via new query params supported by Traffic Ops + ## unreleased ### Added - [#4982](https://github.com/apache/trafficcontrol/issues/4982) Added the ability to support queueing updates by server type and profile diff --git a/cache-config/t3cutil/getdatacfg.go b/cache-config/t3cutil/getdatacfg.go index 9f021495ff..84e6fdfc79 100644 --- a/cache-config/t3cutil/getdatacfg.go +++ b/cache-config/t3cutil/getdatacfg.go @@ -332,7 +332,7 @@ func GetConfigData(toClient *toreq.TOClient, disableProxy bool, cacheHostName st if oldCfg != nil { reqHdr = MakeReqHdr(oldCfg.MetaData.DeliveryServiceServers) } - dss, reqInf, err := toClient.GetDeliveryServiceServers(nil, nil, reqHdr) + dss, reqInf, err := toClient.GetDeliveryServiceServers(nil, nil, *server.CDNName, reqHdr) if err != nil { return errors.New("getting delivery service servers: " + err.Error()) } @@ -515,7 +515,30 @@ func GetConfigData(toClient *toreq.TOClient, disableProxy bool, cacheHostName st } return nil } - fs := []func() error{dsF, serverParamsF, cdnF, profileF} + jobsF := func() error { + defer func(start time.Time) { log.Infof("jobsF took %v\n", time.Since(start)) }(time.Now()) + { + reqHdr := (http.Header)(nil) + if oldCfg != nil { + reqHdr = MakeReqHdr(oldCfg.MetaData.Jobs) + } + jobs, reqInf, err := toClient.GetJobs(reqHdr, *server.CDNName) + if err != nil { + return errors.New("getting jobs: " + err.Error()) + } + if reqInf.StatusCode == http.StatusNotModified { + log.Infof("Getting config: %v not modified, using old config", "Jobs") + toData.Jobs = oldCfg.Jobs + } else { + log.Infof("Getting config: %v is modified, using new response", "Jobs") + toData.Jobs = jobs + } + toData.MetaData.Jobs = MakeReqMetaData(reqInf.RespHeaders) + toIPs.Store(reqInf.RemoteAddr, nil) + } + return nil + } + fs := []func() error{dsF, serverParamsF, cdnF, profileF, jobsF} if !revalOnly { fs = append([]func() error{sslF}, fs...) // skip ssl keys for reval only, which doesn't need them } @@ -545,29 +568,6 @@ func GetConfigData(toClient *toreq.TOClient, disableProxy bool, cacheHostName st } return nil } - jobsF := func() error { - defer func(start time.Time) { log.Infof("jobsF took %v\n", time.Since(start)) }(time.Now()) - { - reqHdr := (http.Header)(nil) - if oldCfg != nil { - reqHdr = MakeReqHdr(oldCfg.MetaData.Jobs) - } - jobs, reqInf, err := toClient.GetJobs(reqHdr) // TODO add cdn query param to jobs endpoint - if err != nil { - return errors.New("getting jobs: " + err.Error()) - } - if reqInf.StatusCode == http.StatusNotModified { - log.Infof("Getting config: %v not modified, using old config", "Jobs") - toData.Jobs = oldCfg.Jobs - } else { - log.Infof("Getting config: %v is modified, using new response", "Jobs") - toData.Jobs = jobs - } - toData.MetaData.Jobs = MakeReqMetaData(reqInf.RespHeaders) - toIPs.Store(reqInf.RemoteAddr, nil) - } - return nil - } capsF := func() error { defer func(start time.Time) { log.Infof("capsF took %v\n", time.Since(start)) }(time.Now()) { @@ -739,7 +739,7 @@ func GetConfigData(toClient *toreq.TOClient, disableProxy bool, cacheHostName st return nil } - fs := []func() error{serversF, cgF, jobsF} + fs := []func() error{serversF, cgF} if !revalOnly { // skip data not needed for reval, if we're reval-only fs = append([]func() error{dsrF, cacheKeyConfigParamsF, remapConfigParamsF, parentConfigParamsF, capsF, dsCapsF, topologiesF}, fs...) diff --git a/cache-config/t3cutil/toreq/clientfuncs.go b/cache-config/t3cutil/toreq/clientfuncs.go index decc7b79b3..a28476ab0b 100644 --- a/cache-config/t3cutil/toreq/clientfuncs.go +++ b/cache-config/t3cutil/toreq/clientfuncs.go @@ -177,7 +177,7 @@ func (cl *TOClient) GetCacheGroups(reqHdr http.Header) ([]tc.CacheGroupNullable, // If your use case is more efficient to only get the needed objects, for example if you're frequently requesting one file, set this false to get and cache the specific needed delivery services and servers. const DeliveryServiceServersAlwaysGetAll = true -func (cl *TOClient) GetDeliveryServiceServers(dsIDs []int, serverIDs []int, reqHdr http.Header) ([]tc.DeliveryServiceServer, toclientlib.ReqInf, error) { +func (cl *TOClient) GetDeliveryServiceServers(dsIDs []int, serverIDs []int, cdnName string, reqHdr http.Header) ([]tc.DeliveryServiceServer, toclientlib.ReqInf, error) { if cl.c == nil { return cl.old.GetDeliveryServiceServers(dsIDs, serverIDs) } @@ -200,7 +200,7 @@ func (cl *TOClient) GetDeliveryServiceServers(dsIDs []int, serverIDs []int, reqH } dsServers := []tc.DeliveryServiceServer{} - err := torequtil.GetRetry(cl.NumRetries, "deliveryservice_servers_s"+serverIDsStr+"_d_"+dsIDsStr, &dsServers, func(obj interface{}) error { + err := torequtil.GetRetry(cl.NumRetries, "deliveryservice_servers_s"+serverIDsStr+"_d_"+dsIDsStr+"_cdn_"+cdnName, &dsServers, func(obj interface{}) error { dsIDStrs := []string{} for _, dsID := range dsIDsToFetch { @@ -214,6 +214,8 @@ func (cl *TOClient) GetDeliveryServiceServers(dsIDs []int, serverIDs []int, reqH queryParams := url.Values{} queryParams.Set("limit", "999999") // TODO add "no limit" param to DSS endpoint + queryParams.Set("cdn", cdnName) + queryParams.Set("orderby", "") // prevent unnecessary sorting of the response if len(dsIDsToFetch) > 0 { queryParams.Set("deliveryserviceids", strings.Join(dsIDStrs, ",")) } @@ -513,7 +515,7 @@ func (cl *TOClient) GetDeliveryServiceRegexes(reqHdr http.Header) ([]tc.Delivery return regexes, reqInf, nil } -func (cl *TOClient) GetJobs(reqHdr http.Header) ([]tc.InvalidationJob, toclientlib.ReqInf, error) { +func (cl *TOClient) GetJobs(reqHdr http.Header, cdnName string) ([]tc.InvalidationJob, toclientlib.ReqInf, error) { if cl.c == nil { oldJobs, inf, err := cl.old.GetJobs() jobs, err := atscfg.JobsToInvalidationJobs(oldJobs) @@ -525,8 +527,11 @@ func (cl *TOClient) GetJobs(reqHdr http.Header) ([]tc.InvalidationJob, toclientl jobs := []tc.InvalidationJob{} reqInf := toclientlib.ReqInf{} - err := torequtil.GetRetry(cl.NumRetries, "jobs", &jobs, func(obj interface{}) error { - toJobs, toReqInf, err := cl.c.GetInvalidationJobs(*ReqOpts(reqHdr)) + err := torequtil.GetRetry(cl.NumRetries, "jobs_cdn_"+cdnName, &jobs, func(obj interface{}) error { + opts := *ReqOpts(reqHdr) + opts.QueryParameters.Set("maxRevalDurationDays", "") // only get jobs with a start time within the window defined by the GLOBAL parameter 'maxRevalDurationDays' + opts.QueryParameters.Set("cdn", cdnName) // only get jobs for delivery services in this server's CDN + toJobs, toReqInf, err := cl.c.GetInvalidationJobs(opts) if err != nil { return errors.New("getting jobs from Traffic Ops '" + torequtil.MaybeIPStr(reqInf.RemoteAddr) + "': " + err.Error()) } diff --git a/cache-config/t3cutil/toreq/clienthlp.go b/cache-config/t3cutil/toreq/clienthlp.go index e356d1e97a..035932fd18 100644 --- a/cache-config/t3cutil/toreq/clienthlp.go +++ b/cache-config/t3cutil/toreq/clienthlp.go @@ -119,5 +119,7 @@ func GetDeliveryServiceURLSigKeys(toClient *toclient.Session, dsName string, opt // ReqOpts takes an http.Header and returns a traffic_ops/v4-client.RequestOptions with that header. // This is a helper function, for brevity. func ReqOpts(hdr http.Header) *toclient.RequestOptions { - return &toclient.RequestOptions{Header: hdr} + opts := toclient.NewRequestOptions() + opts.Header = hdr + return &opts } diff --git a/infrastructure/cdn-in-a-box/cache/traffic_ops_ort.crontab b/infrastructure/cdn-in-a-box/cache/traffic_ops_ort.crontab index 9caebac639..14964b682d 100644 --- a/infrastructure/cdn-in-a-box/cache/traffic_ops_ort.crontab +++ b/infrastructure/cdn-in-a-box/cache/traffic_ops_ort.crontab @@ -15,4 +15,4 @@ # specific language governing permissions and limitations # under the License. -*/1 * * * * t3c apply --run-mode=syncds --traffic-ops-url=$TO_URL --traffic-ops-user=$TO_USER --traffic-ops-password=$TO_PASSWORD --git=yes -vv 2>&1 >> /var/log/ort.log +*/1 * * * * root t3c apply --run-mode=syncds --traffic-ops-url=$TO_URL --traffic-ops-user=$TO_USER --traffic-ops-password=$TO_PASSWORD --git=yes -vv --cache-host-name=$(hostname -s) >> /var/log/ort.log 2>&1