Make t3c request less unnecessary DSS and jobs data

CHANGELOG.md

@@ -7,6 +7,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 ### Added
 - [#5674](https://github.com/apache/trafficcontrol/issues/5674) Added new query parameters `cdn` and `maxRevalDurationDays` to the `GET /api/x/jobs` Traffic Ops API to filter by CDN name and within the start_time window defined by the `maxRevalDurationDays` GLOBAL profile parameter, respectively.
 
+### Changed
+- Updated `t3c` to request less unnecessary deliveryservice-server assignment and invalidation jobs data via new query params supported by Traffic Ops
+
 ## unreleased
 ### Added
 - [#4982](https://github.com/apache/trafficcontrol/issues/4982) Added the ability to support queueing updates by server type and profile

cache-config/t3cutil/getdatacfg.go

@@ -332,7 +332,7 @@ func GetConfigData(toClient *toreq.TOClient, disableProxy bool, cacheHostName st
 					if oldCfg != nil {
 						reqHdr = MakeReqHdr(oldCfg.MetaData.DeliveryServiceServers)
 					}
-					dss, reqInf, err := toClient.GetDeliveryServiceServers(nil, nil, reqHdr)
+					dss, reqInf, err := toClient.GetDeliveryServiceServers(nil, nil, *server.CDNName, reqHdr)
 					if err != nil {
 						return errors.New("getting delivery service servers: " + err.Error())
 					}
@@ -515,7 +515,30 @@ func GetConfigData(toClient *toreq.TOClient, disableProxy bool, cacheHostName st
 			}
 			return nil
 		}
-		fs := []func() error{dsF, serverParamsF, cdnF, profileF}
+		jobsF := func() error {
+			defer func(start time.Time) { log.Infof("jobsF took %v\n", time.Since(start)) }(time.Now())
+			{
+				reqHdr := (http.Header)(nil)
+				if oldCfg != nil {
+					reqHdr = MakeReqHdr(oldCfg.MetaData.Jobs)
+				}
+				jobs, reqInf, err := toClient.GetJobs(reqHdr, *server.CDNName)
+				if err != nil {
+					return errors.New("getting jobs: " + err.Error())
+				}
+				if reqInf.StatusCode == http.StatusNotModified {
+					log.Infof("Getting config: %v not modified, using old config", "Jobs")
+					toData.Jobs = oldCfg.Jobs
+				} else {
+					log.Infof("Getting config: %v is modified, using new response", "Jobs")
+					toData.Jobs = jobs
+				}
+				toData.MetaData.Jobs = MakeReqMetaData(reqInf.RespHeaders)
+				toIPs.Store(reqInf.RemoteAddr, nil)
+			}
+			return nil
+		}
+		fs := []func() error{dsF, serverParamsF, cdnF, profileF, jobsF}
 		if !revalOnly {
 			fs = append([]func() error{sslF}, fs...) // skip ssl keys for reval only, which doesn't need them
 		}
@@ -545,29 +568,6 @@ func GetConfigData(toClient *toreq.TOClient, disableProxy bool, cacheHostName st
 		}
 		return nil
 	}
-	jobsF := func() error {
-		defer func(start time.Time) { log.Infof("jobsF took %v\n", time.Since(start)) }(time.Now())
-		{
-			reqHdr := (http.Header)(nil)
-			if oldCfg != nil {
-				reqHdr = MakeReqHdr(oldCfg.MetaData.Jobs)
-			}
-			jobs, reqInf, err := toClient.GetJobs(reqHdr) // TODO add cdn query param to jobs endpoint
-			if err != nil {
-				return errors.New("getting jobs: " + err.Error())
-			}
-			if reqInf.StatusCode == http.StatusNotModified {
-				log.Infof("Getting config: %v not modified, using old config", "Jobs")
-				toData.Jobs = oldCfg.Jobs
-			} else {
-				log.Infof("Getting config: %v is modified, using new response", "Jobs")
-				toData.Jobs = jobs
-			}
-			toData.MetaData.Jobs = MakeReqMetaData(reqInf.RespHeaders)
-			toIPs.Store(reqInf.RemoteAddr, nil)
-		}
-		return nil
-	}
 	capsF := func() error {
 		defer func(start time.Time) { log.Infof("capsF took %v\n", time.Since(start)) }(time.Now())
 		{
@@ -739,7 +739,7 @@ func GetConfigData(toClient *toreq.TOClient, disableProxy bool, cacheHostName st
 		return nil
 	}
 
-	fs := []func() error{serversF, cgF, jobsF}
+	fs := []func() error{serversF, cgF}
 	if !revalOnly {
 		// skip data not needed for reval, if we're reval-only
 		fs = append([]func() error{dsrF, cacheKeyConfigParamsF, remapConfigParamsF, parentConfigParamsF, capsF, dsCapsF, topologiesF}, fs...)

cache-config/t3cutil/toreq/clientfuncs.go

@@ -177,7 +177,7 @@ func (cl *TOClient) GetCacheGroups(reqHdr http.Header) ([]tc.CacheGroupNullable,
 // If your use case is more efficient to only get the needed objects, for example if you're frequently requesting one file, set this false to get and cache the specific needed delivery services and servers.
 const DeliveryServiceServersAlwaysGetAll = true
 
-func (cl *TOClient) GetDeliveryServiceServers(dsIDs []int, serverIDs []int, reqHdr http.Header) ([]tc.DeliveryServiceServer, toclientlib.ReqInf, error) {
+func (cl *TOClient) GetDeliveryServiceServers(dsIDs []int, serverIDs []int, cdnName string, reqHdr http.Header) ([]tc.DeliveryServiceServer, toclientlib.ReqInf, error) {
 	if cl.c == nil {
 		return cl.old.GetDeliveryServiceServers(dsIDs, serverIDs)
 	}
@@ -200,7 +200,7 @@ func (cl *TOClient) GetDeliveryServiceServers(dsIDs []int, serverIDs []int, reqH
 	}
 
 	dsServers := []tc.DeliveryServiceServer{}
-	err := torequtil.GetRetry(cl.NumRetries, "deliveryservice_servers_s"+serverIDsStr+"_d_"+dsIDsStr, &dsServers, func(obj interface{}) error {
+	err := torequtil.GetRetry(cl.NumRetries, "deliveryservice_servers_s"+serverIDsStr+"_d_"+dsIDsStr+"_cdn_"+cdnName, &dsServers, func(obj interface{}) error {
 
 		dsIDStrs := []string{}
 		for _, dsID := range dsIDsToFetch {
@@ -214,6 +214,8 @@ func (cl *TOClient) GetDeliveryServiceServers(dsIDs []int, serverIDs []int, reqH
 
 		queryParams := url.Values{}
 		queryParams.Set("limit", "999999") // TODO add "no limit" param to DSS endpoint
+		queryParams.Set("cdn", cdnName)
+		queryParams.Set("orderby", "") // prevent unnecessary sorting of the response
 		if len(dsIDsToFetch) > 0 {
 			queryParams.Set("deliveryserviceids", strings.Join(dsIDStrs, ","))
 		}
@@ -513,7 +515,7 @@ func (cl *TOClient) GetDeliveryServiceRegexes(reqHdr http.Header) ([]tc.Delivery
 	return regexes, reqInf, nil
 }
 
-func (cl *TOClient) GetJobs(reqHdr http.Header) ([]tc.InvalidationJob, toclientlib.ReqInf, error) {
+func (cl *TOClient) GetJobs(reqHdr http.Header, cdnName string) ([]tc.InvalidationJob, toclientlib.ReqInf, error) {
 	if cl.c == nil {
 		oldJobs, inf, err := cl.old.GetJobs()
 		jobs, err := atscfg.JobsToInvalidationJobs(oldJobs)
@@ -525,8 +527,11 @@ func (cl *TOClient) GetJobs(reqHdr http.Header) ([]tc.InvalidationJob, toclientl
 
 	jobs := []tc.InvalidationJob{}
 	reqInf := toclientlib.ReqInf{}
-	err := torequtil.GetRetry(cl.NumRetries, "jobs", &jobs, func(obj interface{}) error {
-		toJobs, toReqInf, err := cl.c.GetInvalidationJobs(*ReqOpts(reqHdr))
+	err := torequtil.GetRetry(cl.NumRetries, "jobs_cdn_"+cdnName, &jobs, func(obj interface{}) error {
+		opts := *ReqOpts(reqHdr)
+		opts.QueryParameters.Set("maxRevalDurationDays", "") // only get jobs with a start time within the window defined by the GLOBAL parameter 'maxRevalDurationDays'
+		opts.QueryParameters.Set("cdn", cdnName)             // only get jobs for delivery services in this server's CDN
+		toJobs, toReqInf, err := cl.c.GetInvalidationJobs(opts)
 		if err != nil {
 			return errors.New("getting jobs from Traffic Ops '" + torequtil.MaybeIPStr(reqInf.RemoteAddr) + "': " + err.Error())
 		}

cache-config/t3cutil/toreq/clienthlp.go

@@ -119,5 +119,7 @@ func GetDeliveryServiceURLSigKeys(toClient *toclient.Session, dsName string, opt
 // ReqOpts takes an http.Header and returns a traffic_ops/v4-client.RequestOptions with that header.
 // This is a helper function, for brevity.
 func ReqOpts(hdr http.Header) *toclient.RequestOptions {
-	return &toclient.RequestOptions{Header: hdr}
+	opts := toclient.NewRequestOptions()
+	opts.Header = hdr
+	return &opts
 }

infrastructure/cdn-in-a-box/cache/traffic_ops_ort.crontab

@@ -15,4 +15,4 @@
 # specific language governing permissions and limitations
 # under the License.
 
-*/1 * * * * t3c apply --run-mode=syncds --traffic-ops-url=$TO_URL --traffic-ops-user=$TO_USER --traffic-ops-password=$TO_PASSWORD --git=yes -vv 2>&1 >> /var/log/ort.log
+*/1 * * * * root t3c apply --run-mode=syncds --traffic-ops-url=$TO_URL --traffic-ops-user=$TO_USER --traffic-ops-password=$TO_PASSWORD --git=yes -vv --cache-host-name=$(hostname -s) >> /var/log/ort.log 2>&1