Feature/refetch

CHANGELOG.md

@@ -170,6 +170,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - CDN in a Box now uses Apache Traffic Server 8.1.
 - Customer names in payloads sent to the `/deliveryservices/request` Traffic Ops API endpoint can no longer contain characters besides alphanumerics, @, !, #, $, %, ^, &, *, (, ), [, ], '.', ' ', and '-'. This fixes a vulnerability that allowed email content injection.
 - Go version 1.17 is used to compile Traffic Ops, T3C, Traffic Monitor, Traffic Stats, and Grove.
+- Changed Invalidation Jobs throughout (TO, TP, T3C, etc.) to account for the ability to do both REFRESH and REFETCH requests for resources.
 
 ### Deprecated
 - The Riak Traffic Vault backend is now deprecated and its support may be removed in a future release. It is highly recommended to use the new PostgreSQL backend instead.

cache-config/t3c-generate/cfgfile/cfgfile_test.go

@@ -162,6 +162,11 @@ func randUint64() *uint64 {
 	return &i
 }
 
+func randUint() *uint {
+	i := uint(rand.Int())
+	return &i
+}
+
 func randFloat64() *float64 {
 	f := rand.Float64()
 	return &f
@@ -351,16 +356,16 @@ func randParam() *tc.Parameter {
 	}
 }
 
-func randJob() *tc.InvalidationJob {
-	now := &tc.Time{Time: time.Now(), Valid: true}
-	return &tc.InvalidationJob{
-		Parameters:      randStr(),
-		Keyword:         randStr(),
-		AssetURL:        randStr(),
-		CreatedBy:       randStr(),
-		StartTime:       now,
-		ID:              randUint64(),
-		DeliveryService: randStr(),
+func randJob() *atscfg.InvalidationJob {
+	now := time.Now()
+	return &atscfg.InvalidationJob{
+		AssetURL:         *randStr(),
+		CreatedBy:        *randStr(),
+		StartTime:        now,
+		ID:               *randUint64(),
+		DeliveryService:  *randStr(),
+		TTLHours:         *randUint(),
+		InvalidationType: tc.REFRESH,
 	}
 }
 
@@ -486,7 +491,7 @@ func MakeFakeTOData() *t3cutil.ConfigData {
 		},
 		DeliveryServiceServers: dss,
 		Server:                 sv0,
-		Jobs: []tc.InvalidationJob{
+		Jobs: []atscfg.InvalidationJob{
 			*randJob(),
 			*randJob(),
 		},

cache-config/t3cutil/getdatacfg.go

@@ -69,7 +69,7 @@ type ConfigData struct {
 	Server *atscfg.Server `json:"server,omitempty"`
 
 	// Jobs must be all Jobs on the server's CDN. May include jobs on other CDNs.
-	Jobs []tc.InvalidationJob `json:"jobs,omitempty"`
+	Jobs []atscfg.InvalidationJob `json:"jobs,omitempty"`
 
 	// CDN must be the CDN of the server.
 	CDN *tc.CDN `json:"cdn,omitempty"`

cache-config/t3cutil/toreq/clientfuncs.go

@@ -515,7 +515,7 @@ func (cl *TOClient) GetDeliveryServiceRegexes(reqHdr http.Header) ([]tc.Delivery
 	return regexes, reqInf, nil
 }
 
-func (cl *TOClient) GetJobs(reqHdr http.Header, cdnName string) ([]tc.InvalidationJob, toclientlib.ReqInf, error) {
+func (cl *TOClient) GetJobs(reqHdr http.Header, cdnName string) ([]atscfg.InvalidationJob, toclientlib.ReqInf, error) {
 	if cl.c == nil {
 		oldJobs, inf, err := cl.old.GetJobs()
 		jobs, err := atscfg.JobsToInvalidationJobs(oldJobs)
@@ -525,7 +525,7 @@ func (cl *TOClient) GetJobs(reqHdr http.Header, cdnName string) ([]tc.Invalidati
 		return jobs, inf, err
 	}
 
-	jobs := []tc.InvalidationJob{}
+	jobs := []atscfg.InvalidationJob{}
 	reqInf := toclientlib.ReqInf{}
 	err := torequtil.GetRetry(cl.NumRetries, "jobs_cdn_"+cdnName, &jobs, func(obj interface{}) error {
 		opts := *ReqOpts(reqHdr)
@@ -535,8 +535,8 @@ func (cl *TOClient) GetJobs(reqHdr http.Header, cdnName string) ([]tc.Invalidati
 		if err != nil {
 			return errors.New("getting jobs from Traffic Ops '" + torequtil.MaybeIPStr(reqInf.RemoteAddr) + "': " + err.Error())
 		}
-		jobs := obj.(*[]tc.InvalidationJob)
-		*jobs = toJobs.Response
+		jobs := obj.(*[]atscfg.InvalidationJob)
+		*jobs = jobsToLatest(toJobs.Response)
 		reqInf = toReqInf
 		return nil
 	})

cache-config/t3cutil/toreq/conversions.go

@@ -36,3 +36,7 @@ func serverToLatest(oldSv *tc.ServerV40) (*atscfg.Server, error) {
 func dsesToLatest(dses []tc.DeliveryServiceV40) []atscfg.DeliveryService {
 	return atscfg.V40ToDeliveryServices(dses)
 }
+
+func jobsToLatest(jobs []tc.InvalidationJobV4) []atscfg.InvalidationJob {
+	return atscfg.ToInvalidationJobs(jobs)
+}

cache-config/testing/ort-tests/tcdata/todb.go

@@ -87,18 +87,6 @@ func (r *TCData) SetupTestData(*sql.DB) error {
 		os.Exit(1)
 	}
 
-	err = SetupJobAgents(db)
-	if err != nil {
-		fmt.Printf("\nError setting up job agents %s - %s, %v\n", r.Config.TrafficOps.URL, r.Config.TrafficOps.Users.Admin, err)
-		os.Exit(1)
-	}
-
-	err = SetupJobStatuses(db)
-	if err != nil {
-		fmt.Printf("\nError setting up job agents %s - %s, %v\n", r.Config.TrafficOps.URL, r.Config.TrafficOps.Users.Admin, err)
-		os.Exit(1)
-	}
-
 	err = SetupTypes(db)
 	if err != nil {
 		fmt.Printf("\nError setting up types %s - %s, %v\n", r.Config.TrafficOps.URL, r.Config.TrafficOps.Users.Admin, err)
@@ -230,39 +218,13 @@ INSERT INTO deliveryservice_tmuser (deliveryservice, tm_user_id, last_updated) V
 	return nil
 }
 
-// SetupJobStatuses ...
-func SetupJobStatuses(db *sql.DB) error {
-
-	sqlStmt := `
-INSERT INTO job_status (id, name, description, last_updated) VALUES (1, 'PENDING', 'Job is queued, but has not been picked up by any agents yet', '2018-01-19 21:19:32.444857');
-`
-	err := execSQL(db, sqlStmt)
-	if err != nil {
-		return fmt.Errorf("exec failed %v", err)
-	}
-	return nil
-}
-
-// SetupJobAgents ...
-func SetupJobAgents(db *sql.DB) error {
-
-	sqlStmt := `
-INSERT INTO job_agent (id, name, description, active, last_updated) VALUES (1, 'agent1', 'Test Agent1', 0, '2018-01-19 21:19:32.448076');
-`
-	err := execSQL(db, sqlStmt)
-	if err != nil {
-		return fmt.Errorf("exec failed %v", err)
-	}
-	return nil
-}
-
 // SetupJobs ...
 func SetupJobs(db *sql.DB) error {
 
 	sqlStmt := `
-INSERT INTO job (id, agent, object_type, object_name, keyword, parameters, asset_url, asset_type, status, start_time, entered_time, job_user, last_updated, job_deliveryservice) VALUES (100, 1, null, null, 'PURGE', 'TTL:48h', 'http://cdn2.edge/job1/.*', 'file', 1, '2018-01-19 21:01:14.000000', '2018-01-19 21:01:14.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.468643', 100);
-INSERT INTO job (id, agent, object_type, object_name, keyword, parameters, asset_url, asset_type, status, start_time, entered_time, job_user, last_updated, job_deliveryservice) VALUES (200, 1, null, null, 'PURGE', 'TTL:48h', 'http://cdn2.edge/job2/.*', 'file', 1, '2018-01-19 21:09:34.000000', '2018-01-19 21:09:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.450915', 200);
-INSERT INTO job (id, agent, object_type, object_name, keyword, parameters, asset_url, asset_type, status, start_time, entered_time, job_user, last_updated, job_deliveryservice) VALUES (300, 1, null, null, 'PURGE', 'TTL:48h', 'http://cdn2.edge/job3/.*', 'file', 1, '2018-01-19 21:14:34.000000', '2018-01-19 21:14:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.460870', 100);
+INSERT INTO job (id, ttl_hr, asset_url, start_time, entered_time, job_user, last_updated, job_deliveryservice, invalidation_type) VALUES (100, 24, 'http://cdn2.edge/job1/.*', '2018-01-19 21:01:14.000000', '2018-01-19 21:01:14.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.468643', 100, 'REFRESH');
+INSERT INTO job (id, ttl_hr, asset_url, start_time, entered_time, job_user, last_updated, job_deliveryservice, invalidation_type) VALUES (200, 36, 'http://cdn2.edge/job2/.*', '2018-01-19 21:09:34.000000', '2018-01-19 21:09:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.450915', 200, 'REFETCH');
+INSERT INTO job (id, ttl_hr, asset_url, start_time, entered_time, job_user, last_updated, job_deliveryservice, invalidation_type) VALUES (300, 72, 'http://cdn2.edge/job3/.*', '2018-01-19 21:14:34.000000', '2018-01-19 21:14:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.460870', 100, 'REFRESH');
 `
 	err := execSQL(db, sqlStmt)
 	if err != nil {
@@ -311,8 +273,6 @@ func (r *TCData) Teardown(db *sql.DB) error {
 	DELETE FROM to_extension;
 	DELETE FROM staticdnsentry;
 	DELETE FROM job;
-	DELETE FROM job_agent;
-	DELETE FROM job_status;
 	DELETE FROM log;
 	DELETE FROM asn;
 	DELETE FROM deliveryservice_tmuser;

lib/go-atscfg/atscfg.go

@@ -29,7 +29,6 @@ import (
 	"strings"
 
 	"github.com/apache/trafficcontrol/lib/go-tc"
-	"github.com/apache/trafficcontrol/lib/go-util"
 )
 
 const InvalidID = -1
@@ -59,6 +58,11 @@ type Server tc.ServerV40
 // but to only have to change it here, and the places where breaking symbol changes were made.
 type DeliveryService tc.DeliveryServiceV40
 
+// InvalidationJob is a tc.InvalidationJob for the latest lib/go-tc and traffic_ops/vx-client type.
+// This allows atscfg to not have to change the type everywhere it's used, every time ATC changes the base type,
+// but to only have to change it here, and the places where breaking symbol changes were made.
+type InvalidationJob tc.InvalidationJobV4
+
 // ToDeliveryServices converts a slice of the latest lib/go-tc and traffic_ops/vx-client type to the local alias.
 func ToDeliveryServices(dses []tc.DeliveryServiceV40) []DeliveryService {
 	ad := []DeliveryService{}
@@ -77,6 +81,15 @@ func V40ToDeliveryServices(dses []tc.DeliveryServiceV40) []DeliveryService {
 	return ad
 }
 
+// ToInvalidationJobs converts a slice of the latest lib/go-tc and traffic_ops/vx-client type to the local alias.
+func ToInvalidationJobs(jobs []tc.InvalidationJobV4) []InvalidationJob {
+	aj := []InvalidationJob{}
+	for _, job := range jobs {
+		aj = append(aj, InvalidationJob(job))
+	}
+	return aj
+}
+
 // ToServers converts a slice of the latest lib/go-tc and traffic_ops/vx-client type to the local alias.
 func ToServers(servers []tc.ServerV40) []Server {
 	as := []Server{}
@@ -669,8 +682,8 @@ type DeliveryServiceServer struct {
 	DeliveryService int `json:"d"`
 }
 
-func JobsToInvalidationJobs(oldJobs []tc.Job) ([]tc.InvalidationJob, error) {
-	jobs := make([]tc.InvalidationJob, len(oldJobs), len(oldJobs))
+func JobsToInvalidationJobs(oldJobs []tc.Job) ([]InvalidationJob, error) {
+	jobs := make([]InvalidationJob, len(oldJobs), len(oldJobs))
 	err := error(nil)
 	for i, oldJob := range oldJobs {
 		jobs[i], err = JobToInvalidationJob(oldJob)
@@ -681,18 +694,22 @@ func JobsToInvalidationJobs(oldJobs []tc.Job) ([]tc.InvalidationJob, error) {
 	return jobs, nil
 }
 
-func JobToInvalidationJob(jb tc.Job) (tc.InvalidationJob, error) {
+func JobToInvalidationJob(jb tc.Job) (InvalidationJob, error) {
 	startTime := tc.Time{}
 	if err := json.Unmarshal([]byte(`"`+jb.StartTime+`"`), &startTime); err != nil {
-		return tc.InvalidationJob{}, errors.New("unmarshalling time: " + err.Error())
-	}
-	return tc.InvalidationJob{
-		AssetURL:        util.StrPtr(jb.AssetURL),
-		CreatedBy:       util.StrPtr(jb.CreatedBy),
-		DeliveryService: util.StrPtr(jb.DeliveryService),
-		ID:              util.Uint64Ptr(uint64(jb.ID)),
-		Keyword:         util.StrPtr(jb.Keyword),
-		Parameters:      util.StrPtr(jb.Parameters),
-		StartTime:       &startTime,
+		return InvalidationJob{}, errors.New("unmarshalling time: " + err.Error())
+	}
+	ttl, err := strconv.Atoi(strings.TrimSuffix(strings.TrimPrefix(jb.Parameters, "TTL:"), "h"))
+	if err != nil {
+		return InvalidationJob{}, errors.New("unmarshalling ttl: " + err.Error())
+	}
+	return InvalidationJob{
+		AssetURL:         jb.AssetURL,
+		CreatedBy:        jb.CreatedBy,
+		DeliveryService:  jb.DeliveryService,
+		ID:               uint64(jb.ID),
+		TTLHours:         uint(ttl),
+		InvalidationType: tc.REFRESH,
+		StartTime:        startTime.Time,
 	}, nil
 }