diff --git a/CHANGELOG.md b/CHANGELOG.md index ae687dc8b8..0c5dbace08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -170,6 +170,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - CDN in a Box now uses Apache Traffic Server 8.1. - Customer names in payloads sent to the `/deliveryservices/request` Traffic Ops API endpoint can no longer contain characters besides alphanumerics, @, !, #, $, %, ^, &, *, (, ), [, ], '.', ' ', and '-'. This fixes a vulnerability that allowed email content injection. - Go version 1.17 is used to compile Traffic Ops, T3C, Traffic Monitor, Traffic Stats, and Grove. +- Changed Invalidation Jobs throughout (TO, TP, T3C, etc.) to account for the ability to do both REFRESH and REFETCH requests for resources. ### Deprecated - The Riak Traffic Vault backend is now deprecated and its support may be removed in a future release. It is highly recommended to use the new PostgreSQL backend instead. diff --git a/cache-config/t3c-generate/cfgfile/cfgfile_test.go b/cache-config/t3c-generate/cfgfile/cfgfile_test.go index 991501ac70..de0e79ed8e 100644 --- a/cache-config/t3c-generate/cfgfile/cfgfile_test.go +++ b/cache-config/t3c-generate/cfgfile/cfgfile_test.go @@ -162,6 +162,11 @@ func randUint64() *uint64 { return &i } +func randUint() *uint { + i := uint(rand.Int()) + return &i +} + func randFloat64() *float64 { f := rand.Float64() return &f @@ -351,16 +356,16 @@ func randParam() *tc.Parameter { } } -func randJob() *tc.InvalidationJob { - now := &tc.Time{Time: time.Now(), Valid: true} - return &tc.InvalidationJob{ - Parameters: randStr(), - Keyword: randStr(), - AssetURL: randStr(), - CreatedBy: randStr(), - StartTime: now, - ID: randUint64(), - DeliveryService: randStr(), +func randJob() *atscfg.InvalidationJob { + now := time.Now() + return &atscfg.InvalidationJob{ + AssetURL: *randStr(), + CreatedBy: *randStr(), + StartTime: now, + ID: *randUint64(), + DeliveryService: *randStr(), + TTLHours: *randUint(), + InvalidationType: tc.REFRESH, } } @@ -486,7 +491,7 @@ func MakeFakeTOData() *t3cutil.ConfigData { }, DeliveryServiceServers: dss, Server: sv0, - Jobs: []tc.InvalidationJob{ + Jobs: []atscfg.InvalidationJob{ *randJob(), *randJob(), }, diff --git a/cache-config/t3cutil/getdatacfg.go b/cache-config/t3cutil/getdatacfg.go index 84e6fdfc79..5e3a280894 100644 --- a/cache-config/t3cutil/getdatacfg.go +++ b/cache-config/t3cutil/getdatacfg.go @@ -69,7 +69,7 @@ type ConfigData struct { Server *atscfg.Server `json:"server,omitempty"` // Jobs must be all Jobs on the server's CDN. May include jobs on other CDNs. - Jobs []tc.InvalidationJob `json:"jobs,omitempty"` + Jobs []atscfg.InvalidationJob `json:"jobs,omitempty"` // CDN must be the CDN of the server. CDN *tc.CDN `json:"cdn,omitempty"` diff --git a/cache-config/t3cutil/toreq/clientfuncs.go b/cache-config/t3cutil/toreq/clientfuncs.go index a28476ab0b..7785e48b9a 100644 --- a/cache-config/t3cutil/toreq/clientfuncs.go +++ b/cache-config/t3cutil/toreq/clientfuncs.go @@ -515,7 +515,7 @@ func (cl *TOClient) GetDeliveryServiceRegexes(reqHdr http.Header) ([]tc.Delivery return regexes, reqInf, nil } -func (cl *TOClient) GetJobs(reqHdr http.Header, cdnName string) ([]tc.InvalidationJob, toclientlib.ReqInf, error) { +func (cl *TOClient) GetJobs(reqHdr http.Header, cdnName string) ([]atscfg.InvalidationJob, toclientlib.ReqInf, error) { if cl.c == nil { oldJobs, inf, err := cl.old.GetJobs() jobs, err := atscfg.JobsToInvalidationJobs(oldJobs) @@ -525,7 +525,7 @@ func (cl *TOClient) GetJobs(reqHdr http.Header, cdnName string) ([]tc.Invalidati return jobs, inf, err } - jobs := []tc.InvalidationJob{} + jobs := []atscfg.InvalidationJob{} reqInf := toclientlib.ReqInf{} err := torequtil.GetRetry(cl.NumRetries, "jobs_cdn_"+cdnName, &jobs, func(obj interface{}) error { opts := *ReqOpts(reqHdr) @@ -535,8 +535,8 @@ func (cl *TOClient) GetJobs(reqHdr http.Header, cdnName string) ([]tc.Invalidati if err != nil { return errors.New("getting jobs from Traffic Ops '" + torequtil.MaybeIPStr(reqInf.RemoteAddr) + "': " + err.Error()) } - jobs := obj.(*[]tc.InvalidationJob) - *jobs = toJobs.Response + jobs := obj.(*[]atscfg.InvalidationJob) + *jobs = jobsToLatest(toJobs.Response) reqInf = toReqInf return nil }) diff --git a/cache-config/t3cutil/toreq/conversions.go b/cache-config/t3cutil/toreq/conversions.go index b73d74a614..3cd368714b 100644 --- a/cache-config/t3cutil/toreq/conversions.go +++ b/cache-config/t3cutil/toreq/conversions.go @@ -36,3 +36,7 @@ func serverToLatest(oldSv *tc.ServerV40) (*atscfg.Server, error) { func dsesToLatest(dses []tc.DeliveryServiceV40) []atscfg.DeliveryService { return atscfg.V40ToDeliveryServices(dses) } + +func jobsToLatest(jobs []tc.InvalidationJobV4) []atscfg.InvalidationJob { + return atscfg.ToInvalidationJobs(jobs) +} diff --git a/cache-config/testing/ort-tests/tcdata/todb.go b/cache-config/testing/ort-tests/tcdata/todb.go index 91e91faf41..2de9861739 100644 --- a/cache-config/testing/ort-tests/tcdata/todb.go +++ b/cache-config/testing/ort-tests/tcdata/todb.go @@ -87,18 +87,6 @@ func (r *TCData) SetupTestData(*sql.DB) error { os.Exit(1) } - err = SetupJobAgents(db) - if err != nil { - fmt.Printf("\nError setting up job agents %s - %s, %v\n", r.Config.TrafficOps.URL, r.Config.TrafficOps.Users.Admin, err) - os.Exit(1) - } - - err = SetupJobStatuses(db) - if err != nil { - fmt.Printf("\nError setting up job agents %s - %s, %v\n", r.Config.TrafficOps.URL, r.Config.TrafficOps.Users.Admin, err) - os.Exit(1) - } - err = SetupTypes(db) if err != nil { fmt.Printf("\nError setting up types %s - %s, %v\n", r.Config.TrafficOps.URL, r.Config.TrafficOps.Users.Admin, err) @@ -230,39 +218,13 @@ INSERT INTO deliveryservice_tmuser (deliveryservice, tm_user_id, last_updated) V return nil } -// SetupJobStatuses ... -func SetupJobStatuses(db *sql.DB) error { - - sqlStmt := ` -INSERT INTO job_status (id, name, description, last_updated) VALUES (1, 'PENDING', 'Job is queued, but has not been picked up by any agents yet', '2018-01-19 21:19:32.444857'); -` - err := execSQL(db, sqlStmt) - if err != nil { - return fmt.Errorf("exec failed %v", err) - } - return nil -} - -// SetupJobAgents ... -func SetupJobAgents(db *sql.DB) error { - - sqlStmt := ` -INSERT INTO job_agent (id, name, description, active, last_updated) VALUES (1, 'agent1', 'Test Agent1', 0, '2018-01-19 21:19:32.448076'); -` - err := execSQL(db, sqlStmt) - if err != nil { - return fmt.Errorf("exec failed %v", err) - } - return nil -} - // SetupJobs ... func SetupJobs(db *sql.DB) error { sqlStmt := ` -INSERT INTO job (id, agent, object_type, object_name, keyword, parameters, asset_url, asset_type, status, start_time, entered_time, job_user, last_updated, job_deliveryservice) VALUES (100, 1, null, null, 'PURGE', 'TTL:48h', 'http://cdn2.edge/job1/.*', 'file', 1, '2018-01-19 21:01:14.000000', '2018-01-19 21:01:14.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.468643', 100); -INSERT INTO job (id, agent, object_type, object_name, keyword, parameters, asset_url, asset_type, status, start_time, entered_time, job_user, last_updated, job_deliveryservice) VALUES (200, 1, null, null, 'PURGE', 'TTL:48h', 'http://cdn2.edge/job2/.*', 'file', 1, '2018-01-19 21:09:34.000000', '2018-01-19 21:09:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.450915', 200); -INSERT INTO job (id, agent, object_type, object_name, keyword, parameters, asset_url, asset_type, status, start_time, entered_time, job_user, last_updated, job_deliveryservice) VALUES (300, 1, null, null, 'PURGE', 'TTL:48h', 'http://cdn2.edge/job3/.*', 'file', 1, '2018-01-19 21:14:34.000000', '2018-01-19 21:14:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.460870', 100); +INSERT INTO job (id, ttl_hr, asset_url, start_time, entered_time, job_user, last_updated, job_deliveryservice, invalidation_type) VALUES (100, 24, 'http://cdn2.edge/job1/.*', '2018-01-19 21:01:14.000000', '2018-01-19 21:01:14.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.468643', 100, 'REFRESH'); +INSERT INTO job (id, ttl_hr, asset_url, start_time, entered_time, job_user, last_updated, job_deliveryservice, invalidation_type) VALUES (200, 36, 'http://cdn2.edge/job2/.*', '2018-01-19 21:09:34.000000', '2018-01-19 21:09:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.450915', 200, 'REFETCH'); +INSERT INTO job (id, ttl_hr, asset_url, start_time, entered_time, job_user, last_updated, job_deliveryservice, invalidation_type) VALUES (300, 72, 'http://cdn2.edge/job3/.*', '2018-01-19 21:14:34.000000', '2018-01-19 21:14:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.460870', 100, 'REFRESH'); ` err := execSQL(db, sqlStmt) if err != nil { @@ -311,8 +273,6 @@ func (r *TCData) Teardown(db *sql.DB) error { DELETE FROM to_extension; DELETE FROM staticdnsentry; DELETE FROM job; - DELETE FROM job_agent; - DELETE FROM job_status; DELETE FROM log; DELETE FROM asn; DELETE FROM deliveryservice_tmuser; diff --git a/lib/go-atscfg/atscfg.go b/lib/go-atscfg/atscfg.go index 7c76df6326..b30b27538b 100644 --- a/lib/go-atscfg/atscfg.go +++ b/lib/go-atscfg/atscfg.go @@ -29,7 +29,6 @@ import ( "strings" "github.com/apache/trafficcontrol/lib/go-tc" - "github.com/apache/trafficcontrol/lib/go-util" ) const InvalidID = -1 @@ -59,6 +58,11 @@ type Server tc.ServerV40 // but to only have to change it here, and the places where breaking symbol changes were made. type DeliveryService tc.DeliveryServiceV40 +// InvalidationJob is a tc.InvalidationJob for the latest lib/go-tc and traffic_ops/vx-client type. +// This allows atscfg to not have to change the type everywhere it's used, every time ATC changes the base type, +// but to only have to change it here, and the places where breaking symbol changes were made. +type InvalidationJob tc.InvalidationJobV4 + // ToDeliveryServices converts a slice of the latest lib/go-tc and traffic_ops/vx-client type to the local alias. func ToDeliveryServices(dses []tc.DeliveryServiceV40) []DeliveryService { ad := []DeliveryService{} @@ -77,6 +81,15 @@ func V40ToDeliveryServices(dses []tc.DeliveryServiceV40) []DeliveryService { return ad } +// ToInvalidationJobs converts a slice of the latest lib/go-tc and traffic_ops/vx-client type to the local alias. +func ToInvalidationJobs(jobs []tc.InvalidationJobV4) []InvalidationJob { + aj := []InvalidationJob{} + for _, job := range jobs { + aj = append(aj, InvalidationJob(job)) + } + return aj +} + // ToServers converts a slice of the latest lib/go-tc and traffic_ops/vx-client type to the local alias. func ToServers(servers []tc.ServerV40) []Server { as := []Server{} @@ -669,8 +682,8 @@ type DeliveryServiceServer struct { DeliveryService int `json:"d"` } -func JobsToInvalidationJobs(oldJobs []tc.Job) ([]tc.InvalidationJob, error) { - jobs := make([]tc.InvalidationJob, len(oldJobs), len(oldJobs)) +func JobsToInvalidationJobs(oldJobs []tc.Job) ([]InvalidationJob, error) { + jobs := make([]InvalidationJob, len(oldJobs), len(oldJobs)) err := error(nil) for i, oldJob := range oldJobs { jobs[i], err = JobToInvalidationJob(oldJob) @@ -681,18 +694,22 @@ func JobsToInvalidationJobs(oldJobs []tc.Job) ([]tc.InvalidationJob, error) { return jobs, nil } -func JobToInvalidationJob(jb tc.Job) (tc.InvalidationJob, error) { +func JobToInvalidationJob(jb tc.Job) (InvalidationJob, error) { startTime := tc.Time{} if err := json.Unmarshal([]byte(`"`+jb.StartTime+`"`), &startTime); err != nil { - return tc.InvalidationJob{}, errors.New("unmarshalling time: " + err.Error()) - } - return tc.InvalidationJob{ - AssetURL: util.StrPtr(jb.AssetURL), - CreatedBy: util.StrPtr(jb.CreatedBy), - DeliveryService: util.StrPtr(jb.DeliveryService), - ID: util.Uint64Ptr(uint64(jb.ID)), - Keyword: util.StrPtr(jb.Keyword), - Parameters: util.StrPtr(jb.Parameters), - StartTime: &startTime, + return InvalidationJob{}, errors.New("unmarshalling time: " + err.Error()) + } + ttl, err := strconv.Atoi(strings.TrimSuffix(strings.TrimPrefix(jb.Parameters, "TTL:"), "h")) + if err != nil { + return InvalidationJob{}, errors.New("unmarshalling ttl: " + err.Error()) + } + return InvalidationJob{ + AssetURL: jb.AssetURL, + CreatedBy: jb.CreatedBy, + DeliveryService: jb.DeliveryService, + ID: uint64(jb.ID), + TTLHours: uint(ttl), + InvalidationType: tc.REFRESH, + StartTime: startTime.Time, }, nil } diff --git a/lib/go-atscfg/regexrevalidatedotconfig.go b/lib/go-atscfg/regexrevalidatedotconfig.go index f20d78bdeb..86674ad289 100644 --- a/lib/go-atscfg/regexrevalidatedotconfig.go +++ b/lib/go-atscfg/regexrevalidatedotconfig.go @@ -20,7 +20,6 @@ package atscfg */ import ( - "fmt" "sort" "strconv" "strings" @@ -57,7 +56,7 @@ func MakeRegexRevalidateDotConfig( server *Server, deliveryServices []DeliveryService, globalParams []tc.Parameter, - jobs []tc.InvalidationJob, + jobs []InvalidationJob, opt *RegexRevalidateDotConfigOpts, ) (Cfg, error) { if opt == nil { @@ -80,13 +79,13 @@ func MakeRegexRevalidateDotConfig( dsNames[*ds.XMLID] = struct{}{} } - dsJobs := []tc.InvalidationJob{} + dsJobs := []InvalidationJob{} for _, job := range jobs { - if job.DeliveryService == nil { - warnings = append(warnings, "got job from Traffic Ops with a nil DeliveryService! Skipping!") + if job.DeliveryService == "" { + warnings = append(warnings, "got job from Traffic Ops with an empty DeliveryService! Skipping!") continue } - if _, ok := dsNames[*job.DeliveryService]; !ok { + if _, ok := dsNames[job.DeliveryService]; !ok { continue } dsJobs = append(dsJobs, job) @@ -106,8 +105,7 @@ func MakeRegexRevalidateDotConfig( maxReval := time.Duration(maxDays) * time.Hour * 24 - cfgJobs, jobWarns := filterJobs(dsJobs, maxReval, RegexRevalidateMinTTL) - warnings = append(warnings, jobWarns...) + cfgJobs := filterJobs(dsJobs, maxReval, RegexRevalidateMinTTL) txt := makeHdrComment(opt.HdrComment) for _, job := range cfgJobs { @@ -144,52 +142,26 @@ func (jb jobsSort) Less(i, j int) bool { } // filterJobs returns only jobs which: -// - have a non-null deliveryservice -// - have parameters of the form TTL:%dh +// - have a non-empty deliveryservice // - have a start time later than (now + maxReval days). That is, we don't query jobs older than maxReval in the past. -// - are "purge" jobs // - have a start_time+ttl > now. That is, jobs that haven't expired yet. -// Returns the filtered jobs, and any warnings. -func filterJobs(tcJobs []tc.InvalidationJob, maxReval time.Duration, minTTL time.Duration) ([]revalJob, []string) { - warnings := []string{} +// Returns the filtered jobs. +func filterJobs(tcJobs []InvalidationJob, maxReval time.Duration, minTTL time.Duration) []revalJob { jobMap := map[string]revalJob{} for _, tcJob := range tcJobs { - if tcJob.DeliveryService == nil || *tcJob.DeliveryService == "" { - continue - } - if tcJob.Parameters == nil { - continue - } - if !strings.HasPrefix(*tcJob.Parameters, `TTL:`) { - continue - } - if !strings.HasSuffix(*tcJob.Parameters, `h`) { + if tcJob.DeliveryService == "" { continue } - ttlHoursStr := *tcJob.Parameters - ttlHoursStr = strings.TrimPrefix(ttlHoursStr, `TTL:`) - ttlHoursStr = strings.TrimSuffix(ttlHoursStr, `h`) - ttlHours, err := strconv.Atoi(ttlHoursStr) - if err != nil { - warnings = append(warnings, fmt.Sprintf("job %+v has unexpected parameters ttl format, config generation skipping!\n", tcJob)) - continue - } - - ttl := time.Duration(ttlHours) * time.Hour + ttl := time.Duration(tcJob.TTLHours) * time.Hour if ttl > maxReval { ttl = maxReval } else if ttl < minTTL { ttl = minTTL } - if tcJob.StartTime == nil { - warnings = append(warnings, fmt.Sprintf("job %+v had nil start time, config generation skipping!\n", tcJob)) - continue - } - if tcJob.StartTime.Add(maxReval).Before(time.Now()) { continue } @@ -197,25 +169,8 @@ func filterJobs(tcJobs []tc.InvalidationJob, maxReval time.Duration, minTTL time if tcJob.StartTime.Add(ttl).Before(time.Now()) { continue } - if tcJob.Keyword == nil || *tcJob.Keyword != JobKeywordPurge { - continue - } - if tcJob.AssetURL == nil { - continue - } - - // process the __REFETCH__ keyword - assetURL := *tcJob.AssetURL - var jobType string - - if strings.HasSuffix(assetURL, RefetchSuffix) { - assetURL = strings.TrimSuffix(assetURL, RefetchSuffix) - jobType = "MISS" - } else if strings.HasSuffix(assetURL, RefreshSuffix) { // also default - assetURL = strings.TrimSuffix(assetURL, RefreshSuffix) - jobType = "STALE" - } + jobType, assetURL := processRefetch(tcJob.InvalidationType, tcJob.AssetURL) purgeEnd := tcJob.StartTime.Add(ttl) @@ -230,5 +185,24 @@ func filterJobs(tcJobs []tc.InvalidationJob, maxReval time.Duration, minTTL time } sort.Sort(jobsSort(newJobs)) - return newJobs, warnings + return newJobs +} + +const MISS = "MISS" +const STALE = "STALE" + +// processRefetch determines the type of Invalidation, returns the corresponding jobtype +// and "cleans" the regex URL for the asset to be invalidated. REFETCH trumps REFRESH, +// whether in the AssetURL or as InvalidationType +func processRefetch(invalidationType, assetURL string) (string, string) { + + if (len(invalidationType) > 0 && invalidationType == tc.REFETCH) || strings.HasSuffix(assetURL, RefetchSuffix) { + assetURL = strings.TrimSuffix(assetURL, RefetchSuffix) + return MISS, assetURL + } + + // Default value. Either the InvalidationType == REFRESH + // or the suffix is ##REFRESH## or neither + assetURL = strings.TrimSuffix(assetURL, RefreshSuffix) + return STALE, assetURL } diff --git a/lib/go-atscfg/regexrevalidatedotconfig_test.go b/lib/go-atscfg/regexrevalidatedotconfig_test.go index add0439ab6..1073a93468 100644 --- a/lib/go-atscfg/regexrevalidatedotconfig_test.go +++ b/lib/go-atscfg/regexrevalidatedotconfig_test.go @@ -41,46 +41,66 @@ func TestMakeRegexRevalidateDotConfig(t *testing.T) { dses := []DeliveryService{*ds} params := makeParamsFromMapArr("GLOBAL", RegexRevalidateFileName, map[string][]string{ - RegexRevalidateMaxRevalDurationDaysParamName: []string{"42"}, - "unrelated": []string{"unrelated0", "unrelated1"}, + RegexRevalidateMaxRevalDurationDaysParamName: {"42"}, + "unrelated": {"unrelated0", "unrelated1"}, }) - jobs := []tc.InvalidationJob{ - tc.InvalidationJob{ - AssetURL: util.StrPtr("assetURL0"), - StartTime: &tc.Time{Time: time.Now().Add(42*24*time.Hour + time.Hour), Valid: true}, - DeliveryService: util.StrPtr("myds"), - CreatedBy: util.StrPtr("me"), - ID: util.UInt64Ptr(42), - Parameters: util.StrPtr("TTL:14h"), - Keyword: util.StrPtr(JobKeywordPurge), + jobs := []InvalidationJob{ + { + AssetURL: "assetURL0", + StartTime: time.Now().Add(42*24*time.Hour + time.Hour), + DeliveryService: "myds", + CreatedBy: "me", + ID: 42, + TTLHours: 14, + InvalidationType: tc.REFRESH, }, - tc.InvalidationJob{ - AssetURL: util.StrPtr("expiredassetURL0"), - StartTime: &tc.Time{Time: time.Now().Add(-24 * time.Hour), Valid: true}, - DeliveryService: util.StrPtr("expiredmyds"), - CreatedBy: util.StrPtr("expiredme"), - ID: util.UInt64Ptr(42), - Parameters: util.StrPtr("TTL:14h"), - Keyword: util.StrPtr(JobKeywordPurge), + { + AssetURL: "expiredassetURL0", + StartTime: time.Now().Add(-24 * time.Hour), + DeliveryService: "expiredmyds", + CreatedBy: "expiredme", + ID: 42, + TTLHours: 14, + InvalidationType: tc.REFRESH, }, - tc.InvalidationJob{ - AssetURL: util.StrPtr("refetchasset##REFETCH##"), - StartTime: &tc.Time{Time: time.Now().Add(24 * time.Hour), Valid: true}, - DeliveryService: util.StrPtr("myds"), - CreatedBy: util.StrPtr("want_refetch"), - ID: util.UInt64Ptr(42), - Parameters: util.StrPtr("TTL:24h"), - Keyword: util.StrPtr(JobKeywordPurge), + { + AssetURL: "refetchasset##REFETCH##", + StartTime: time.Now().Add(24 * time.Hour), + DeliveryService: "myds", + CreatedBy: "want_refetch", + ID: 42, + TTLHours: 24, + InvalidationType: tc.REFETCH, }, - tc.InvalidationJob{ - AssetURL: util.StrPtr("refreshasset##REFRESH##"), - StartTime: &tc.Time{Time: time.Now().Add(24 * time.Hour), Valid: true}, - DeliveryService: util.StrPtr("myds"), - CreatedBy: util.StrPtr("want_refresh"), - ID: util.UInt64Ptr(42), - Parameters: util.StrPtr("TTL:24h"), - Keyword: util.StrPtr(JobKeywordPurge), + { + AssetURL: "refetchtype", + StartTime: time.Now().Add(24 * time.Hour), + DeliveryService: "myds", + CreatedBy: "want_refetch", + ID: 42, + TTLHours: 24, + InvalidationType: tc.REFETCH, + }, + { + // Mixed assetURL and invalidation type. REFETCH should trump REFRESH + // for backwards compatibility + AssetURL: "shouldbeREFETCH##REFETCH##", + StartTime: time.Now().Add(24 * time.Hour), + DeliveryService: "myds", + CreatedBy: "want_refetch", + ID: 42, + TTLHours: 24, + InvalidationType: tc.REFRESH, + }, + { + AssetURL: "refreshasset##REFRESH##", + StartTime: time.Now().Add(24 * time.Hour), + DeliveryService: "myds", + CreatedBy: "want_refresh", + ID: 42, + TTLHours: 24, + InvalidationType: tc.REFRESH, }, } diff --git a/lib/go-tc/enum.go b/lib/go-tc/enum.go index ec910d7a57..3bb9da6316 100644 --- a/lib/go-tc/enum.go +++ b/lib/go-tc/enum.go @@ -219,6 +219,19 @@ type ParameterName string // fetched by t3c. const UseRevalPendingParameterName = ParameterName("use_reval_pending") +// RefetchEnabled is the name of a Parameter used to determine if the +// Refetch feature is enabled. If enabled, this allows a consumer of the TO API +// to submit Refetch InvalidationJob types. These will subsequently be treated +// as a MISS by cache servers. Previously, the only capability was Refresh +// which was, in turn, treated as a STALE by cache servers. This value should +// be used with caution, since coupled with regex, could cause significant +// performance impacts by implementing cache servers if used incorrectly. +// +// Note that there is no guarantee that a Parameter with this name exists in +// Traffic Ops at any given time, and while it's implementation relies on +// a boolean Value, this it not gauranteed either. +const RefetchEnabled = ParameterName("refetch_enabled") + // ConfigFileName is a Parameter ConfigFile value. // // This has no additional attached semantics, and so while it is known to most diff --git a/lib/go-tc/invalidationjobs.go b/lib/go-tc/invalidationjobs.go index 6239f27a8f..a1d1488de3 100644 --- a/lib/go-tc/invalidationjobs.go +++ b/lib/go-tc/invalidationjobs.go @@ -246,6 +246,12 @@ func (job *InvalidationJobInput) Validate(tx *sql.Tx) error { return nil } +type compareJob struct { + AssetURL string + TTLHours uint + StartTime time.Time +} + // ValidateJobUniqueness returns a message describing each overlap between // existing content invalidation jobs for the same assetURL as the one passed. // @@ -255,35 +261,35 @@ func ValidateJobUniqueness(tx *sql.Tx, dsID uint, startTime time.Time, assetURL var errs []string const readQuery = ` -SELECT job.id, - keyword, - parameters, - asset_url, +SELECT asset_url, + ttl_hr, start_time FROM job WHERE job.job_deliveryservice = $1 ` rows, err := tx.Query(readQuery, dsID) if err != nil { - errs = append(errs, fmt.Sprintf("unable to query for other invalidation jobs")) + errs = append(errs, "unable to query for invalidation jobs while validating job uniqueness") } else { defer rows.Close() jobStart := startTime for rows.Next() { - testJob := InvalidationJob{} - err = rows.Scan(&testJob.ID, &testJob.Keyword, &testJob.Parameters, &testJob.AssetURL, &testJob.StartTime) + testJob := compareJob{} + err = rows.Scan( + &testJob.AssetURL, + &testJob.TTLHours, + &testJob.StartTime) if err != nil { continue } - if !strings.HasSuffix(*testJob.AssetURL, assetURL) { + if !strings.HasSuffix(testJob.AssetURL, assetURL) { continue } - testJobTTL := testJob.TTLHours() - if testJobTTL == 0 { + if testJob.TTLHours == 0 { continue } - testJobStart := testJob.StartTime.Time - testJobEnd := testJobStart.Add(time.Hour * time.Duration(testJobTTL)) + testJobStart := testJob.StartTime + testJobEnd := testJobStart.Add(time.Hour * time.Duration(testJob.TTLHours)) jobEnd := jobStart.Add(time.Hour * time.Duration(ttlHours)) // jobStart in testJob range if (testJobStart.Before(jobStart) && jobStart.Before(testJobEnd)) || @@ -292,7 +298,7 @@ WHERE job.job_deliveryservice = $1 // job range encaspulates testJob range (testJobEnd.Before(jobEnd) && jobStart.Before(jobStart)) { errs = append(errs, fmt.Sprintf("Invalidation request duplicate found for %v, start:%v end:%v", - *testJob.AssetURL, testJobStart, testJobEnd)) + testJob.AssetURL, testJobStart, testJobEnd)) } } } @@ -456,3 +462,67 @@ func (job *UserInvalidationJobInput) Validate(tx *sql.Tx) error { } return nil } + +const REFRESH = "REFRESH" +const REFETCH = "REFETCH" + +// InvalidationJobsResponse is the type of a response from Traffic Ops to a +// request made to its /jobs API endpoint for v 4.0+ +type InvalidationJobsResponseV4 struct { + Response []InvalidationJobV4 `json:"response"` + Alerts +} + +// InvalidationJobCreateV4 is an alias for the InvalidationJobCreateV40 struct used for the latest minor version associated with api major version 4. +type InvalidationJobCreateV4 InvalidationJobCreateV40 + +// InvalidationJobCreateV40 represents user input intending to create a content invalidation job. +type InvalidationJobCreateV40 struct { + // The Delivery Service XML-ID for which the Invalidation Job is to be applied. + DeliveryService string `json:"deliveryService"` + + // Regex is a regular expression which not only must be valid, but should also start with '/' + // (or escaped: '\/') + Regex string `json:"regex"` + + // StartTime is the time at which the job will come into effect. Must be in the future. + StartTime time.Time `json:"startTime"` + + // TTLHours indicates the Time-to-Live of the job in hours. Must be a positive integer value. + TTLHours uint32 `json:"ttlHours"` + + // InvalidationType must be either REFRESH (default behavior) or REFETCH. If REFETCH, must + // also comply with global parameter setting + InvalidationType string `json:"invalidationType"` +} + +// InvalidationJobV4 is an alias for the InvalidationJobV4 struct used for the latest minor version associated with api major version 4. +type InvalidationJobV4 InvalidationJobV40 + +// InvalidationJobV40 represents a content invalidation job as returned by the API. +// Also used for Update calls. +type InvalidationJobV40 struct { + ID uint64 `json:"id"` + AssetURL string `json:"assetUrl"` + CreatedBy string `json:"createdBy"` + DeliveryService string `json:"deliveryService"` + TTLHours uint `json:"ttlHours"` + InvalidationType string `json:"invalidationType"` + StartTime time.Time `json:"startTime"` +} + +func (job InvalidationJobV4) String() string { + + ID := strconv.FormatUint(job.ID, 10) + TTLHours := strconv.FormatUint(uint64(job.TTLHours), 10) + StartTime := job.StartTime.String() + + return fmt.Sprintf("{\nID:\t\t\t\t\t%s,\nAssetURL:\t\t\t%s,\nCreatedBy:\t\t\t%s,\nDeliveryService:\t%s,\nTTLHours:\t\t\t%s,\nInvalidationType:\t%s,\nStartTime:\t\t\t%s,\n}", + ID, + job.AssetURL, + job.CreatedBy, + job.DeliveryService, + TTLHours, + job.InvalidationType, + StartTime) +} diff --git a/lib/go-util/ptr.go b/lib/go-util/ptr.go index 5ffc3d5f1e..e25ecbc85b 100644 --- a/lib/go-util/ptr.go +++ b/lib/go-util/ptr.go @@ -1,5 +1,3 @@ -package util - /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -19,6 +17,10 @@ package util * under the License. */ +package util + +import "time" + func StrPtr(str string) *string { return &str } @@ -54,3 +56,7 @@ func FloatPtr(f float64) *float64 { func InterfacePtr(i interface{}) *interface{} { return &i } + +func TimePtr(t time.Time) *time.Time { + return &t +} diff --git a/traffic_ops/app/db/migrations/2021110211525600_refetch.down.sql b/traffic_ops/app/db/migrations/2021110211525600_refetch.down.sql new file mode 100644 index 0000000000..452b1bab62 --- /dev/null +++ b/traffic_ops/app/db/migrations/2021110211525600_refetch.down.sql @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +-- Revert + +/** + * Restore job_agent table with constraints and data. + */ + +CREATE TABLE public.job_agent ( + id bigserial NOT NULL, + "name" text NULL, + description text NULL, + active int4 NOT NULL DEFAULT 0, + last_updated timestamptz NOT NULL DEFAULT now(), + CONSTRAINT idx_job_agent_id_primary PRIMARY KEY (id), + CONSTRAINT job_agent_name_unique UNIQUE (name) +); +CREATE INDEX job_agent_last_updated_idx ON public.job_agent USING btree (last_updated DESC NULLS LAST); + +-- Table Triggers + +create trigger on_delete_current_timestamp after +delete + on + public.job_agent for each row execute function on_delete_current_timestamp_last_updated('job_agent'); +create trigger on_update_current_timestamp before +update + on + public.job_agent for each row execute function on_update_current_timestamp_last_updated(); + +-- Permissions + +ALTER TABLE public.job_agent OWNER TO traffic_ops; +GRANT ALL ON TABLE public.job_agent TO traffic_ops; + +-- Data + +INSERT INTO public.job_agent (name, description, active) +VALUES ('Example', 'Description of Purge Agent', 1) +ON CONFLICT (name) DO NOTHING; + +/** + * Restore job_status table with constraints and data + */ + +CREATE TABLE public.job_status ( + id bigserial NOT NULL, + "name" text NULL, + description text NULL, + last_updated timestamptz NOT NULL DEFAULT now(), + CONSTRAINT idx_job_status_id_primary PRIMARY KEY (id), + CONSTRAINT job_status_name_unique UNIQUE (name) +); +CREATE INDEX job_status_last_updated_idx ON public.job_status USING btree (last_updated DESC NULLS LAST); + +-- Table Triggers + +create trigger on_delete_current_timestamp after +delete + on + public.job_status for each row execute function on_delete_current_timestamp_last_updated('job_status'); +create trigger on_update_current_timestamp before +update + on + public.job_status for each row execute function on_update_current_timestamp_last_updated(); + +-- Permissions + +ALTER TABLE public.job_status OWNER TO traffic_ops; +GRANT ALL ON TABLE public.job_status TO traffic_ops; + +-- Data + +INSERT INTO job_status (name, description) +VALUES ('PENDING', 'Job is queued, but has not been picked up by any agents yet'), +('IN_PROGRESS', 'Job is being processed by agents'), +('COMPLETED', 'Job has finished'), +('CANCELLED', 'Job was cancelled') +ON CONFLICT (name) DO NOTHING; + +/** + * Restore job table with constraints and data + */ + +-- Restore table +ALTER TABLE public.job +ADD COLUMN IF NOT EXISTS agent int8 NULL, +ADD COLUMN IF NOT EXISTS object_type text NULL, +ADD COLUMN IF NOT EXISTS object_name text NULL, +ADD COLUMN IF NOT EXISTS keyword text NULL, +ADD COLUMN IF NOT EXISTS asset_type text NULL, +ADD COLUMN IF NOT EXISTS status int8 NULL; + +ALTER TABLE public.job +DROP COLUMN IF EXISTS invalidation_type CASCADE; + +-- Restore indices and foreign key constraints +CREATE INDEX idx_job_fk_job_agent_id ON public.job USING btree (agent); +ALTER TABLE public.job ADD CONSTRAINT job_fk_job_agent_id FOREIGN KEY (agent) REFERENCES public.job_agent(id) ON DELETE CASCADE; + +CREATE INDEX idx_job_fk_job_status_id ON public.job USING btree (status); +ALTER TABLE public.job ADD CONSTRAINT job_fk_job_status_id FOREIGN KEY (status) REFERENCES public.job_status(id); + +-- Restore data +UPDATE public.job +SET agent = 1; + +UPDATE public.job +SET status = ( + SELECT id FROM public.job_status WHERE name = 'PENDING' +); +ALTER TABLE public.job +ALTER COLUMN status SET NOT NULL; + +UPDATE public.job +SET keyword = 'PURGE'; +ALTER TABLE public.job +ALTER COLUMN keyword SET NOT NULL; + +UPDATE public.job +SET asset_type = 'file'; +ALTER TABLE public.job +ALTER COLUMN asset_type SET NOT NULL; + +ALTER TABLE public.job +RENAME COLUMN ttl_hr TO parameters; + +ALTER TABLE public.job +ALTER COLUMN parameters TYPE TEXT +USING CONCAT('TTL:', parameters, 'h'); diff --git a/traffic_ops/app/db/migrations/2021110211525600_refetch.up.sql b/traffic_ops/app/db/migrations/2021110211525600_refetch.up.sql new file mode 100644 index 0000000000..bab872e31b --- /dev/null +++ b/traffic_ops/app/db/migrations/2021110211525600_refetch.up.sql @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +-- Migrate + +/** + * Alter job_agent table and data + */ + +DROP TABLE IF EXISTS public.job_agent CASCADE; + +/** + * Alter job_status table and data + */ + +DROP TABLE IF EXISTS public.job_status CASCADE; + +/** + * Alter job table and data + */ + +ALTER TABLE public.job +DROP COLUMN IF EXISTS agent CASCADE, +DROP COLUMN IF EXISTS object_type CASCADE, +DROP COLUMN IF EXISTS object_name CASCADE, +DROP COLUMN IF EXISTS keyword CASCADE, +DROP COLUMN IF EXISTS asset_type CASCADE, +DROP COLUMN IF EXISTS status CASCADE; + +ALTER TABLE public.job +ADD COLUMN IF NOT EXISTS invalidation_type text NOT NULL DEFAULT 'REFRESH'; + +/* + * If the asset_url contains the temporary fix for refetch + * (adding ##REFETCH## to the end of the url) then assign the + * correct invalidation_type. + */ +UPDATE public.job +SET invalidation_type = 'REFETCH' +WHERE asset_url LIKE '%##REFETCH##%'; + +ALTER TABLE public.job +RENAME COLUMN parameters TO ttl_hr; + +ALTER TABLE public.job +ALTER COLUMN ttl_hr TYPE integer +USING CAST(substring(ttl_hr,'TTL:([0-9]+)h') AS integer); diff --git a/traffic_ops/app/db/seeds.sql b/traffic_ops/app/db/seeds.sql index 67cd77de45..49ef3546e7 100644 --- a/traffic_ops/app/db/seeds.sql +++ b/traffic_ops/app/db/seeds.sql @@ -19,12 +19,6 @@ -- cdns insert into cdn (name, dnssec_enabled, domain_name) values ('ALL', false, '-') ON CONFLICT (name) DO NOTHING; --- job agents -insert into job_agent (name, description, active) values ('dummy', 'Description of Purge Agent', 1) ON CONFLICT (name) DO NOTHING; - --- job statuses -insert into job_status (name, description) values ('PENDING', 'Job is queued, but has not been picked up by any agents yet') ON CONFLICT (name) DO NOTHING; - -- parameters -- Moved into postinstall global parameters insert into profile (name, description, type, cdn) values ('GLOBAL', 'Global Traffic Ops profile, DO NOT DELETE', 'UNK_PROFILE', (SELECT id FROM cdn WHERE name='ALL')) ON CONFLICT (name) DO NOTHING; @@ -917,8 +911,6 @@ insert into last_deleted (table_name) VALUES ('federation_resolver') ON CONFLICT insert into last_deleted (table_name) VALUES ('federation_tmuser') ON CONFLICT (table_name) DO NOTHING; insert into last_deleted (table_name) VALUES ('hwinfo') ON CONFLICT (table_name) DO NOTHING; insert into last_deleted (table_name) VALUES ('job') ON CONFLICT (table_name) DO NOTHING; -insert into last_deleted (table_name) VALUES ('job_agent') ON CONFLICT (table_name) DO NOTHING; -insert into last_deleted (table_name) VALUES ('job_status') ON CONFLICT (table_name) DO NOTHING; insert into last_deleted (table_name) VALUES ('log') ON CONFLICT (table_name) DO NOTHING; insert into last_deleted (table_name) VALUES ('origin') ON CONFLICT (table_name) DO NOTHING; insert into last_deleted (table_name) VALUES ('parameter') ON CONFLICT (table_name) DO NOTHING; diff --git a/traffic_ops/testing/api/README.md b/traffic_ops/testing/api/README.md index 87537ea2db..056d5517b0 100644 --- a/traffic_ops/testing/api/README.md +++ b/traffic_ops/testing/api/README.md @@ -31,9 +31,10 @@ In order to run the tests you will need the following: To get your to_test database setup do the following: - `$ cd trafficcontrol/traffic_ops/app` - - `$ db/admin --env=test reset` + ```shell + $ cd trafficcontrol/traffic_ops/app + $ db/admin --env=test reset + ``` NOTE on passwords: Check that the passwords defined defined for your `to_test` database match @@ -45,6 +46,7 @@ In order to run the tests you will need the following: Note that for the database to successfully set up the tables and run the migrations, you will the `db/admin` tool. To test if `db/admin` ran all migrations successfully, you can run the following command from the `traffic_ops/app` directory: + ```shell db/admin -env=test dbversion ``` @@ -61,29 +63,20 @@ In order to run the tests you will need the following: For more info see: http://trafficcontrol.apache.org/docs/latest/development/traffic_ops.html?highlight=reset -3. A running Traffic Ops instance running with the `secure` (https) and is pointing to the `to_test` - database by running in `MOJO_MODE=test` which will point to your `to_test` database. - To get your to_test database setup do the following: - - `$ export MOJO_MODE=test` - - `$ cd trafficcontrol/traffic_ops/app` - - `$ bin/start.pl --secure` - -4. A running Traffic Ops Golang proxy pointing to the to_test database. - `$ cd trafficcontrol/traffic_ops/traffic_ops_golang` - `$ cp ../app/conf/cdn.conf $HOME/cdn.conf` - change `traffic_ops_golang->port` to 8443 - - `$ go build && ./traffic_ops_golang -cfg $HOME/cdn.conf -dbcfg ../app/conf/test/database.conf` +3. A running Traffic Ops Golang instance pointing to the `to_test` database. + + ```shell + $ cd trafficcontrol/traffic_ops/traffic_ops_golang + $ cp ../app/conf/cdn.conf $HOME/cdn.conf # change `traffic_ops_golang->port` to 8443 + $ go build && ./traffic_ops_golang -cfg $HOME/cdn.conf -dbcfg ../app/conf/test/database.conf + ``` In your local development environment, if the above command fails for an error similar to `ERROR: traffic_ops_golang.go:193: 2020-04-10T10:55:53.190298-06:00: cannot open /etc/pki/tls/certs/localhost.crt for read: open /etc/pki/tls/certs/localhost.crt: no such file or directory` you might not have the right certificates at the right locations. Follow the procedure listed [here](https://traffic-control-cdn.readthedocs.io/en/latest/admin/traffic_ops.html#id12) to fix it. ## Running the API Tests -The integration tests are run using `go test`, however, there are some flags that need to be provided in order for the tests to work. +The integration tests are run using `go test` from the traffic_ops/testing/api/ directory, however, there are some flags that need to be provided in order for the tests to work. The flags are: @@ -94,9 +87,16 @@ The flags are: * test_data - traffic control * run - Go runtime flag for executing a specific test case -Example command to run the tests: -`TO_URL=https://localhost:8443 go test -v -cfg=traffic-ops-test.conf -run TestCDNs` +Example command to run the tests: +```shell +$ TO_URL=https://localhost:8443 go test -v -cfg=traffic-ops-test.conf -run TestCDNs +``` + +or, since the cfg file location is inferred, the call could be shortened to test a specific API version with something like: +```shell +$ go test -v -run TestJobs ./v4 +``` * It can take several minutes for the API tests to complete, so using the `-v` flag is recommended to see progress.* diff --git a/traffic_ops/testing/api/v2/todb_test.go b/traffic_ops/testing/api/v2/todb_test.go index 370187d886..b5aa360549 100644 --- a/traffic_ops/testing/api/v2/todb_test.go +++ b/traffic_ops/testing/api/v2/todb_test.go @@ -86,18 +86,6 @@ func SetupTestData(*sql.DB) error { os.Exit(1) } - err = SetupJobAgents(db) - if err != nil { - fmt.Printf("\nError setting up job agents %s - %s, %v\n", Config.TrafficOps.URL, Config.TrafficOps.Users.Admin, err) - os.Exit(1) - } - - err = SetupJobStatuses(db) - if err != nil { - fmt.Printf("\nError setting up job agents %s - %s, %v\n", Config.TrafficOps.URL, Config.TrafficOps.Users.Admin, err) - os.Exit(1) - } - err = SetupTypes(db) if err != nil { fmt.Printf("\nError setting up types %s - %s, %v\n", Config.TrafficOps.URL, Config.TrafficOps.Users.Admin, err) @@ -229,39 +217,13 @@ INSERT INTO deliveryservice_tmuser (deliveryservice, tm_user_id, last_updated) V return nil } -// SetupJobStatuses ... -func SetupJobStatuses(db *sql.DB) error { - - sqlStmt := ` -INSERT INTO job_status (id, name, description, last_updated) VALUES (1, 'PENDING', 'Job is queued, but has not been picked up by any agents yet', '2018-01-19 21:19:32.444857'); -` - err := execSQL(db, sqlStmt) - if err != nil { - return fmt.Errorf("exec failed %v", err) - } - return nil -} - -// SetupJobAgents ... -func SetupJobAgents(db *sql.DB) error { - - sqlStmt := ` -INSERT INTO job_agent (id, name, description, active, last_updated) VALUES (1, 'agent1', 'Test Agent1', 0, '2018-01-19 21:19:32.448076'); -` - err := execSQL(db, sqlStmt) - if err != nil { - return fmt.Errorf("exec failed %v", err) - } - return nil -} - // SetupJobs ... func SetupJobs(db *sql.DB) error { sqlStmt := ` -INSERT INTO job (id, agent, object_type, object_name, keyword, parameters, asset_url, asset_type, status, start_time, entered_time, job_user, last_updated, job_deliveryservice) VALUES (100, 1, null, null, 'PURGE', 'TTL:48h', 'http://cdn2.edge/job1/.*', 'file', 1, '2018-01-19 21:01:14.000000', '2018-01-19 21:01:14.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.468643', 100); -INSERT INTO job (id, agent, object_type, object_name, keyword, parameters, asset_url, asset_type, status, start_time, entered_time, job_user, last_updated, job_deliveryservice) VALUES (200, 1, null, null, 'PURGE', 'TTL:48h', 'http://cdn2.edge/job2/.*', 'file', 1, '2018-01-19 21:09:34.000000', '2018-01-19 21:09:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.450915', 200); -INSERT INTO job (id, agent, object_type, object_name, keyword, parameters, asset_url, asset_type, status, start_time, entered_time, job_user, last_updated, job_deliveryservice) VALUES (300, 1, null, null, 'PURGE', 'TTL:48h', 'http://cdn2.edge/job3/.*', 'file', 1, '2018-01-19 21:14:34.000000', '2018-01-19 21:14:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.460870', 100); +INSERT INTO job (id, ttl_hr, asset_url, start_time, entered_time, job_user, last_updated, job_deliveryservice, invalidation_type) VALUES (100, 24, 'http://cdn2.edge/job1/.*', '2018-01-19 21:01:14.000000', '2018-01-19 21:01:14.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.468643', 100, 'REFRESH'); +INSERT INTO job (id, ttl_hr, asset_url, start_time, entered_time, job_user, last_updated, job_deliveryservice, invalidation_type) VALUES (200, 36, 'http://cdn2.edge/job2/.*', '2018-01-19 21:09:34.000000', '2018-01-19 21:09:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.450915', 200, 'REFETCH'); +INSERT INTO job (id, ttl_hr, asset_url, start_time, entered_time, job_user, last_updated, job_deliveryservice, invalidation_type) VALUES (300, 72, 'http://cdn2.edge/job3/.*', '2018-01-19 21:14:34.000000', '2018-01-19 21:14:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.460870', 100, 'REFRESH'); ` err := execSQL(db, sqlStmt) if err != nil { @@ -310,8 +272,6 @@ func Teardown(db *sql.DB) error { DELETE FROM to_extension; DELETE FROM staticdnsentry; DELETE FROM job; - DELETE FROM job_agent; - DELETE FROM job_status; DELETE FROM log; DELETE FROM asn; DELETE FROM deliveryservice_tmuser; diff --git a/traffic_ops/testing/api/v3/todb_test.go b/traffic_ops/testing/api/v3/todb_test.go index c0ab6a66da..088717fdc8 100644 --- a/traffic_ops/testing/api/v3/todb_test.go +++ b/traffic_ops/testing/api/v3/todb_test.go @@ -86,18 +86,6 @@ func SetupTestData(*sql.DB) error { os.Exit(1) } - err = SetupJobAgents(db) - if err != nil { - fmt.Printf("\nError setting up job agents %s - %s, %v\n", Config.TrafficOps.URL, Config.TrafficOps.Users.Admin, err) - os.Exit(1) - } - - err = SetupJobStatuses(db) - if err != nil { - fmt.Printf("\nError setting up job agents %s - %s, %v\n", Config.TrafficOps.URL, Config.TrafficOps.Users.Admin, err) - os.Exit(1) - } - err = SetupTypes(db) if err != nil { fmt.Printf("\nError setting up types %s - %s, %v\n", Config.TrafficOps.URL, Config.TrafficOps.Users.Admin, err) @@ -229,39 +217,13 @@ INSERT INTO deliveryservice_tmuser (deliveryservice, tm_user_id, last_updated) V return nil } -// SetupJobStatuses ... -func SetupJobStatuses(db *sql.DB) error { - - sqlStmt := ` -INSERT INTO job_status (id, name, description, last_updated) VALUES (1, 'PENDING', 'Job is queued, but has not been picked up by any agents yet', '2018-01-19 21:19:32.444857'); -` - err := execSQL(db, sqlStmt) - if err != nil { - return fmt.Errorf("exec failed %v", err) - } - return nil -} - -// SetupJobAgents ... -func SetupJobAgents(db *sql.DB) error { - - sqlStmt := ` -INSERT INTO job_agent (id, name, description, active, last_updated) VALUES (1, 'agent1', 'Test Agent1', 0, '2018-01-19 21:19:32.448076'); -` - err := execSQL(db, sqlStmt) - if err != nil { - return fmt.Errorf("exec failed %v", err) - } - return nil -} - // SetupJobs ... func SetupJobs(db *sql.DB) error { sqlStmt := ` -INSERT INTO job (id, agent, object_type, object_name, keyword, parameters, asset_url, asset_type, status, start_time, entered_time, job_user, last_updated, job_deliveryservice) VALUES (100, 1, null, null, 'PURGE', 'TTL:48h', 'http://cdn2.edge/job1/.*', 'file', 1, '2018-01-19 21:01:14.000000', '2018-01-19 21:01:14.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.468643', 100); -INSERT INTO job (id, agent, object_type, object_name, keyword, parameters, asset_url, asset_type, status, start_time, entered_time, job_user, last_updated, job_deliveryservice) VALUES (200, 1, null, null, 'PURGE', 'TTL:48h', 'http://cdn2.edge/job2/.*', 'file', 1, '2018-01-19 21:09:34.000000', '2018-01-19 21:09:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.450915', 200); -INSERT INTO job (id, agent, object_type, object_name, keyword, parameters, asset_url, asset_type, status, start_time, entered_time, job_user, last_updated, job_deliveryservice) VALUES (300, 1, null, null, 'PURGE', 'TTL:48h', 'http://cdn2.edge/job3/.*', 'file', 1, '2018-01-19 21:14:34.000000', '2018-01-19 21:14:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.460870', 100); +INSERT INTO job (id, ttl_hr, asset_url, start_time, entered_time, job_user, last_updated, job_deliveryservice, invalidation_type) VALUES (100, 24, 'http://cdn2.edge/job1/.*', '2018-01-19 21:01:14.000000', '2018-01-19 21:01:14.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.468643', 100, 'REFRESH'); +INSERT INTO job (id, ttl_hr, asset_url, start_time, entered_time, job_user, last_updated, job_deliveryservice, invalidation_type) VALUES (200, 36, 'http://cdn2.edge/job2/.*', '2018-01-19 21:09:34.000000', '2018-01-19 21:09:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.450915', 200, 'REFETCH'); +INSERT INTO job (id, ttl_hr, asset_url, start_time, entered_time, job_user, last_updated, job_deliveryservice, invalidation_type) VALUES (300, 72, 'http://cdn2.edge/job3/.*', '2018-01-19 21:14:34.000000', '2018-01-19 21:14:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.460870', 100, 'REFRESH'); ` err := execSQL(db, sqlStmt) if err != nil { @@ -310,8 +272,6 @@ func Teardown(db *sql.DB) error { DELETE FROM to_extension; DELETE FROM staticdnsentry; DELETE FROM job; - DELETE FROM job_agent; - DELETE FROM job_status; DELETE FROM log; DELETE FROM asn; DELETE FROM deliveryservice_tmuser; diff --git a/traffic_ops/testing/api/v4/jobs_test.go b/traffic_ops/testing/api/v4/jobs_test.go index 047ddc75dc..dcd62f5457 100644 --- a/traffic_ops/testing/api/v4/jobs_test.go +++ b/traffic_ops/testing/api/v4/jobs_test.go @@ -16,6 +16,7 @@ package v4 */ import ( + "fmt" "net/http" "strconv" "strings" @@ -23,7 +24,6 @@ import ( "time" "github.com/apache/trafficcontrol/lib/go-tc" - "github.com/apache/trafficcontrol/lib/go-util" client "github.com/apache/trafficcontrol/traffic_ops/v4-client" ) @@ -48,6 +48,8 @@ func TestJobs(t *testing.T) { UpdateTestJobsInvalidDS(t) DeleteTestJobs(t) DeleteTestJobsByInvalidId(t) + CreateRefetchJobParameterFail(t) + CreateRefetchJobParameterSuccess(t) }) } @@ -58,19 +60,17 @@ func CreateTestJobs(t *testing.T) { } for i, job := range testData.InvalidationJobs { - job.StartTime = &tc.Time{ - Time: time.Now().Add(time.Minute).UTC(), - Valid: true, - } + job.StartTime = time.Now().Add(time.Minute).UTC() testData.InvalidationJobs[i] = job } for _, job := range testData.InvalidationJobs { - request := tc.InvalidationJobInput{ - DeliveryService: job.DeliveryService, - Regex: job.Regex, - StartTime: job.StartTime, - TTL: job.TTL, + request := tc.InvalidationJobCreateV4{ + DeliveryService: job.DeliveryService, + Regex: job.Regex, + StartTime: job.StartTime, + TTLHours: job.TTLHours, + InvalidationType: job.InvalidationType, } resp, _, err := TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err != nil { @@ -88,15 +88,12 @@ func JobCollisionWarningTest(t *testing.T) { } xmlID := *testData.DeliveryServices[0].XMLID - startTime := tc.Time{ - Time: time.Now().Add(time.Hour), - Valid: true, - } - firstJob := tc.InvalidationJobInput{ - DeliveryService: util.InterfacePtr(&xmlID), - Regex: util.StrPtr(`/\.*([A-Z]0?)`), - TTL: util.InterfacePtr(16), - StartTime: &startTime, + firstJob := tc.InvalidationJobCreateV4{ + DeliveryService: xmlID, + Regex: `/\.*([A-Z]0?)`, + TTLHours: 16, + StartTime: time.Now().Add(time.Hour), + InvalidationType: tc.REFRESH, } resp, _, err := TOSession.CreateInvalidationJob(firstJob, client.RequestOptions{}) @@ -104,15 +101,12 @@ func JobCollisionWarningTest(t *testing.T) { t.Fatalf("Unexpected error creating a content invalidation Job: %v - alerts: %+v", err, resp.Alerts) } - newTime := tc.Time{ - Time: startTime.Time.Add(time.Hour), - Valid: true, - } - newJob := tc.InvalidationJobInput{ - DeliveryService: firstJob.DeliveryService, - Regex: firstJob.Regex, - TTL: firstJob.TTL, - StartTime: &newTime, + newJob := tc.InvalidationJobCreateV4{ + DeliveryService: firstJob.DeliveryService, + Regex: firstJob.Regex, + TTLHours: firstJob.TTLHours, + StartTime: firstJob.StartTime.Add(time.Hour), + InvalidationType: tc.REFRESH, } alerts, _, err := TOSession.CreateInvalidationJob(newJob, client.RequestOptions{}) @@ -126,7 +120,7 @@ func JobCollisionWarningTest(t *testing.T) { found := false for _, alert := range alerts.Alerts { - if alert.Level == tc.WarnLevel.String() && strings.Contains(alert.Text, *firstJob.Regex) { + if alert.Level == tc.WarnLevel.String() && strings.Contains(alert.Text, firstJob.Regex) { found = true } } @@ -141,25 +135,21 @@ func JobCollisionWarningTest(t *testing.T) { t.Fatalf("unable to get invalidation jobs: %v - alerts: %+v", err, jobs.Alerts) } - var realJob *tc.InvalidationJob + var realJob *tc.InvalidationJobV4 for i, job := range jobs.Response { - if job.StartTime == nil || job.DeliveryService == nil || job.CreatedBy == nil { - t.Error("Traffic Ops returned a representation of a content invalidation Job that had null or undefined Start Time and/or Delivery Service and/or Created By") - continue - } - diff := newJob.StartTime.Time.Sub(job.StartTime.Time) - if *job.DeliveryService == xmlID && *job.CreatedBy == "admin" && diff < time.Second { + diff := newJob.StartTime.Sub(job.StartTime) + if job.DeliveryService == xmlID && job.CreatedBy == "admin" && diff < time.Second { realJob = &jobs.Response[i] break } } - if realJob == nil || realJob.ID == nil || *realJob.ID == 0 { + if realJob == nil || realJob.ID == 0 { t.Fatal("could not find new job") } - newTime.Time = startTime.Time.Add(time.Hour * 2) - realJob.StartTime = &newTime + time := firstJob.StartTime.Add(time.Hour * 2) + realJob.StartTime = time alerts, _, err = TOSession.UpdateInvalidationJob(*realJob, client.RequestOptions{}) if err != nil { t.Fatalf("expected invalidation job update to succeed: %v - alerts: %+v", err, alerts.Alerts) @@ -171,7 +161,7 @@ func JobCollisionWarningTest(t *testing.T) { found = false for _, alert := range alerts.Alerts { - if alert.Level == tc.WarnLevel.String() && strings.Contains(alert.Text, *firstJob.Regex) { + if alert.Level == tc.WarnLevel.String() && strings.Contains(alert.Text, firstJob.Regex) { found = true } } @@ -195,10 +185,7 @@ func CreateTestInvalidationJobs(t *testing.T) { } for _, job := range testData.InvalidationJobs { - if job.DeliveryService == nil { - t.Error("Found a Job in the test data that has null or undefined Delivery Service") - } - _, ok := dsNameIDs[(*job.DeliveryService).(string)] + _, ok := dsNameIDs[job.DeliveryService] if !ok { t.Fatalf("can't create test data job: delivery service '%v' not found in Traffic Ops", job.DeliveryService) } @@ -226,10 +213,7 @@ func CreateTestInvalidJob(t *testing.T) { t.Fatal("Need at least one Invalidation Job to test creating an invalid Job") } job := testData.InvalidationJobs[0] - if job.DeliveryService == nil { - t.Fatal("Found a Job in the testing data that has null or undefined Delivery Service") - } - _, ok := dsNameIDs[(*job.DeliveryService).(string)] + _, ok := dsNameIDs[job.DeliveryService] if !ok { t.Fatalf("can't create test data job: delivery service '%v' not found in Traffic Ops", job.DeliveryService) } @@ -248,8 +232,8 @@ func CreateTestInvalidJob(t *testing.T) { if !foundMaxRevalDays { t.Fatalf("expected: parameter named maxRevalDurationDays, actual: not found") } - tooHigh := interface{}((maxRevalDays * 24) + 1) - job.TTL = &tooHigh + tooHigh := (uint32)(maxRevalDays*24) + 1 + job.TTLHours = tooHigh _, reqInf, err := TOSession.CreateInvalidationJob(job, client.RequestOptions{}) if err == nil { t.Error("creating invalid job (TTL higher than maxRevalDurationDays) - expected: error, actual: nil error") @@ -268,10 +252,8 @@ func GetTestJobsQueryParams(t *testing.T) { } foundOne := false for _, j := range toJobs.Response { - if j.DeliveryService == nil { - t.Error("expected: non-nil DeliveryService pointer, actual: nil") - } else if *j.DeliveryService != "ds2" { - t.Errorf("expected: DeliveryService == ds2, actual: DeliveryService == %s", *j.DeliveryService) + if j.DeliveryService != "ds2" { + t.Errorf("expected: DeliveryService == ds2, actual: DeliveryService == %s", j.DeliveryService) } else { foundOne = true } @@ -292,41 +274,34 @@ func GetTestJobs(t *testing.T) { t.Fatalf("cannot get Delivery Services: %v - alerts: %+v", err, toDSes.Alerts) } - for i, testJob := range testData.InvalidationJobs { + for _, testJob := range testData.InvalidationJobs { found := false - if testJob.DeliveryService == nil { - t.Errorf("test job (index %v) has nil delivery service", i) - continue - } else if testJob.Regex == nil { - t.Errorf("test job (index %v) has nil regex", i) - continue - } for j, toJob := range toJobs.Response { - if toJob.DeliveryService == nil { - t.Errorf("to job (index %v) has nil delivery service", j) + if toJob.DeliveryService == "" { + t.Errorf("to job (index %v) has empty delivery service", j) continue } - if toJob.AssetURL == nil { - t.Errorf("to job (index %v) has nil asset url", j) + if toJob.AssetURL == "" { + t.Errorf("to job (index %v) has empty asset url", j) continue } - if *toJob.DeliveryService != *testJob.DeliveryService { + if toJob.DeliveryService != testJob.DeliveryService { continue } - if !strings.HasSuffix(*toJob.AssetURL, *testJob.Regex) { + if !strings.HasSuffix(toJob.AssetURL, testJob.Regex) { continue } toJobTime := toJob.StartTime.Round(time.Minute) testJobTime := testJob.StartTime.Round(time.Minute) if !toJobTime.Equal(testJobTime) { - t.Errorf("test job ds %v regex %s start time expected '%+v' actual '%+v'", *testJob.DeliveryService, *testJob.Regex, testJobTime, toJobTime) + t.Errorf("test job ds %v regex %s start time expected '%+v' actual '%+v'", testJob.DeliveryService, testJob.Regex, testJobTime, toJobTime) continue } found = true break } if !found { - t.Errorf("test job ds %v regex %s expected: exists, actual: not found", *testJob.DeliveryService, *testJob.Regex) + t.Errorf("test job ds %v regex %s expected: exists, actual: not found", testJob.DeliveryService, testJob.Regex) } } } @@ -354,10 +329,10 @@ func GetTestInvalidationJobs(t *testing.T) { for _, testJob := range testData.InvalidationJobs { found := false for _, toJob := range jobs.Response { - if *toJob.DeliveryService != (*testJob.DeliveryService).(string) { + if toJob.DeliveryService != testJob.DeliveryService { continue } - if !strings.HasSuffix(*toJob.AssetURL, *testJob.Regex) { + if !strings.HasSuffix(toJob.AssetURL, testJob.Regex) { continue } if !toJob.StartTime.Round(time.Minute).Equal(testJob.StartTime.Round(time.Minute)) { @@ -387,12 +362,12 @@ func GetTestJobsByValidData(t *testing.T) { createdBy := jobs.CreatedBy id := jobs.ID dsName := jobs.DeliveryService - keyword := jobs.Keyword + invalidationType := jobs.InvalidationType //Get Jobs by Asset URL - if len(*assetUrl) > 0 { + if len(assetUrl) > 0 { opts := client.NewRequestOptions() - opts.QueryParameters.Set("assetUrl", *assetUrl) + opts.QueryParameters.Set("assetUrl", assetUrl) toJobs, _, _ = TOSession.GetInvalidationJobs(opts) if len(toJobs.Response) < 1 { t.Errorf("Expected atleast one Jobs response for GET Jobs by Asset URL, but found %d ", len(toJobs.Response)) @@ -402,9 +377,9 @@ func GetTestJobsByValidData(t *testing.T) { } //Get Jobs by CreatedBy - if len(*createdBy) > 1 { + if len(createdBy) > 1 { opts := client.NewRequestOptions() - opts.QueryParameters.Set("createdBy", *createdBy) + opts.QueryParameters.Set("createdBy", createdBy) toJobs, _, _ = TOSession.GetInvalidationJobs(opts) if len(toJobs.Response) < 1 { t.Errorf("Expected atleast one Jobs response for GET Jobs by CreatedBy, but found %d ", len(toJobs.Response)) @@ -414,21 +389,21 @@ func GetTestJobsByValidData(t *testing.T) { } //Get Jobs by ID - if id != nil && *id >= 1 { + if id >= 1 { opts := client.NewRequestOptions() - opts.QueryParameters.Set("id", strconv.FormatUint(uint64(*id), 10)) + opts.QueryParameters.Set("id", strconv.FormatUint(uint64(id), 10)) toJobs, _, _ = TOSession.GetInvalidationJobs(opts) if len(toJobs.Response) != 1 { t.Errorf("Expected only one Jobs response for GET Jobs by ID, but found %d ", len(toJobs.Response)) } } else { - t.Errorf("ID Field is empty, so can't test get jobs %d", *id) + t.Errorf("ID Field is empty, so can't test get jobs %d", id) } - //Get Jobs by Keyword - if keyword != nil && len(*keyword) > 1 { + //Get Jobs by Invalidation Type + if len(invalidationType) > 1 { opts := client.NewRequestOptions() - opts.QueryParameters.Set("Keyword", *keyword) + opts.QueryParameters.Set("InvalidationType", invalidationType) toJobs, _, _ = TOSession.GetInvalidationJobs(opts) if len(toJobs.Response) < 1 { t.Errorf("Expected atleast one Jobs response for GET Jobs by keyword, but found %d ", len(toJobs.Response)) @@ -438,9 +413,9 @@ func GetTestJobsByValidData(t *testing.T) { } //Get Delivery Service ID by Name - if dsName != nil && len(*dsName) > 0 { + if len(dsName) > 0 { opts := client.NewRequestOptions() - opts.QueryParameters.Set("xmlId", *dsName) + opts.QueryParameters.Set("xmlId", dsName) toDSes, _, _ := TOSession.GetDeliveryServices(opts) if len(toDSes.Response) > 0 { dsId := toDSes.Response[0].ID @@ -508,8 +483,8 @@ func GetTestJobsByValidData(t *testing.T) { t.Errorf("GET /jobs by maxRevalDurationDays - expected at least 1 job") } for _, j := range maxRevalJobs.Response { - if time.Since((*j.StartTime).Time) > time.Duration(maxRevalDurationDays)*24*time.Hour { - t.Errorf("GET /jobs by maxRevalDurationDays returned job that is older than %d days: {%s, %s, %v}", maxRevalDurationDays, *j.DeliveryService, *j.AssetURL, *j.StartTime) + if time.Since(j.StartTime) > time.Duration(maxRevalDurationDays)*24*time.Hour { + t.Errorf("GET /jobs by maxRevalDurationDays returned job that is older than %d days: {%s, %s, %v}", maxRevalDurationDays, j.DeliveryService, j.AssetURL, j.StartTime) } } @@ -534,8 +509,8 @@ func GetTestJobsByValidData(t *testing.T) { t.Errorf("GET /jobs by cdn - expected at least 1 job") } for _, j := range cdnJobs.Response { - if dsToCDN[*j.DeliveryService] != cdn { - t.Errorf("GET /jobs by cdn returned job that does not belong to CDN %s: {%s, %s, %v}", cdn, *j.DeliveryService, *j.AssetURL, *j.StartTime) + if dsToCDN[j.DeliveryService] != cdn { + t.Errorf("GET /jobs by cdn returned job that does not belong to CDN %s: {%s, %s, %v}", cdn, j.DeliveryService, j.AssetURL, j.StartTime) } } } @@ -566,9 +541,9 @@ func GetTestJobsByInvalidData(t *testing.T) { t.Errorf("Expected no response from Get Jobs by Invalid ID, but found %d ", len(toJobs.Response)) } - //Get Jobs by Invalid Keyword + //Get Jobs by Invalid Invalidation Type opts = client.NewRequestOptions() - opts.QueryParameters.Set("keyword", "invalid") + opts.QueryParameters.Set("invalidationType", "invalid") toJobs, _, _ = TOSession.GetInvalidationJobs(opts) if len(toJobs.Response) != 0 { t.Errorf("Expected no response from Get Jobs by Invalid Keyword, but found %d ", len(toJobs.Response)) @@ -604,32 +579,31 @@ func CreateTestJobsInvalidDS(t *testing.T) { t.Error("Need at least one Invalidation Jobs to test invalid ds") } job := testData.InvalidationJobs[0] - job.StartTime = &tc.Time{ - Time: time.Now().Add(time.Minute).UTC(), - Valid: true, - } + job.StartTime = time.Now().Add(time.Minute).UTC() testData.InvalidationJobs[0] = job - //Invalid DS - request := tc.InvalidationJobInput{ - DeliveryService: util.InterfacePtr("invalid"), - Regex: job.Regex, - StartTime: job.StartTime, - TTL: job.TTL, + //Invalid DS XML ID (Does not exist) + request := tc.InvalidationJobCreateV4{ + DeliveryService: "invalid", + Regex: job.Regex, + StartTime: job.StartTime, + TTLHours: job.TTLHours, + InvalidationType: job.InvalidationType, } resp, reqInf, err := TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err == nil { t.Errorf("Expected No DeliveryService exists matching identifier: %v - alerts: %v", request.DeliveryService, resp.Alerts) } - if reqInf.StatusCode != http.StatusBadRequest { - t.Errorf("Expected status code 400, got %v", reqInf.StatusCode) + if reqInf.StatusCode != http.StatusNotFound { + t.Errorf("Expected status code 404, got %v", reqInf.StatusCode) } //Missing DS - request = tc.InvalidationJobInput{ - Regex: job.Regex, - StartTime: job.StartTime, - TTL: job.TTL, + request = tc.InvalidationJobCreateV4{ + Regex: job.Regex, + StartTime: job.StartTime, + TTLHours: job.TTLHours, + InvalidationType: job.InvalidationType, } resp, reqInf, err = TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err == nil { @@ -640,11 +614,12 @@ func CreateTestJobsInvalidDS(t *testing.T) { } //Empty DS - request = tc.InvalidationJobInput{ - DeliveryService: util.InterfacePtr(""), - Regex: job.Regex, - StartTime: job.StartTime, - TTL: job.TTL, + request = tc.InvalidationJobCreateV4{ + DeliveryService: "", + Regex: job.Regex, + StartTime: job.StartTime, + TTLHours: job.TTLHours, + InvalidationType: job.InvalidationType, } resp, reqInf, err = TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err == nil { @@ -660,17 +635,15 @@ func CreateTestJobsAlreadyExistTTL(t *testing.T) { t.Error("Need at least one Invalidation Jobs to create duplicate data") } job := testData.InvalidationJobs[0] - job.StartTime = &tc.Time{ - Time: time.Now().Add(time.Minute).UTC(), - Valid: true, - } + job.StartTime = time.Now().Add(time.Minute).UTC() testData.InvalidationJobs[0] = job - request := tc.InvalidationJobInput{ - DeliveryService: job.DeliveryService, - Regex: job.Regex, - StartTime: job.StartTime, - TTL: job.TTL, + request := tc.InvalidationJobCreateV4{ + DeliveryService: job.DeliveryService, + Regex: job.Regex, + StartTime: job.StartTime, + TTLHours: job.TTLHours, + InvalidationType: job.InvalidationType, } resp, _, err := TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err != nil { @@ -686,16 +659,14 @@ func CreateTestJobsWithPastDate(t *testing.T) { dt := time.Now() dt.Format("2019-06-18 21:28:31") job := testData.InvalidationJobs[0] - job.StartTime = &tc.Time{ - Time: dt.AddDate(0, 0, -1), - Valid: true, - } + job.StartTime = dt.AddDate(0, 0, -1) testData.InvalidationJobs[0] = job - request := tc.InvalidationJobInput{ - DeliveryService: job.DeliveryService, - Regex: job.Regex, - StartTime: job.StartTime, - TTL: job.TTL, + request := tc.InvalidationJobCreateV4{ + DeliveryService: job.DeliveryService, + Regex: job.Regex, + StartTime: job.StartTime, + TTLHours: job.TTLHours, + InvalidationType: job.InvalidationType, } resp, reqInf, err := TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err == nil { @@ -709,16 +680,14 @@ func CreateTestJobsWithPastDate(t *testing.T) { dt = time.Now() dt.Format("2019-10-12T07:20:50.52Z") job = testData.InvalidationJobs[0] - job.StartTime = &tc.Time{ - Time: dt.AddDate(0, 0, -1), - Valid: true, - } + job.StartTime = dt.AddDate(0, 0, -1) testData.InvalidationJobs[0] = job - request = tc.InvalidationJobInput{ - DeliveryService: job.DeliveryService, - Regex: job.Regex, - StartTime: job.StartTime, - TTL: job.TTL, + request = tc.InvalidationJobCreateV4{ + DeliveryService: job.DeliveryService, + Regex: job.Regex, + StartTime: job.StartTime, + TTLHours: job.TTLHours, + InvalidationType: job.InvalidationType, } resp, reqInf, err = TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err == nil { @@ -732,16 +701,14 @@ func CreateTestJobsWithPastDate(t *testing.T) { dt = time.Now() dt.Format("2020-03-11 14:12:20-06") job = testData.InvalidationJobs[0] - job.StartTime = &tc.Time{ - Time: dt.AddDate(0, 0, -5), - Valid: true, - } + job.StartTime = dt.AddDate(0, 0, -5) testData.InvalidationJobs[0] = job - request = tc.InvalidationJobInput{ - DeliveryService: job.DeliveryService, - Regex: job.Regex, - StartTime: job.StartTime, - TTL: job.TTL, + request = tc.InvalidationJobCreateV4{ + DeliveryService: job.DeliveryService, + Regex: job.Regex, + StartTime: job.StartTime, + TTLHours: job.TTLHours, + InvalidationType: job.InvalidationType, } resp, reqInf, err = TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err == nil { @@ -753,16 +720,14 @@ func CreateTestJobsWithPastDate(t *testing.T) { //unix standard format past start date job = testData.InvalidationJobs[0] - job.StartTime = &tc.Time{ - Time: time.Unix(1, 0), - Valid: true, - } + job.StartTime = time.Unix(1, 0) testData.InvalidationJobs[0] = job - request = tc.InvalidationJobInput{ - DeliveryService: job.DeliveryService, - Regex: job.Regex, - StartTime: job.StartTime, - TTL: job.TTL, + request = tc.InvalidationJobCreateV4{ + DeliveryService: job.DeliveryService, + Regex: job.Regex, + StartTime: job.StartTime, + TTLHours: job.TTLHours, + InvalidationType: job.InvalidationType, } resp, reqInf, err = TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err == nil { @@ -781,16 +746,14 @@ func CreateTestJobsWithFutureDate(t *testing.T) { dt := time.Now() dt.Format("2019-10-12T07:20:50.52Z") job := testData.InvalidationJobs[0] - job.StartTime = &tc.Time{ - Time: dt.AddDate(0, 0, 1), - Valid: true, - } + job.StartTime = dt.AddDate(0, 0, 1) testData.InvalidationJobs[0] = job - request := tc.InvalidationJobInput{ - DeliveryService: job.DeliveryService, - Regex: job.Regex, - StartTime: job.StartTime, - TTL: job.TTL, + request := tc.InvalidationJobCreateV4{ + DeliveryService: job.DeliveryService, + Regex: job.Regex, + StartTime: job.StartTime, + TTLHours: job.TTLHours, + InvalidationType: job.InvalidationType, } resp, reqInf, err := TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err != nil { @@ -804,16 +767,14 @@ func CreateTestJobsWithFutureDate(t *testing.T) { dt = time.Now() dt.Format("2020-03-11 14:12:20-06") job = testData.InvalidationJobs[0] - job.StartTime = &tc.Time{ - Time: dt.AddDate(0, 0, 1), - Valid: true, - } + job.StartTime = dt.AddDate(0, 0, 1) testData.InvalidationJobs[0] = job - request = tc.InvalidationJobInput{ - DeliveryService: job.DeliveryService, - Regex: job.Regex, - StartTime: job.StartTime, - TTL: job.TTL, + request = tc.InvalidationJobCreateV4{ + DeliveryService: job.DeliveryService, + Regex: job.Regex, + StartTime: job.StartTime, + TTLHours: job.TTLHours, + InvalidationType: job.InvalidationType, } resp, reqInf, err = TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err != nil { @@ -827,16 +788,14 @@ func CreateTestJobsWithFutureDate(t *testing.T) { dt = time.Now() dt.Format(".000") job = testData.InvalidationJobs[0] - job.StartTime = &tc.Time{ - Time: dt.AddDate(0, 0, 1), - Valid: true, - } + job.StartTime = dt.AddDate(0, 0, 1) testData.InvalidationJobs[0] = job - request = tc.InvalidationJobInput{ - DeliveryService: job.DeliveryService, - Regex: job.Regex, - StartTime: job.StartTime, - TTL: job.TTL, + request = tc.InvalidationJobCreateV4{ + DeliveryService: job.DeliveryService, + Regex: job.Regex, + StartTime: job.StartTime, + TTLHours: job.TTLHours, + InvalidationType: job.InvalidationType, } resp, reqInf, err = TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err != nil { @@ -851,12 +810,13 @@ func CreateJobsMissingDate(t *testing.T) { if len(testData.InvalidationJobs) < 1 { t.Fatal("Need at least one Invalidation Job to test creating an invalid Job") } - //Missing date + //Missing start date job := testData.InvalidationJobs[0] - request := tc.InvalidationJobInput{ - DeliveryService: job.DeliveryService, - Regex: job.Regex, - TTL: job.TTL, + request := tc.InvalidationJobCreateV4{ + DeliveryService: job.DeliveryService, + Regex: job.Regex, + TTLHours: job.TTLHours, + InvalidationType: job.InvalidationType, } resp, reqInf, err := TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err == nil { @@ -876,15 +836,12 @@ func CreateJobsMissingRegex(t *testing.T) { dt := time.Now() dt.Format("2019-10-12T07:20:50.52Z") job := testData.InvalidationJobs[0] - job.StartTime = &tc.Time{ - Time: dt.AddDate(0, 0, 1), - Valid: true, - } + job.StartTime = dt.AddDate(0, 0, 1) testData.InvalidationJobs[0] = job - request := tc.InvalidationJobInput{ - DeliveryService: job.DeliveryService, - TTL: job.TTL, - StartTime: job.StartTime, + request := tc.InvalidationJobCreateV4{ + DeliveryService: job.DeliveryService, + TTLHours: job.TTLHours, + InvalidationType: job.InvalidationType, } resp, reqInf, err := TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err == nil { @@ -895,11 +852,11 @@ func CreateJobsMissingRegex(t *testing.T) { } //Empty Regex - job.Regex = nil - request = tc.InvalidationJobInput{ + job.Regex = "" + request = tc.InvalidationJobCreateV4{ DeliveryService: job.DeliveryService, Regex: job.Regex, - TTL: job.TTL, + TTLHours: job.TTLHours, StartTime: job.StartTime, } resp, _, err = TOSession.CreateInvalidationJob(request, client.RequestOptions{}) @@ -920,15 +877,12 @@ func CreateJobsMissingTtl(t *testing.T) { dt := time.Now() dt.Format("2019-10-12T07:20:50.52Z") job := testData.InvalidationJobs[0] - job.StartTime = &tc.Time{ - Time: dt.AddDate(0, 0, 1), - Valid: true, - } + job.StartTime = dt.AddDate(0, 0, 1) testData.InvalidationJobs[0] = job - request := tc.InvalidationJobInput{ - DeliveryService: job.DeliveryService, - Regex: job.Regex, - StartTime: job.StartTime, + request := tc.InvalidationJobCreateV4{ + DeliveryService: job.DeliveryService, + Regex: job.Regex, + InvalidationType: job.InvalidationType, } resp, reqInf, err := TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err == nil { @@ -938,13 +892,14 @@ func CreateJobsMissingTtl(t *testing.T) { t.Errorf("Expected status code 400, got %v", reqInf.StatusCode) } - //Empty TTL - job.TTL = nil - request = tc.InvalidationJobInput{ - DeliveryService: job.DeliveryService, - Regex: job.Regex, - TTL: job.TTL, - StartTime: job.StartTime, + //Invalid TTL + job.TTLHours = 0 + request = tc.InvalidationJobCreateV4{ + DeliveryService: job.DeliveryService, + Regex: job.Regex, + TTLHours: job.TTLHours, + StartTime: job.StartTime, + InvalidationType: job.InvalidationType, } resp, _, err = TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err == nil { @@ -963,15 +918,13 @@ func UpdateTestJobsInvalidDS(t *testing.T) { t.Fatal("Found a Delivery Service in the testing data with null or undefined XMLID") } xmlID := *testData.DeliveryServices[0].XMLID - startTime := tc.Time{ - Time: time.Now().Add(time.Hour), - Valid: true, - } - firstJob := tc.InvalidationJobInput{ - DeliveryService: util.InterfacePtr(&xmlID), - Regex: util.StrPtr(`/\.*([A-Z]0?)`), - TTL: util.InterfacePtr(16), - StartTime: &startTime, + startTime := time.Now().Add(time.Hour) + firstJob := tc.InvalidationJobCreateV4{ + DeliveryService: xmlID, + Regex: `/\.*([A-Z]0?)`, + TTLHours: 16, + StartTime: startTime, + InvalidationType: tc.REFRESH, } resp, _, err := TOSession.CreateInvalidationJob(firstJob, client.RequestOptions{}) @@ -985,30 +938,23 @@ func UpdateTestJobsInvalidDS(t *testing.T) { t.Fatalf("unable to get invalidation jobs: %v - alerts: %+v", err, jobs.Alerts) } - var realJob tc.InvalidationJob + var realJob tc.InvalidationJobV4 for i, job := range jobs.Response { - if job.StartTime == nil || job.DeliveryService == nil || job.CreatedBy == nil { - t.Error("Traffic Ops returned a representation of a content invalidation Job that had null or undefined Start Time and/or Delivery Service and/or Created By") - continue - } - diff := firstJob.StartTime.Time.Sub(job.StartTime.Time) - if *job.DeliveryService == xmlID && *job.CreatedBy == "admin" && diff < time.Second { + diff := firstJob.StartTime.Sub(job.StartTime) + if job.DeliveryService == xmlID && job.CreatedBy == "admin" && diff < time.Second { realJob = jobs.Response[i] break } } - if realJob.ID == nil || *realJob.ID == 0 { + if realJob.ID == 0 { t.Fatal("could not find new job") } //update existing jobs with new ds id originalJob := realJob - newTime := tc.Time{ - Time: startTime.Time.Add(time.Hour * 2), - Valid: true, - } - originalJob.StartTime = &newTime - originalJob.DeliveryService = testData.DeliveryServices[1].XMLID + newTime := startTime.Add(time.Hour * 2) + originalJob.StartTime = newTime + originalJob.DeliveryService = *testData.DeliveryServices[1].XMLID alerts, reqInf, err := TOSession.UpdateInvalidationJob(originalJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected Cannot change 'deliveryService' of existing invalidation job! - alerts: %+v", alerts.Alerts) @@ -1020,7 +966,7 @@ func UpdateTestJobsInvalidDS(t *testing.T) { //update existing jobs with invalid ds id invalidDsIdJob := realJob invalidDsId := "abcd" - invalidDsIdJob.DeliveryService = &invalidDsId + invalidDsIdJob.DeliveryService = invalidDsId alerts, reqInf, err = TOSession.UpdateInvalidationJob(invalidDsIdJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected Cannot change 'deliveryService' of existing invalidation job! - alerts: %+v", alerts.Alerts) @@ -1032,7 +978,7 @@ func UpdateTestJobsInvalidDS(t *testing.T) { //update existing jobs with blank ds id blankDsIdJob := realJob blankDsId := "" - blankDsIdJob.DeliveryService = &blankDsId + blankDsIdJob.DeliveryService = blankDsId alerts, reqInf, err = TOSession.UpdateInvalidationJob(blankDsIdJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected deliveryService: cannot be blank. - alerts: %+v", alerts.Alerts) @@ -1044,7 +990,7 @@ func UpdateTestJobsInvalidDS(t *testing.T) { //update existing jobs with asset url not starts with origin.infra invalidAssetURLJob := realJob assetURL := "http://google.com" - invalidAssetURLJob.AssetURL = &assetURL + invalidAssetURLJob.AssetURL = assetURL alerts, reqInf, err = TOSession.UpdateInvalidationJob(invalidAssetURLJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected Cannot set asset URL that does not start with Delivery Service origin URL: http://origin.infra.ciab.test. - alerts: %+v", alerts.Alerts) @@ -1056,7 +1002,7 @@ func UpdateTestJobsInvalidDS(t *testing.T) { //update existing jobs with blank asset url blankAssetURLJob := realJob assetURL = "" - blankAssetURLJob.AssetURL = &assetURL + blankAssetURLJob.AssetURL = assetURL alerts, reqInf, err = TOSession.UpdateInvalidationJob(blankAssetURLJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected assetUrl: cannot be blank. alerts: %+v", alerts.Alerts) @@ -1068,7 +1014,7 @@ func UpdateTestJobsInvalidDS(t *testing.T) { //update existing jobs with blank created by blankCreatedByJob := realJob createdBy := "" - blankCreatedByJob.CreatedBy = &createdBy + blankCreatedByJob.CreatedBy = createdBy alerts, reqInf, err = TOSession.UpdateInvalidationJob(blankCreatedByJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected createdBy: cannot be blank. alerts: %+v", alerts.Alerts) @@ -1080,7 +1026,7 @@ func UpdateTestJobsInvalidDS(t *testing.T) { //update existing jobs created by createdByJob := realJob createdBy = "operator" - createdByJob.CreatedBy = &createdBy + createdByJob.CreatedBy = createdBy alerts, reqInf, err = TOSession.UpdateInvalidationJob(createdByJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected Cannot change 'createdBy' of existing invalidation jobs!. alerts: %+v", alerts.Alerts) @@ -1089,11 +1035,11 @@ func UpdateTestJobsInvalidDS(t *testing.T) { t.Errorf("Expected status code 409, got %v", reqInf.StatusCode) } - //update existing jobs with blank parameters - blankParametersJob := realJob - parameters := "" - blankParametersJob.Parameters = ¶meters - alerts, reqInf, err = TOSession.UpdateInvalidationJob(blankParametersJob, client.RequestOptions{}) + //update existing jobs with blank invalidation types + blankInvalidationTypeJob := realJob + invalidationType := "" + blankInvalidationTypeJob.InvalidationType = invalidationType + alerts, reqInf, err = TOSession.UpdateInvalidationJob(blankInvalidationTypeJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected parameters: cannot be blank. alerts: %+v", alerts.Alerts) } @@ -1105,10 +1051,8 @@ func UpdateTestJobsInvalidDS(t *testing.T) { startDateFutureJob := realJob dt := time.Now() dt.Format("2019-10-12T07:20:50.52Z") - startDateFutureJob.StartTime = &tc.Time{ - Time: dt.AddDate(0, 0, 3), - Valid: true, - } + addThreeDays := dt.AddDate(0, 0, 3) + startDateFutureJob.StartTime = addThreeDays alerts, reqInf, err = TOSession.UpdateInvalidationJob(startDateFutureJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected startTime: must be within two days from now. alerts: %+v", alerts.Alerts) @@ -1121,10 +1065,8 @@ func UpdateTestJobsInvalidDS(t *testing.T) { pastStartDateJob := realJob dt = time.Now() dt.Format("2019-06-18 21:28:31") - pastStartDateJob.StartTime = &tc.Time{ - Time: dt.AddDate(0, 0, -3), - Valid: true, - } + minusThreeDays := dt.AddDate(0, 0, -3) + pastStartDateJob.StartTime = minusThreeDays alerts, reqInf, err = TOSession.UpdateInvalidationJob(pastStartDateJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected startTime: cannot be in the past. alerts: %+v", alerts.Alerts) @@ -1137,10 +1079,8 @@ func UpdateTestJobsInvalidDS(t *testing.T) { pastStartDateJob = realJob dt = time.Now() dt.Format("2019-10-12T07:20:50.52Z") - pastStartDateJob.StartTime = &tc.Time{ - Time: dt.AddDate(0, 0, -1), - Valid: true, - } + minusOneDay := dt.AddDate(0, 0, -1) + pastStartDateJob.StartTime = minusOneDay alerts, reqInf, err = TOSession.UpdateInvalidationJob(pastStartDateJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected startTime: cannot be in the past. alerts: %+v", alerts.Alerts) @@ -1151,10 +1091,8 @@ func UpdateTestJobsInvalidDS(t *testing.T) { //update jobs with UNIX Format past start date pastStartDateJob = realJob - pastStartDateJob.StartTime = &tc.Time{ - Time: time.Unix(1, 0), - Valid: true, - } + unixTimeFormat := time.Unix(1, 0) + pastStartDateJob.StartTime = unixTimeFormat alerts, reqInf, err = TOSession.UpdateInvalidationJob(pastStartDateJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected startTime: cannot be in the past. alerts: %+v", alerts.Alerts) @@ -1167,10 +1105,8 @@ func UpdateTestJobsInvalidDS(t *testing.T) { pastStartDateJob = realJob dt = time.Now() dt.Format("2020-03-11 14:12:20-06") - pastStartDateJob.StartTime = &tc.Time{ - Time: dt.AddDate(0, 0, -1), - Valid: true, - } + oneLessDay := dt.AddDate(0, 0, -1) + pastStartDateJob.StartTime = oneLessDay alerts, reqInf, err = TOSession.UpdateInvalidationJob(pastStartDateJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected startTime: cannot be in the past. alerts: %+v", alerts.Alerts) @@ -1183,10 +1119,8 @@ func UpdateTestJobsInvalidDS(t *testing.T) { startDateFutureJob = realJob dt = time.Now() dt.Format("2019-10-12T07:20:50.52Z") - startDateFutureJob.StartTime = &tc.Time{ - Time: dt.AddDate(0, 0, 1), - Valid: true, - } + oneMoreDay := dt.AddDate(0, 0, 1) + startDateFutureJob.StartTime = oneMoreDay alerts, reqInf, err = TOSession.UpdateInvalidationJob(startDateFutureJob, client.RequestOptions{}) if err != nil { t.Fatalf("Expected Content invalidation job updated. alerts: %+v", alerts.Alerts) @@ -1199,10 +1133,8 @@ func UpdateTestJobsInvalidDS(t *testing.T) { startDateFutureJob = realJob dt = time.Now() dt.Format(".000") - startDateFutureJob.StartTime = &tc.Time{ - Time: dt.AddDate(0, 0, 1), - Valid: true, - } + andAnotherDay := dt.AddDate(0, 0, 1) + startDateFutureJob.StartTime = andAnotherDay alerts, reqInf, err = TOSession.UpdateInvalidationJob(startDateFutureJob, client.RequestOptions{}) if err != nil { t.Fatalf("Expected Content invalidation job updated. alerts: %+v", alerts.Alerts) @@ -1215,10 +1147,8 @@ func UpdateTestJobsInvalidDS(t *testing.T) { startDateFutureJob = realJob dt = time.Now() dt.Format("2020-03-11 14:12:20-06") - startDateFutureJob.StartTime = &tc.Time{ - Time: dt.AddDate(0, 0, 1), - Valid: true, - } + addEvenMore := dt.AddDate(0, 0, 1) + startDateFutureJob.StartTime = addEvenMore alerts, reqInf, err = TOSession.UpdateInvalidationJob(startDateFutureJob, client.RequestOptions{}) if err != nil { t.Fatalf("Expected Content invalidation job updated. alerts: %+v", alerts.Alerts) @@ -1231,7 +1161,7 @@ func UpdateTestJobsInvalidDS(t *testing.T) { newIdJob := realJob var b uint64 = 1111 var a *uint64 = &b - newIdJob.ID = a + newIdJob.ID = *a alerts, reqInf, err = TOSession.UpdateInvalidationJob(newIdJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected Cannot change an invalidation job 'id'! - alerts: %+v", alerts.Alerts) @@ -1255,7 +1185,7 @@ func DeleteTestJobs(t *testing.T) { id := jobs.ID //Delete Jobs by valid id - alerts, reqInf, err := TOSession.DeleteInvalidationJob(uint64(*id), client.RequestOptions{}) + alerts, reqInf, err := TOSession.DeleteInvalidationJob(uint64(id), client.RequestOptions{}) if err != nil { t.Errorf("Expected Content invalidation job was deleted Error - %v, Alerts %v", err, alerts.Alerts) } @@ -1277,3 +1207,100 @@ func DeleteTestJobsByInvalidId(t *testing.T) { t.Errorf("Expected status code 404, got %v", reqInf.StatusCode) } } + +func CreateRefetchJobParameterFail(t *testing.T) { + + // Ensure clean slate for parameters + err := clearRefetchEnabledParameter() + if err != nil { + t.Fatal(err) + } + + // Attempt to create Refetch job w/o refetch_enabled + job := testData.InvalidationJobsRefetch[0] + createJob := tc.InvalidationJobCreateV4{ + DeliveryService: job.DeliveryService, + Regex: job.Regex, + TTLHours: job.TTLHours, + StartTime: time.Now().Add(time.Hour).UTC(), + InvalidationType: job.InvalidationType, + } + + _, _, err = TOSession.CreateInvalidationJob(createJob, client.RequestOptions{}) + if err == nil { + t.Fatalf("expected error preventing the creation of the Refetch Invalidation Job.") + } + +} + +func CreateRefetchJobParameterSuccess(t *testing.T) { + + // Ensure clean slate for parameters + err := clearRefetchEnabledParameter() + if err != nil { + t.Fatal(err) + } + + // Create refetch_enabled parameter + param := tc.Parameter{ + ConfigFile: string(tc.GlobalConfigFileName), + Name: string(tc.RefetchEnabled), + Secure: false, + Value: "true", + } + + paramsResp, _, err := TOSession.CreateParameter(param, client.RequestOptions{}) + if err != nil { + t.Fatalf("error creating RefetchEnabled parameter. err: %v \n alerts: %v", err, paramsResp.Alerts) + } + + // Create Refetch jobs + for _, job := range testData.InvalidationJobsRefetch { + createJob := tc.InvalidationJobCreateV4{ + DeliveryService: job.DeliveryService, + Regex: job.Regex, + TTLHours: job.TTLHours, + StartTime: time.Now().Add(time.Hour).UTC(), + InvalidationType: job.InvalidationType, + } + + createResp, _, err := TOSession.CreateInvalidationJob(createJob, client.RequestOptions{}) + if err != nil { + t.Fatalf("error posting Refetch Invalidation Job. err: %v \n alerts: %v", err, createResp.Alerts) + } + } + + // Get all jobs + jobsResp, _, err := TOSession.GetInvalidationJobs(client.RequestOptions{}) + if err != nil { + t.Fatalf("error requesting Invalidation Jobs. err: %v \n alerts: %v", err, jobsResp.Alerts) + } + + // Ensure expected created refetch jobs matches actual + var refetchJobs int + for _, job := range jobsResp.Response { + if job.InvalidationType == tc.REFETCH { + refetchJobs++ + } + } + + if refetchJobs != len(testData.InvalidationJobsRefetch) { + t.Fatalf("failed to verify creation of Refetch Invalidation Jobs. Refetch Job count: %v Expected job count: %v", refetchJobs, len(testData.InvalidationJobsRefetch)) + } + +} + +func clearRefetchEnabledParameter() error { + // Ensure Parameter is not set + paramsResp, _, err := TOSession.GetParameters(client.RequestOptions{}) + if err != nil { + return fmt.Errorf("error retrieving parameters. err: %v \n alerts: %v", err, paramsResp.Alerts) + } + + for _, param := range paramsResp.Response { + if param.Name == string(tc.RefetchEnabled) { + TOSession.DeleteParameter(param.ID, client.RequestOptions{}) + } + } + return nil +} diff --git a/traffic_ops/testing/api/v4/tc-fixtures.json b/traffic_ops/testing/api/v4/tc-fixtures.json index aefe7a71d1..9d04c57ded 100644 --- a/traffic_ops/testing/api/v4/tc-fixtures.json +++ b/traffic_ops/testing/api/v4/tc-fixtures.json @@ -5678,44 +5678,67 @@ { "deliveryService": "ds1", "regex": "/.*", - "startTime": 4117118271000, - "ttl": "121m" + "startTime": "2100-06-19T13:57:51Z", + "ttlHours": 72, + "invalidationType" : "REFRESH" }, { "deliveryService": "ds1", "regex": "/foo", - "startTime": 4117118271000, - "ttl": 2160 + "startTime": "2100-06-19T13:57:51Z", + "ttlHours": 36, + "invalidationType" : "REFRESH" }, { "deliveryService": "ds2", "regex": "\\/some-path?.+\\.jpg", - "startTime": "2100-06-19T13:57:51-06:00", - "ttl": 2.1 + "startTime": "2100-06-19T13:57:51Z", + "ttlHours": 144, + "invalidationType" : "REFRESH" }, { - "deliveryService": "ds1", + "deliveryService": "ds1", "regex": "/oldest", - "startTime": 1000, - "ttl": 2160 + "startTime": "1970-01-01T12:00:00Z", + "ttlHours": 2160, + "invalidationType" : "REFRESH" }, { - "deliveryService": "ds1", + "deliveryService": "ds1", "regex": "/older", - "startTime": 1001, - "ttl": 2160 + "startTime": "1970-01-01T12:00:00Z", + "ttlHours": 2160, + "invalidationType" : "REFRESH" }, { - "deliveryService": "ds1", + "deliveryService": "ds1", "regex": "/old", - "startTime": 1002, - "ttl": 2160 + "startTime": "1970-01-01T12:00:00Z", + "ttlHours": 2160, + "invalidationType" : "REFRESH" }, { - "deliveryService": "ds-forked-topology", + "deliveryService": "ds-forked-topology", "regex": "/this-ds-is-in-cdn2", - "startTime": 4117118271000, - "ttl": 2160 + "startTime": "2100-06-19T13:57:51Z", + "ttlHours": 2160, + "invalidationType" : "REFRESH" + } + ], + "invalidationJobsRefetch": [ + { + "deliveryService": "ds1", + "regex": "/.*", + "startTime": "2100-06-19T13:57:51Z", + "ttlHours": 72, + "invalidationType" : "REFETCH" + }, + { + "deliveryService": "ds1", + "regex": "/foo", + "startTime": "2100-06-19T13:57:51Z", + "ttlHours": 36, + "invalidationType" : "REFETCH" } ], "statsSummaries": [ diff --git a/traffic_ops/testing/api/v4/todb_test.go b/traffic_ops/testing/api/v4/todb_test.go index 7e4ccdd27d..83f34d41a4 100644 --- a/traffic_ops/testing/api/v4/todb_test.go +++ b/traffic_ops/testing/api/v4/todb_test.go @@ -80,18 +80,6 @@ func SetupTestData(*sql.DB) error { os.Exit(1) } - err = SetupJobAgents(db) - if err != nil { - fmt.Printf("\nError setting up job agents %s - %s, %v\n", Config.TrafficOps.URL, Config.TrafficOps.Users.Admin, err) - os.Exit(1) - } - - err = SetupJobStatuses(db) - if err != nil { - fmt.Printf("\nError setting up job agents %s - %s, %v\n", Config.TrafficOps.URL, Config.TrafficOps.Users.Admin, err) - os.Exit(1) - } - err = SetupTypes(db) if err != nil { fmt.Printf("\nError setting up types %s - %s, %v\n", Config.TrafficOps.URL, Config.TrafficOps.Users.Admin, err) @@ -361,39 +349,13 @@ INSERT INTO deliveryservice_tmuser (deliveryservice, tm_user_id, last_updated) V return nil } -// SetupJobStatuses ... -func SetupJobStatuses(db *sql.DB) error { - - sqlStmt := ` -INSERT INTO job_status (id, name, description, last_updated) VALUES (1, 'PENDING', 'Job is queued, but has not been picked up by any agents yet', '2018-01-19 21:19:32.444857'); -` - err := execSQL(db, sqlStmt) - if err != nil { - return fmt.Errorf("exec failed %v", err) - } - return nil -} - -// SetupJobAgents ... -func SetupJobAgents(db *sql.DB) error { - - sqlStmt := ` -INSERT INTO job_agent (id, name, description, active, last_updated) VALUES (1, 'agent1', 'Test Agent1', 0, '2018-01-19 21:19:32.448076'); -` - err := execSQL(db, sqlStmt) - if err != nil { - return fmt.Errorf("exec failed %v", err) - } - return nil -} - // SetupJobs ... func SetupJobs(db *sql.DB) error { sqlStmt := ` -INSERT INTO job (id, agent, object_type, object_name, keyword, parameters, asset_url, asset_type, status, start_time, entered_time, job_user, last_updated, job_deliveryservice) VALUES (100, 1, null, null, 'PURGE', 'TTL:48h', 'http://cdn2.edge/job1/.*', 'file', 1, '2018-01-19 21:01:14.000000', '2018-01-19 21:01:14.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.468643', 100); -INSERT INTO job (id, agent, object_type, object_name, keyword, parameters, asset_url, asset_type, status, start_time, entered_time, job_user, last_updated, job_deliveryservice) VALUES (200, 1, null, null, 'PURGE', 'TTL:48h', 'http://cdn2.edge/job2/.*', 'file', 1, '2018-01-19 21:09:34.000000', '2018-01-19 21:09:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.450915', 200); -INSERT INTO job (id, agent, object_type, object_name, keyword, parameters, asset_url, asset_type, status, start_time, entered_time, job_user, last_updated, job_deliveryservice) VALUES (300, 1, null, null, 'PURGE', 'TTL:48h', 'http://cdn2.edge/job3/.*', 'file', 1, '2018-01-19 21:14:34.000000', '2018-01-19 21:14:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.460870', 100); +INSERT INTO job (id, ttl_hr, asset_url, start_time, entered_time, job_user, last_updated, job_deliveryservice, invalidation_type) VALUES (100, 24, 'http://cdn2.edge/job1/.*', '2018-01-19 21:01:14.000000', '2018-01-19 21:01:14.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.468643', 100, 'REFRESH'); +INSERT INTO job (id, ttl_hr, asset_url, start_time, entered_time, job_user, last_updated, job_deliveryservice, invalidation_type) VALUES (200, 36, 'http://cdn2.edge/job2/.*', '2018-01-19 21:09:34.000000', '2018-01-19 21:09:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.450915', 200, 'REFETCH'); +INSERT INTO job (id, ttl_hr, asset_url, start_time, entered_time, job_user, last_updated, job_deliveryservice, invalidation_type) VALUES (300, 72, 'http://cdn2.edge/job3/.*', '2018-01-19 21:14:34.000000', '2018-01-19 21:14:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.460870', 100, 'REFRESH'); ` err := execSQL(db, sqlStmt) if err != nil { @@ -442,8 +404,6 @@ func Teardown(db *sql.DB) error { DELETE FROM to_extension; DELETE FROM staticdnsentry; DELETE FROM job; - DELETE FROM job_agent; - DELETE FROM job_status; DELETE FROM log; DELETE FROM asn; DELETE FROM deliveryservice_tmuser; diff --git a/traffic_ops/testing/api/v4/traffic_control_test.go b/traffic_ops/testing/api/v4/traffic_control_test.go index 2de4786ae8..e1d68ec69f 100644 --- a/traffic_ops/testing/api/v4/traffic_control_test.go +++ b/traffic_ops/testing/api/v4/traffic_control_test.go @@ -56,5 +56,6 @@ type TrafficControl struct { SteeringTargets []tc.SteeringTargetNullable `json:"steeringTargets"` Serverchecks []tc.ServercheckRequestNullable `json:"serverchecks"` Users []tc.UserV4 `json:"users"` - InvalidationJobs []tc.InvalidationJobInput `json:"invalidationJobs"` + InvalidationJobs []tc.InvalidationJobCreateV4 `json:"invalidationJobs"` + InvalidationJobsRefetch []tc.InvalidationJobCreateV4 `json:"invalidationJobsRefetch"` } diff --git a/traffic_ops/traffic_ops_golang/dbhelpers/db_helpers.go b/traffic_ops/traffic_ops_golang/dbhelpers/db_helpers.go index 0205acffa3..efd1e06057 100644 --- a/traffic_ops/traffic_ops_golang/dbhelpers/db_helpers.go +++ b/traffic_ops/traffic_ops_golang/dbhelpers/db_helpers.go @@ -504,6 +504,18 @@ func GetDSNameFromID(tx *sql.Tx, id int) (tc.DeliveryServiceName, bool, error) { return name, true, nil } +// GetDSNameFromID loads the DeliveryService's xml_id from the database, from the ID. Returns whether the delivery service was found, and any error. +func GetDSIDFromXMLID(tx *sql.Tx, xmlID string) (int, bool, error) { + var id int + if err := tx.QueryRow(`SELECT id FROM deliveryservice WHERE xml_id = $1`, xmlID).Scan(&id); err != nil { + if err == sql.ErrNoRows { + return id, false, nil + } + return id, false, fmt.Errorf("querying ID for delivery service XMLID '%v': %v", xmlID, err) + } + return id, true, nil +} + // GetDSCDNIdFromID loads the DeliveryService's cdn ID from the database, from the delivery service ID. Returns whether the delivery service was found, and any error. func GetDSCDNIdFromID(tx *sql.Tx, dsID int) (int, bool, error) { var cdnID int diff --git a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go index 6124863387..c49b5ad68f 100644 --- a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go +++ b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go @@ -25,14 +25,11 @@ import ( "errors" "fmt" "net/http" + "regexp" "strconv" "strings" "time" - "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/util/ims" - - "github.com/lib/pq" - "github.com/apache/trafficcontrol/lib/go-log" "github.com/apache/trafficcontrol/lib/go-rfc" "github.com/apache/trafficcontrol/lib/go-tc" @@ -40,69 +37,104 @@ import ( "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/api" "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/dbhelpers" "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/tenant" + "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/util/ims" + + validation "github.com/go-ozzo/ozzo-validation" + "github.com/go-ozzo/ozzo-validation/is" + "github.com/lib/pq" ) +// Deprecated, only to be used with versions below 4.0 type InvalidationJob struct { api.APIInfoImpl `json:"-"` tc.InvalidationJob } -const readQuery = ` -SELECT job.id, - keyword, - parameters, - asset_url, - start_time, - u.username AS createdBy, - ds.xml_id AS dsId -FROM job -JOIN tm_user u ON job.job_user = u.id -JOIN deliveryservice ds ON job.job_deliveryservice = ds.id -` +type InvalidationJobV4 struct { + api.APIInfoImpl `json:"-"` + tc.InvalidationJobV4 +} +// Deprecated, only to be used with versions below 4.0 const insertQuery = ` INSERT INTO job ( - agent, - asset_type, + ttl_hr, asset_url, + start_time, entered_time, - job_deliveryservice, job_user, - keyword, - parameters, - start_time, - status) + job_deliveryservice, + invalidation_type) VALUES ( - 1::bigint, - 'file', + $1, ( SELECT o.protocol::text || '://' || o.fqdn || rtrim(concat(':', o.port::text), ':') FROM origin o - WHERE o.deliveryservice = $1 + WHERE o.deliveryservice = $2 AND o.is_primary - ) || $2, - $3, + ) || $3, $4, $5, - 'PURGE', $6, $7, - 1::bigint + $8 ) RETURNING asset_url, - (SELECT deliveryservice.xml_id - FROM deliveryservice - WHERE deliveryservice.id=job_deliveryservice) AS deliveryservice, + ( + SELECT deliveryservice.xml_id + FROM deliveryservice + WHERE deliveryservice.id=job_deliveryservice) AS deliveryservice, id, - (SELECT tm_user.username - FROM tm_user - WHERE tm_user.id=job_user) AS createdBy, - keyword, - parameters, + ( + SELECT tm_user.username + FROM tm_user + WHERE tm_user.id=job_user) AS createdBy, + 'PURGE' AS keyword, + CONCAT('TTL:', ttl_hr, 'h') AS parameters, start_time ` +// Almost the same as insertQuery, but returns appropriate values for API 4.0+ +const insertQueryV4 = ` +INSERT INTO job ( + ttl_hr, + asset_url, + start_time, + entered_time, + job_user, + job_deliveryservice, + invalidation_type) +VALUES ( + $1, + ( + SELECT o.protocol::text || '://' || o.fqdn || rtrim(concat(':', o.port::text), ':') + FROM origin o + WHERE o.deliveryservice = $2 + AND o.is_primary + ) || $3, + $4, + $5, + $6, + $7, + $8 +) +RETURNING + id, + asset_url, + ( + SELECT tm_user.username + FROM tm_user + WHERE tm_user.id=job_user) AS createdBy, + ( + SELECT deliveryservice.xml_id + FROM deliveryservice + WHERE deliveryservice.id=job_deliveryservice) AS deliveryServiceXML, + ttl_hr as ttlHrs, + invalidation_type as invalidationType, + start_time as startTime +` + const revalQuery = ` UPDATE server SET %s=TRUE WHERE server.status NOT IN ( @@ -130,37 +162,62 @@ WHERE server.status NOT IN ( const updateQuery = ` UPDATE job SET asset_url=$1, - keyword=$2, - parameters=$3, - start_time=$4 + ttl_hr=$2, + start_time=$3 +WHERE job.id=$4 +RETURNING asset_url, + ( + SELECT tm_user.username + FROM tm_user + WHERE tm_user.id=job.job_user + ) AS created_by, + ( + SELECT deliveryservice.xml_id + FROM deliveryservice + WHERE deliveryservice.id=job.job_deliveryservice + ) AS delivery_service, + job.id, + 'PURGE' as keyword, + CONCAT('TTL:', ttl_hr, 'h') AS parameters, + start_time +` + +// Almost the same as updateQuery, but returns appropriate values for API 4.0+ +const updateQueryV4 = ` +UPDATE job +SET asset_url=$1, + ttl_hr=$2, + start_time=$3, + invalidation_type=$4 WHERE job.id=$5 -RETURNING job.asset_url, - ( - SELECT tm_user.username - FROM tm_user - WHERE tm_user.id=job.job_user - ) AS created_by, - ( - SELECT deliveryservice.xml_id - FROM deliveryservice - WHERE deliveryservice.id=job.job_deliveryservice - ) AS delivery_service, - job.id, - job.keyword, - job.parameters, - job.start_time +RETURNING asset_url, + ( + SELECT tm_user.username + FROM tm_user + WHERE tm_user.id=job.job_user + ) AS created_by, + ( + SELECT deliveryservice.xml_id + FROM deliveryservice + WHERE deliveryservice.id=job.job_deliveryservice + ) AS delivery_service, + job.id, + ttl_hr, + start_time, + invalidation_type ` +// Deprecated, only to be used with versions below 4.0 const putInfoQuery = ` SELECT job.id AS id, - tm_user.username AS createdBy, - job.job_user AS createdByID, - job.job_deliveryservice AS dsid, - deliveryservice.xml_id AS dsxmlid, - job.asset_url AS assetURL, - job.parameters, - job.start_time AS start_time, - origin.protocol || '://' || origin.fqdn || rtrim(concat(':', origin.port), ':') AS OFQDN + tm_user.username AS createdBy, + job.job_user AS createdByID, + job.job_deliveryservice AS dsid, + deliveryservice.xml_id AS dsxmlid, + job.asset_url AS assetURL, + CONCAT('TTL:', ttl_hr, 'h') AS parameters, + job.start_time AS start_time, + origin.protocol || '://' || origin.fqdn || rtrim(concat(':', origin.port), ':') AS OFQDN FROM job INNER JOIN origin ON origin.deliveryservice=job.job_deliveryservice AND origin.is_primary INNER JOIN tm_user ON tm_user.id=job.job_user @@ -168,25 +225,68 @@ INNER JOIN deliveryservice ON deliveryservice.id=job.job_deliveryservice WHERE job.id=$1 ` +// Almost the same as putInfoQuery, but returns appropriate values for API 4.0+ +const putInfoQueryV4 = ` +SELECT job.id AS id, + tm_user.username AS createdBy, + job.job_user AS createdByID, + job.job_deliveryservice AS dsid, + deliveryservice.xml_id AS dsxmlid, + job.asset_url AS assetURL, + job.ttl_hr AS ttlhrs, + job.start_time AS start_time, + job.invalidation_type as invalidationType, + origin.protocol || '://' || origin.fqdn || rtrim(concat(':', origin.port), ':') AS OFQDN +FROM job +INNER JOIN origin ON origin.deliveryservice=job.job_deliveryservice AND origin.is_primary +INNER JOIN tm_user ON tm_user.id=job.job_user +INNER JOIN deliveryservice ON deliveryservice.id=job.job_deliveryservice +WHERE job.id=$1 +` + +// Deprecated, only to be used with versions below 4.0 const deleteQuery = ` DELETE FROM job WHERE job.id=$1 RETURNING job.asset_url, - ( - SELECT tm_user.username - FROM tm_user - WHERE tm_user.id=job.job_user - ) AS created_by, - ( - SELECT deliveryservice.xml_id - FROM deliveryservice - WHERE deliveryservice.id=job.job_deliveryservice - ) AS deliveryservice, - job.id, - job.keyword, - job.parameters, - job.start_time + ( + SELECT tm_user.username + FROM tm_user + WHERE tm_user.id=job.job_user + ) AS created_by, + ( + SELECT deliveryservice.xml_id + FROM deliveryservice + WHERE deliveryservice.id=job.job_deliveryservice + ) AS deliveryservice, + job.id, + 'PURGE' as keyword, + CONCAT('TTL:', ttl_hr, 'h') AS parameters, + job.start_time +` + +// Almost the same as deleteQuery, but returns appropriate values for API 4.0+ +const deleteQueryV4 = ` +DELETE +FROM job +WHERE job.id=$1 +RETURNING + job.id, + job.asset_url, + ( + SELECT tm_user.username + FROM tm_user + WHERE tm_user.id=job.job_user + ) AS created_by, + ( + SELECT deliveryservice.xml_id + FROM deliveryservice + WHERE deliveryservice.id=job.job_deliveryservice + ) AS deliveryservice, + ttl_hr, + job.invalidation_type, + job.start_time ` type apiResponse struct { @@ -194,6 +294,11 @@ type apiResponse struct { Response tc.InvalidationJob `json:"response,omitempty"` } +type apiResponseV4 struct { + Alerts []tc.Alert `json:"alerts,omitempty"` + Response tc.InvalidationJobV4 `json:"response,omitempty"` +} + func selectMaxLastUpdatedQuery(where string) string { return `SELECT max(t) from ( SELECT max(job.last_updated) as t FROM job @@ -203,17 +308,141 @@ func selectMaxLastUpdatedQuery(where string) string { select max(last_updated) as t from last_deleted l where l.table_name='job') as res` } +// Deprecated, only to be used with versions below 4.0 +const readQuery = ` +SELECT job.id, + 'PURGE' AS keyword, + CONCAT('TTL::', ttl_hr, 'h') AS parameters, + asset_url, + start_time, + u.username as createdBy, + ds.xml_id as dsId +FROM job +JOIN tm_user u ON job.job_user = u.id +JOIN deliveryservice ds ON job.job_deliveryservice = ds.id +` + +// Almost the same as readQuery, but returns appropriate values for API 4.0+ +const readQueryV4 = ` +SELECT job.id, + asset_url, + u.username as createdBy, + ds.xml_id, + ttl_hr, + invalidation_type, + start_time +FROM job +JOIN tm_user u ON job.job_user = u.id +JOIN deliveryservice ds ON job.job_deliveryservice = ds.id +` + // Used by GET requests to `/jobs`, simply returns a filtered list of // content invalidation jobs according to the provided query parameters. +func (job *InvalidationJobV4) Read(h http.Header, useIMS bool) ([]interface{}, error, error, int, *time.Time) { + var maxTime time.Time + var runSecond bool + queryParamsToSQLCols := map[string]dbhelpers.WhereColumnInfo{ + "id": dbhelpers.WhereColumnInfo{Column: "job.id", Checker: api.IsInt}, + "assetUrl": dbhelpers.WhereColumnInfo{Column: "asset_url"}, + "startTime": dbhelpers.WhereColumnInfo{Column: "start_time"}, + "userId": dbhelpers.WhereColumnInfo{Column: "job_user", Checker: api.IsInt}, + "createdBy": dbhelpers.WhereColumnInfo{Column: `(SELECT tm_user.username FROM tm_user WHERE tm_user.id=job.job_user)`}, + "deliveryService": dbhelpers.WhereColumnInfo{Column: `(SELECT deliveryservice.xml_id FROM deliveryservice WHERE deliveryservice.id=job.job_deliveryservice)`}, + "dsId": dbhelpers.WhereColumnInfo{Column: "job.job_deliveryservice", Checker: api.IsInt}, + "invalidationType": dbhelpers.WhereColumnInfo{Column: "invalidation_type"}, + } + + where, orderBy, pagination, queryValues, errs := dbhelpers.BuildWhereAndOrderByAndPagination(job.APIInfo().Params, queryParamsToSQLCols) + if len(errs) > 0 { + return nil, util.JoinErrs(errs), nil, http.StatusBadRequest, nil + } + + accessibleTenants, err := tenant.GetUserTenantIDListTx(job.APIInfo().Tx.Tx, job.APIInfo().User.TenantID) + if err != nil { + return nil, nil, fmt.Errorf("getting accessible tenants for user - %v", err), http.StatusInternalServerError, nil + } + cdn := "" + if cdnName, ok := job.APIInfo().Params["cdn"]; ok { + queryValues["cdn"] = cdnName + cdn = ` AND ds.cdn_id = (SELECT id FROM cdn WHERE name = :cdn) ` + } + maxDays := "" + if _, ok := job.APIInfo().Params["maxRevalDurationDays"]; ok { + // jobs started within the last $maxRevalDurationDays days (defaulting to 90 days if the parameter doesn't exist) + maxDays = ` AND job.start_time >= NOW() - CAST( + (SELECT COALESCE( + (SELECT value + FROM parameter + WHERE name = 'maxRevalDurationDays' + AND config_file = 'regex_revalidate.config' + LIMIT 1), + '90')) + || ' days' AS INTERVAL) ` + } + if len(where) > 0 { + where += " AND ds.tenant_id = ANY(:tenants) " + maxDays + cdn + } else { + where = dbhelpers.BaseWhere + " ds.tenant_id = ANY(:tenants) " + maxDays + cdn + } + queryValues["tenants"] = pq.Array(accessibleTenants) + + if useIMS { + runSecond, maxTime = ims.TryIfModifiedSinceQuery(job.APIInfo().Tx, h, queryValues, selectMaxLastUpdatedQuery(where)) + if !runSecond { + log.Debugln("IMS HIT") + return []interface{}{}, nil, nil, http.StatusNotModified, &maxTime + } + log.Debugln("IMS MISS") + } else { + log.Debugln("Non IMS request") + } + + query := readQueryV4 + where + orderBy + pagination + log.Debugln("generated job query: " + query) + log.Debugf("executing with values: %++v\n", queryValues) + + returnable := []interface{}{} + rows, err := job.APIInfo().Tx.NamedQuery(query, queryValues) + if err != nil { + return nil, nil, fmt.Errorf("querying: %v", err), http.StatusInternalServerError, nil + } + defer rows.Close() + + for rows.Next() { + job := tc.InvalidationJobV4{} + if err := rows.Scan(&job.ID, + &job.AssetURL, + &job.CreatedBy, + &job.DeliveryService, + &job.TTLHours, + &job.InvalidationType, + &job.StartTime); err != nil { + return nil, nil, fmt.Errorf("parsing db response: %v", err), http.StatusInternalServerError, nil + } + + returnable = append(returnable, job) + } + + if err := rows.Err(); err != nil { + return nil, nil, fmt.Errorf("Parsing db responses: %v", err), http.StatusInternalServerError, nil + } + + return returnable, nil, nil, http.StatusOK, &maxTime +} + +// Used by GET requests to `/jobs`, simply returns a filtered list of +// content invalidation jobs according to the provided query parameters. +// +// Deprecated. To be used only with versions less than 4.0 func (job *InvalidationJob) Read(h http.Header, useIMS bool) ([]interface{}, error, error, int, *time.Time) { var maxTime time.Time var runSecond bool queryParamsToSQLCols := map[string]dbhelpers.WhereColumnInfo{ "id": dbhelpers.WhereColumnInfo{Column: "job.id", Checker: api.IsInt}, - "keyword": dbhelpers.WhereColumnInfo{Column: "job.keyword"}, - "assetUrl": dbhelpers.WhereColumnInfo{Column: "job.asset_url"}, - "startTime": dbhelpers.WhereColumnInfo{Column: "job.start_time"}, - "userId": dbhelpers.WhereColumnInfo{Column: "job.job_user", Checker: api.IsInt}, + "keyword": dbhelpers.WhereColumnInfo{Column: "keyword"}, + "assetUrl": dbhelpers.WhereColumnInfo{Column: "asset_url"}, + "startTime": dbhelpers.WhereColumnInfo{Column: "start_time"}, + "userId": dbhelpers.WhereColumnInfo{Column: "job_user", Checker: api.IsInt}, "createdBy": dbhelpers.WhereColumnInfo{Column: `(SELECT tm_user.username FROM tm_user WHERE tm_user.id=job.job_user)`}, "deliveryService": dbhelpers.WhereColumnInfo{Column: `(SELECT deliveryservice.xml_id FROM deliveryservice WHERE deliveryservice.id=job.job_deliveryservice)`}, "dsId": dbhelpers.WhereColumnInfo{Column: "job.job_deliveryservice", Checker: api.IsInt}, @@ -300,6 +529,161 @@ func (job *InvalidationJob) Read(h http.Header, useIMS bool) ([]interface{}, err // Used by POST requests to `/jobs`, creates a new content invalidation job // from the provided request body. +func CreateV40(w http.ResponseWriter, r *http.Request) { + inf, userErr, sysErr, errCode := api.NewInfo(r, nil, nil) + if userErr != nil || sysErr != nil { + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr) + return + } + defer inf.Close() + + job := tc.InvalidationJobCreateV4{} + if err := json.NewDecoder(r.Body).Decode(&job); err != nil { + api.HandleErr(w, r, inf.Tx.Tx, http.StatusBadRequest, errors.New("Unable to parse Invalidation Job"), fmt.Errorf("parsing jobs/ POST: %v", err)) + return + } + + // Check if request object is valid + w.Header().Set(rfc.ContentType, rfc.ApplicationJSON) + if err := validateJobCreateV4(job, inf.Tx.Tx); err != nil { + response := tc.Alerts{ + Alerts: []tc.Alert{ + { + Text: err.Error(), + Level: tc.ErrorLevel.String(), + }, + }, + } + + resp, err := json.Marshal(response) + if err != nil { + api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, fmt.Errorf("encoding bad request response: %v", err)) + return + } + + api.HandleErr(w, r, inf.Tx.Tx, http.StatusBadRequest, errors.New(string(append(resp, '\n'))), fmt.Errorf("error validating job: %v", err)) + return + } + + // Check if authorized + if ok, err := IsUserAuthorizedToModifyDSXMLID(inf, job.DeliveryService); err != nil { + sysErr = fmt.Errorf("failed checking current user permissions for DS #%s: %v", job.DeliveryService, err) + errCode = http.StatusInternalServerError + api.HandleErr(w, r, inf.Tx.Tx, errCode, nil, sysErr) + return + } else if !ok { + userErr = fmt.Errorf("failed to authorize based on tenancy") + errCode = http.StatusNotFound + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) + return + } + + // DS existence was already verified in the Validate() function + dsid, exists, err := dbhelpers.GetDSIDFromXMLID(inf.Tx.Tx, job.DeliveryService) + if err != nil { + sysErr = fmt.Errorf("failed to match XML ID to int ID for Delivery Service %s: %v", job.DeliveryService, err) + errCode = http.StatusInternalServerError + api.HandleErr(w, r, inf.Tx.Tx, errCode, nil, sysErr) + } + if !exists { + userErr = fmt.Errorf("delivery service \"%v\" does not exist", job.DeliveryService) + errCode = http.StatusNotFound + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) + } + + row := inf.Tx.Tx.QueryRow(insertQueryV4, + job.TTLHours, + dsid, // Used in inner select for deliveryservice + job.Regex, + job.StartTime, + time.Now(), + inf.User.ID, + dsid, + job.InvalidationType) // Defaults for all api versions below 4.0 + + result := tc.InvalidationJobV4{} + err = row.Scan( + &result.ID, + &result.AssetURL, + &result.CreatedBy, + &result.DeliveryService, + &result.TTLHours, + &result.InvalidationType, + &result.StartTime) + if err != nil { + userErr, sysErr, errCode = api.ParseDBError(err) + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr) + return + } + + if err := setRevalFlags(uint(dsid), inf.Tx.Tx); err != nil { + api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, fmt.Errorf("setting reval flags: %v", err)) + return + } + + conflicts := tc.ValidateJobUniqueness(inf.Tx.Tx, uint(dsid), result.StartTime, result.AssetURL, result.TTLHours) + response := apiResponseV4{ + make([]tc.Alert, len(conflicts)+1), + result, + } + for i, conflict := range conflicts { + response.Alerts[i] = tc.Alert{ + Text: conflict, + Level: tc.WarnLevel.String(), + } + } + response.Alerts[len(conflicts)] = tc.Alert{ + Text: fmt.Sprintf("Invalidation (%s) request created for %v, start:%v end %v", + result.InvalidationType, + result.AssetURL, + result.StartTime, + result.StartTime.Add(time.Hour*time.Duration(job.TTLHours))), + Level: tc.SuccessLevel.String(), + } + resp, err := json.Marshal(response) + + if err != nil { + api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, fmt.Errorf("Marshaling JSON: %v", err)) + return + } + + if inf.Version == nil { + api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, errors.New("nil API version")) + return + } + + w.Header().Set(http.CanonicalHeaderKey("location"), fmt.Sprintf("%s://%s/api/%d.%d/jobs?id=%d", + inf.Config.URL.Scheme, + r.Host, + inf.Version.Major, + inf.Version.Minor, + result.ID)) + w.WriteHeader(http.StatusOK) + api.WriteAndLogErr(w, r, append(resp, '\n')) + + duplicate := "" + if len(conflicts) > 0 { + duplicate = "(duplicate) " + } + changeLogMsg := fmt.Sprintf("%s content invalidation job %s- ID: %d DSXMLID: %s ASSET_URL: '%s' TTLHRs: %d INVALIDATION: %s", + api.Created, + duplicate, + result.ID, + result.DeliveryService, + result.AssetURL, + result.TTLHours, + result.InvalidationType, + ) + api.CreateChangeLogRawTx(api.ApiChange, + changeLogMsg, + inf.User, + inf.Tx.Tx) +} + +// Used by POST requests to `/jobs`, creates a new content invalidation job +// from the provided request body. +// +// Deprecated. To be used only with versions less than 4.0 func Create(w http.ResponseWriter, r *http.Request) { inf, userErr, sysErr, errCode := api.NewInfo(r, nil, nil) if userErr != nil || sysErr != nil { @@ -336,78 +720,265 @@ func Create(w http.ResponseWriter, r *http.Request) { return } - // Validate() would have already checked for deliveryservice existence and - // parsed the ttl, so if either of these throws an error now, something - // weird has happened - dsid, err := job.DSID(nil) - if err != nil { - sysErr = fmt.Errorf("retrieving parsed DSID: %v", err) + // Validate() would have already checked for deliveryservice existence and + // parsed the ttl, so if either of these throws an error now, something + // weird has happened + dsid, err := job.DSID(nil) + if err != nil { + sysErr = fmt.Errorf("retrieving parsed DSID: %v", err) + errCode = http.StatusInternalServerError + api.HandleErr(w, r, inf.Tx.Tx, errCode, nil, sysErr) + return + } + var ttl uint + if ttl, err = job.TTLHours(); err != nil { + sysErr = fmt.Errorf("retrieving parsed TTL: %v", err) + errCode = http.StatusInternalServerError + api.HandleErr(w, r, inf.Tx.Tx, errCode, nil, sysErr) + return + } + + if ok, err := IsUserAuthorizedToModifyDSID(inf, dsid); err != nil { + sysErr = fmt.Errorf("Checking current user permissions for DS #%d: %v", dsid, err) + errCode = http.StatusInternalServerError + api.HandleErr(w, r, inf.Tx.Tx, errCode, nil, sysErr) + return + } else if !ok { + userErr = fmt.Errorf("No such Delivery Service!") + errCode = http.StatusNotFound + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) + return + } + + _, cdnName, _, err := dbhelpers.GetDSNameAndCDNFromID(inf.Tx.Tx, int(dsid)) + if err != nil { + api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, errors.New("getting delivery service and CDN name from ID: "+err.Error())) + return + } + userErr, sysErr, statusCode := dbhelpers.CheckIfCurrentUserCanModifyCDN(inf.Tx.Tx, string(cdnName), inf.User.UserName) + if userErr != nil || sysErr != nil { + api.HandleErr(w, r, inf.Tx.Tx, statusCode, userErr, sysErr) + return + } + row := inf.Tx.Tx.QueryRow(insertQuery, + ttl, + dsid, // Used in inner select for deliveryservice + *job.Regex, + (*job.StartTime).Time, + time.Now(), + inf.User.ID, + dsid, + tc.REFRESH) // Defaults for all api versions below 4.0 + + result := tc.InvalidationJob{} + err = row.Scan(&result.AssetURL, + &result.DeliveryService, + &result.ID, + &result.CreatedBy, + &result.Keyword, + &result.Parameters, + &result.StartTime) + if err != nil { + userErr, sysErr, errCode = api.ParseDBError(err) + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr) + return + } + + if err := setRevalFlags(dsid, inf.Tx.Tx); err != nil { + api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, fmt.Errorf("setting reval flags: %v", err)) + return + } + + conflicts := tc.ValidateJobUniqueness(inf.Tx.Tx, dsid, job.StartTime.Time, *result.AssetURL, ttl) + response := apiResponse{ + make([]tc.Alert, len(conflicts)+1), + result, + } + for i, conflict := range conflicts { + response.Alerts[i] = tc.Alert{ + Text: conflict, + Level: tc.WarnLevel.String(), + } + } + response.Alerts[len(conflicts)] = tc.Alert{ + Text: fmt.Sprintf("Invalidation request created for %v, start:%v end %v", *result.AssetURL, job.StartTime.Time, + job.StartTime.Add(time.Hour*time.Duration(ttl))), + Level: tc.SuccessLevel.String(), + } + resp, err := json.Marshal(response) + + if err != nil { + api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, fmt.Errorf("Marshaling JSON: %v", err)) + return + } + + if inf.Version == nil { + api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, errors.New("nil API version")) + return + } + + w.Header().Set(http.CanonicalHeaderKey("location"), fmt.Sprintf("%s://%s/api/%d.%d/jobs?id=%d", inf.Config.URL.Scheme, r.Host, inf.Version.Major, inf.Version.Minor, *result.ID)) + w.WriteHeader(http.StatusOK) + api.WriteAndLogErr(w, r, append(resp, '\n')) + + duplicate := "" + if len(conflicts) > 0 { + duplicate = "(duplicate) " + } + api.CreateChangeLogRawTx(api.ApiChange, api.Created+" content invalidation job "+duplicate+"- ID: "+ + strconv.FormatUint(*result.ID, 10)+" DS: "+*result.DeliveryService+" URL: '"+*result.AssetURL+ + "' Params: '"+*result.Parameters+"'", inf.User, inf.Tx.Tx) +} + +// Used by PUT requests to `/jobs`, replaces an existing content invalidation job +// with the provided request body. +func UpdateV40(w http.ResponseWriter, r *http.Request) { + inf, userErr, sysErr, errCode := api.NewInfo(r, []string{"id"}, []string{"id"}) + if userErr != nil || sysErr != nil { + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr) + return + } + defer inf.Close() + + var oFQDN string + var dsid uint + var uid uint + job := tc.InvalidationJobV4{} + row := inf.Tx.Tx.QueryRow(putInfoQueryV4, inf.Params["id"]) + err := row.Scan(&job.ID, + &job.CreatedBy, + &uid, + &dsid, + &job.DeliveryService, + &job.AssetURL, + &job.TTLHours, + &job.StartTime, + &job.InvalidationType, + &oFQDN) + if err != nil { + if err == sql.ErrNoRows { + userErr = fmt.Errorf("No job by id '%s'!", inf.Params["id"]) + errCode = http.StatusNotFound + } else { + sysErr = fmt.Errorf("fetching job update info: %v", err) + errCode = http.StatusInternalServerError + } + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr) + return + } + + if ok, err := IsUserAuthorizedToModifyDSID(inf, dsid); err != nil { + sysErr = fmt.Errorf("Checking user permissions on DS #%d: %v", dsid, err) errCode = http.StatusInternalServerError api.HandleErr(w, r, inf.Tx.Tx, errCode, nil, sysErr) return - } - var ttl uint - if ttl, err = job.TTLHours(); err != nil { - sysErr = fmt.Errorf("retrieving parsed TTL: %v", err) - errCode = http.StatusInternalServerError - api.HandleErr(w, r, inf.Tx.Tx, errCode, nil, sysErr) + } else if !ok { + userErr = errors.New("No such Delivery Service!") + errCode = http.StatusNotFound + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) return } - if ok, err := IsUserAuthorizedToModifyDSID(inf, dsid); err != nil { - sysErr = fmt.Errorf("Checking current user permissions for DS #%d: %v", dsid, err) + if ok, err := IsUserAuthorizedToModifyJobsMadeByUsername(inf, job.CreatedBy); err != nil { + sysErr = fmt.Errorf("Checking user permissions against user %s: %v", job.CreatedBy, err) errCode = http.StatusInternalServerError api.HandleErr(w, r, inf.Tx.Tx, errCode, nil, sysErr) return } else if !ok { - userErr = fmt.Errorf("No such Delivery Service!") + userErr = fmt.Errorf("No job by id '%s'!", inf.Params["id"]) errCode = http.StatusNotFound api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) return } - _, cdnName, _, err := dbhelpers.GetDSNameAndCDNFromID(inf.Tx.Tx, int(dsid)) - if err != nil { - api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, errors.New("getting delivery service and CDN name from ID: "+err.Error())) + input := tc.InvalidationJobV4{} + if err := json.NewDecoder(r.Body).Decode(&input); err != nil { + userErr = fmt.Errorf("Unable to parse input: %v", err) + sysErr = fmt.Errorf("parsing input to PUT jobs?id=%s: %v", inf.Params["id"], err) + errCode = http.StatusBadRequest + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr) return } - userErr, sysErr, statusCode := dbhelpers.CheckIfCurrentUserCanModifyCDN(inf.Tx.Tx, string(cdnName), inf.User.UserName) - if userErr != nil || sysErr != nil { - api.HandleErr(w, r, inf.Tx.Tx, statusCode, userErr, sysErr) + + if err := validateInvalidationJobV4(input); err != nil { + api.HandleErr(w, r, inf.Tx.Tx, http.StatusBadRequest, err, nil) return } - row := inf.Tx.Tx.QueryRow(insertQuery, - dsid, - *job.Regex, - time.Now(), - dsid, - inf.User.ID, - fmt.Sprintf("TTL:%dh", ttl), - (*job.StartTime).Time) - result := tc.InvalidationJob{} - err = row.Scan(&result.AssetURL, - &result.DeliveryService, - &result.ID, - &result.CreatedBy, - &result.Keyword, - &result.Parameters, - &result.StartTime) + if !strings.HasPrefix(input.AssetURL, oFQDN) { + userErr = fmt.Errorf("Cannot set asset URL that does not start with Delivery Service origin URL: %s", oFQDN) + errCode = http.StatusBadRequest + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) + return + } + + if job.StartTime.Before(time.Now()) { + userErr = errors.New("Cannot modify a job that has already started!") + errCode = http.StatusMethodNotAllowed + w.Header().Set(http.CanonicalHeaderKey("allow"), "GET,HEAD,DELETE") + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) + return + } + + if job.DeliveryService != input.DeliveryService { + userErr = errors.New("Cannot change 'deliveryService' of existing invalidation job!") + errCode = http.StatusConflict + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) + return + } + + if job.CreatedBy != input.CreatedBy { + userErr = errors.New("Cannot change 'createdBy' of existing invalidation jobs!") + errCode = http.StatusConflict + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) + return + } + + if job.ID != input.ID { + userErr = errors.New("Cannot change an invalidation job 'id'!") + errCode = http.StatusConflict + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) + return + } + + if job.InvalidationType != input.InvalidationType { + if input.InvalidationType == tc.REFETCH && !refetchAllowed(inf.Tx.Tx) { + userErr = errors.New("Invalidation Type REFRESH is disallowed") + errCode = http.StatusBadRequest + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) + return + } + } + + row = inf.Tx.Tx.QueryRow(updateQueryV4, + input.AssetURL, + input.TTLHours, + input.StartTime, + input.InvalidationType, + job.ID) + err = row.Scan(&job.AssetURL, + &job.CreatedBy, + &job.DeliveryService, + &job.ID, + &job.TTLHours, + &job.StartTime, + &job.InvalidationType) if err != nil { - userErr, sysErr, errCode = api.ParseDBError(err) - api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr) + sysErr = fmt.Errorf("Updating a job: %v", err) + errCode = http.StatusInternalServerError + api.HandleErr(w, r, inf.Tx.Tx, errCode, nil, sysErr) return } - if err := setRevalFlags(dsid, inf.Tx.Tx); err != nil { - api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, fmt.Errorf("setting reval flags: %v", err)) + if err = setRevalFlags(job.DeliveryService, inf.Tx.Tx); err != nil { + api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, fmt.Errorf("Setting reval flags: %v", err)) return } - conflicts := tc.ValidateJobUniqueness(inf.Tx.Tx, dsid, job.StartTime.Time, *result.AssetURL, ttl) - response := apiResponse{ + conflicts := tc.ValidateJobUniqueness(inf.Tx.Tx, dsid, input.StartTime, input.AssetURL, input.TTLHours) + response := apiResponseV4{ make([]tc.Alert, len(conflicts)+1), - result, + job, } for i, conflict := range conflicts { response.Alerts[i] = tc.Alert{ @@ -416,37 +987,43 @@ func Create(w http.ResponseWriter, r *http.Request) { } } response.Alerts[len(conflicts)] = tc.Alert{ - Text: fmt.Sprintf("Invalidation request created for %v, start:%v end %v", *result.AssetURL, job.StartTime.Time, - job.StartTime.Add(time.Hour*time.Duration(ttl))), + Text: fmt.Sprintf("Invalidation request created for %s, start: %v end: %v invalidation type: %v", + job.AssetURL, + job.StartTime, + job.StartTime.Add(time.Hour*time.Duration(job.TTLHours)), + job.InvalidationType), Level: tc.SuccessLevel.String(), } - resp, err := json.Marshal(response) + resp, err := json.Marshal(response) if err != nil { - api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, fmt.Errorf("Marshaling JSON: %v", err)) - return - } - - if inf.Version == nil { - api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, errors.New("nil API version")) + sysErr = fmt.Errorf("encoding response: %v", err) + errCode = http.StatusInternalServerError + api.HandleErr(w, r, inf.Tx.Tx, errCode, nil, sysErr) return } - w.Header().Set(http.CanonicalHeaderKey("location"), fmt.Sprintf("%s://%s/api/%d.%d/jobs?id=%d", inf.Config.URL.Scheme, r.Host, inf.Version.Major, inf.Version.Minor, *result.ID)) - w.WriteHeader(http.StatusOK) + w.Header().Set(http.CanonicalHeaderKey("content-type"), rfc.ApplicationJSON) api.WriteAndLogErr(w, r, append(resp, '\n')) - duplicate := "" - if len(conflicts) > 0 { - duplicate = "(duplicate) " - } - api.CreateChangeLogRawTx(api.ApiChange, api.Created+" content invalidation job "+duplicate+"- ID: "+ - strconv.FormatUint(*result.ID, 10)+" DS: "+*result.DeliveryService+" URL: '"+*result.AssetURL+ - "' Params: '"+*result.Parameters+"'", inf.User, inf.Tx.Tx) + changeLogMsg := fmt.Sprintf("%s content invalidation job - ID: %d DSXMLID: %s ASSET_URL: '%s' TTLHRs: %d INVALIDATION: %s", + api.Updated, + input.ID, + input.DeliveryService, + input.AssetURL, + input.TTLHours, + input.InvalidationType, + ) + api.CreateChangeLogRawTx(api.ApiChange, + changeLogMsg, + inf.User, + inf.Tx.Tx) } // Used by PUT requests to `/jobs`, replaces an existing content invalidation job // with the provided request body. +// +// Deprecated. To be used only with versions less than 4.0 func Update(w http.ResponseWriter, r *http.Request) { inf, userErr, sysErr, errCode := api.NewInfo(r, nil, nil) if userErr != nil || sysErr != nil { @@ -568,8 +1145,7 @@ func Update(w http.ResponseWriter, r *http.Request) { row = inf.Tx.Tx.QueryRow(updateQuery, input.AssetURL, - input.Keyword, - input.Parameters, + strings.TrimSuffix(strings.TrimPrefix(*input.Parameters, "TTL:"), "h"), // Strip TTL: and h from 'TTL:##h' input.StartTime.Time, *job.ID) err = row.Scan(&job.AssetURL, @@ -624,6 +1200,110 @@ func Update(w http.ResponseWriter, r *http.Request) { } // Used by DELETE requests to `/jobs`, deletes an existing content invalidation job +func DeleteV40(w http.ResponseWriter, r *http.Request) { + inf, userErr, sysErr, errCode := api.NewInfo(r, []string{"id"}, []string{"id"}) + if userErr != nil || sysErr != nil { + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr) + return + } + defer inf.Close() + + var dsid uint + var createdBy uint + row := inf.Tx.Tx.QueryRow(`SELECT job_deliveryservice, job_user FROM job WHERE id=$1`, inf.Params["id"]) + if err := row.Scan(&dsid, &createdBy); err != nil { + if err == sql.ErrNoRows { + userErr = fmt.Errorf("No job by id '%s'!", inf.Params["id"]) + errCode = http.StatusNotFound + } else { + sysErr = fmt.Errorf("Getting info for job #%s: %v", inf.Params["id"], err) + errCode = http.StatusInternalServerError + } + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr) + return + } + + if ok, err := IsUserAuthorizedToModifyDSID(inf, dsid); err != nil { + sysErr = fmt.Errorf("Checking user permissions on DS #%d: %v", dsid, err) + errCode = http.StatusInternalServerError + api.HandleErr(w, r, inf.Tx.Tx, errCode, nil, sysErr) + return + } else if !ok { + userErr = errors.New("No such Delivery Service!") + errCode = http.StatusNotFound + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) + return + } + + if ok, err := IsUserAuthorizedToModifyJobsMadeByUserID(inf, createdBy); err != nil { + sysErr = fmt.Errorf("Checking user permissions against user %v: %v", createdBy, err) + errCode = http.StatusInternalServerError + api.HandleErr(w, r, inf.Tx.Tx, errCode, nil, sysErr) + return + } else if !ok { + userErr = fmt.Errorf("No job by id '%s'!", inf.Params["id"]) + errCode = http.StatusNotFound + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) + return + } + + result := tc.InvalidationJobV4{} + row = inf.Tx.Tx.QueryRow(deleteQueryV4, inf.Params["id"]) + err := row.Scan( + &result.ID, + &result.AssetURL, + &result.CreatedBy, + &result.DeliveryService, + &result.TTLHours, + &result.InvalidationType, + &result.StartTime) + if err != nil { + sysErr = fmt.Errorf("deleting job #%s: %v", inf.Params["id"], err) + errCode = http.StatusInternalServerError + api.HandleErr(w, r, inf.Tx.Tx, errCode, nil, sysErr) + return + } + + if err = setRevalFlags(dsid, inf.Tx.Tx); err != nil { + sysErr = fmt.Errorf("setting reval_pending after deleting job #%s: %v", inf.Params["id"], err) + errCode = http.StatusInternalServerError + api.HandleErr(w, r, inf.Tx.Tx, errCode, nil, sysErr) + return + } + + response := apiResponseV4{[]tc.Alert{ + {Text: "Content invalidation job was deleted", Level: tc.SuccessLevel.String()}, + }, + result, + } + resp, err := json.Marshal(response) + if err != nil { + sysErr = fmt.Errorf("encoding response: %v", err) + errCode = http.StatusInternalServerError + api.HandleErr(w, r, inf.Tx.Tx, errCode, nil, sysErr) + return + } + + w.Header().Set(http.CanonicalHeaderKey("content-type"), rfc.ApplicationJSON) + api.WriteAndLogErr(w, r, append(resp, '\n')) + + changeLogMsg := fmt.Sprintf("%s content invalidation job - ID: %d DSXMLID: %s ASSET_URL: '%s' TTLHRs: %d INVALIDATION: %s", + api.Deleted, + result.ID, + result.DeliveryService, + result.AssetURL, + result.TTLHours, + result.InvalidationType, + ) + api.CreateChangeLogRawTx(api.ApiChange, + changeLogMsg, + inf.User, + inf.Tx.Tx) +} + +// Used by DELETE requests to `/jobs`, deletes an existing content invalidation job +// +// Deprecated. To be used only with versions less than 4.0 func Delete(w http.ResponseWriter, r *http.Request) { inf, userErr, sysErr, errCode := api.NewInfo(r, []string{"id"}, []string{"id"}) if userErr != nil || sysErr != nil { @@ -720,6 +1400,121 @@ func Delete(w http.ResponseWriter, r *http.Request) { api.CreateChangeLogRawTx(api.ApiChange, api.Deleted+" content invalidation job - ID: "+strconv.FormatUint(*result.ID, 10)+" DS: "+*result.DeliveryService+" URL: '"+*result.AssetURL+"' Params: '"+*result.Parameters+"'", inf.User, inf.Tx.Tx) } +// Validates the fields submitted for an InvalidationJobCreateV40. These errors +// are ultimately returned to the user +func validateJobCreateV4(job tc.InvalidationJobCreateV4, tx *sql.Tx) error { + errs := []string{} + err := validation.ValidateStruct(&job, + validation.Field(&job.DeliveryService, validation.Required), + validation.Field(&job.Regex, validation.Required, validation.NewStringRule(func(s string) bool { + return strings.HasPrefix(s, `\/`) || strings.HasPrefix(s, "/") + }, `must start with '/' (or '\/')`)), + validation.Field(&job.StartTime, validation.Required), + validation.Field(&job.TTLHours, validation.Required), + validation.Field(&job.InvalidationType, validation.Required, validation.NewStringRule(func(s string) bool { + return s == tc.REFRESH || s == tc.REFETCH + }, fmt.Sprintf("must be either %s or %s (case sensitive)", tc.REFRESH, tc.REFETCH))), + ) + if err != nil { + errs = append(errs, err.Error()) + } + + if _, _, err := dbhelpers.GetDSIDFromXMLID(tx, job.DeliveryService); err != nil { + errs = append(errs, "Delivery Service is invalid: "+err.Error()) + } + + if _, err := regexp.Compile(job.Regex); err != nil { + errs = append(errs, "regex: is not a valid Regular Expression: "+err.Error()) + } + + if job.StartTime.Before(time.Now()) { + errs = append(errs, "startTime: must be in the future") + } + + if valid, err := validateTLLHours(job.TTLHours, tx); !valid { + if err != nil { + errs = append(errs, "TTL is invalid: "+err.Error()) + } else { + errs = append(errs, "TTL is invalid") + } + } + + if job.InvalidationType == tc.REFETCH && !refetchAllowed(tx) { + errs = append(errs, "InvalidationType is invalid") + } + + if len(errs) > 0 { + return errors.New(strings.Join(errs, ", ")) + } + + return nil +} + +// validateInvalidationJobV4 checks that the InvalidationJob is valid, by ensuring all of its fields are well-defined. +// This returns an error describing any and all problematic fields encountered during validation. +func validateInvalidationJobV4(job tc.InvalidationJobV4) error { + errs := []string{} + err := validation.ValidateStruct(&job, + validation.Field(&job.DeliveryService, validation.Required), + validation.Field(&job.AssetURL, validation.Required, is.URL), + validation.Field(&job.CreatedBy, validation.Required), + validation.Field(&job.ID, validation.Required), + validation.Field(&job.TTLHours, validation.Required), + validation.Field(&job.StartTime, validation.Required), + validation.Field(&job.InvalidationType, validation.Required, validation.NewStringRule(func(s string) bool { + return s == tc.REFRESH || s == tc.REFETCH + }, fmt.Sprintf("must be either %s or %s (case sensitive)", tc.REFRESH, tc.REFETCH))), + ) + + if err != nil { + errs = append(errs, err.Error()) + } + + if job.StartTime.After(time.Now().Add(time.Hour * 48)) { + errs = append(errs, "startTime: must be within two days from now") + } + + if job.StartTime.Before(time.Now()) { + errs = append(errs, "startTime: cannot be in the past") + } + + if len(errs) > 0 { + return errors.New(strings.Join(errs, ", ")) + } + + return nil +} + +// validateTLLHours ensures the supplied TTL hours is within acceptable limits +func validateTLLHours(ttlHours uint32, tx *sql.Tx) (bool, error) { + var maxDays uint + err := tx.QueryRow(`SELECT value FROM parameter WHERE name='maxRevalDurationDays' AND config_file='regex_revalidate.config'`).Scan(&maxDays) + maxHours := maxDays * 24 + if err != nil { + log.Errorf("error querying \"maxRevalDurationDays\" parameter: %v", err) + return false, nil // sent to the user, hide server error + } + if err == nil && uint(ttlHours) > maxHours { + return false, fmt.Errorf("cannot exceed %s", strconv.FormatUint(uint64(maxHours), 10)) + } + return true, nil +} + +// refetchAllowed checks whether Refetch is allowed and enabled in the parameter table +func refetchAllowed(tx *sql.Tx) bool { + refetchEnabled := false + err := tx.QueryRow(`SELECT 'true' = lower(trim(p.value)) FROM "parameter" p WHERE p.name=$1 AND p.config_file=$2`, + tc.RefetchEnabled, tc.GlobalConfigFileName).Scan(&refetchEnabled) + if err != nil { + log.Errorf("error querying \"refetch_enabled\" from parameter: %v", err) + return refetchEnabled // sent to the user, hide server error + } + return refetchEnabled +} + +// API versions below 4.0 allowed for either the Delivery Service ID (uint) OR Delivery Service XML-ID (string). +// This can be refactored once api versions below 4.0 are removed to take a Delivery Service XML-ID (string), rather +// than an empty interface {}. func setRevalFlags(d interface{}, tx *sql.Tx) error { var useReval string row := tx.QueryRow(`SELECT value FROM parameter WHERE name=$1 AND config_file=$2`, tc.UseRevalPendingParameterName, tc.GlobalConfigFileName) @@ -765,7 +1560,7 @@ func setRevalFlags(d interface{}, tx *sql.Tx) error { // user isn't authorized. func IsUserAuthorizedToModifyDSID(inf *api.APIInfo, ds uint) (bool, error) { var t uint - row := inf.Tx.Tx.QueryRow(`SELECT tenant_id FROM deliveryservice where id=$1`, ds) + row := inf.Tx.Tx.QueryRow(`SELECT tenant_id FROM deliveryservice WHERE id=$1`, ds) if err := row.Scan(&t); err != nil { if err == sql.ErrNoRows { return false, nil //I do this to conceal the existence of DSes for which the user has no permission to see @@ -789,7 +1584,7 @@ func IsUserAuthorizedToModifyDSID(inf *api.APIInfo, ds uint) (bool, error) { // user isn't authorized. func IsUserAuthorizedToModifyDSXMLID(inf *api.APIInfo, ds string) (bool, error) { var t uint - row := inf.Tx.Tx.QueryRow(`SELECT tenant_id FROM deliveryservice where xml_id=$1`, ds) + row := inf.Tx.Tx.QueryRow(`SELECT tenant_id FROM deliveryservice WHERE xml_id=$1`, ds) if err := row.Scan(&t); err != nil { if err == sql.ErrNoRows { return false, nil //I do this to conceal the existence of DSes for which the user has no permission to see @@ -813,7 +1608,7 @@ func IsUserAuthorizedToModifyDSXMLID(inf *api.APIInfo, ds string) (bool, error) // user isn't authorized. func IsUserAuthorizedToModifyJobsMadeByUserID(inf *api.APIInfo, u uint) (bool, error) { var t uint - row := inf.Tx.Tx.QueryRow(`SELECT tenant_id FROM tm_user where id=$1`, u) + row := inf.Tx.Tx.QueryRow(`SELECT tenant_id FROM tm_user WHERE id=$1`, u) if err := row.Scan(&t); err != nil { if err == sql.ErrNoRows { return false, nil //I do this to conceal the existence of DSes for which the user has no permission to see @@ -837,7 +1632,7 @@ func IsUserAuthorizedToModifyJobsMadeByUserID(inf *api.APIInfo, u uint) (bool, e // user isn't authorized. func IsUserAuthorizedToModifyJobsMadeByUsername(inf *api.APIInfo, u string) (bool, error) { var t uint - row := inf.Tx.Tx.QueryRow(`SELECT tenant_id FROM tm_user where username=$1`, u) + row := inf.Tx.Tx.QueryRow(`SELECT tenant_id FROM tm_user WHERE username=$1`, u) if err := row.Scan(&t); err != nil { if err == sql.ErrNoRows { return false, nil //I do this to conceal the existence of DSes for which the user has no permission to see diff --git a/traffic_ops/traffic_ops_golang/routing/routes.go b/traffic_ops/traffic_ops_golang/routing/routes.go index 7bf4435640..c6112c2471 100644 --- a/traffic_ops/traffic_ops_golang/routing/routes.go +++ b/traffic_ops/traffic_ops_golang/routing/routes.go @@ -216,10 +216,10 @@ func Routes(d ServerData) ([]Route, http.Handler, error) { {api.Version{Major: 4, Minor: 0}, http.MethodGet, `logs/newcount/?$`, logs.GetNewCount, auth.PrivLevelReadOnly, []string{"LOG:READ"}, Authenticated, nil, 44058330123}, //Content invalidation jobs - {api.Version{Major: 4, Minor: 0}, http.MethodGet, `jobs/?$`, api.ReadHandler(&invalidationjobs.InvalidationJob{}), auth.PrivLevelReadOnly, []string{"JOB:READ", "DELIVERY-SERVICE:READ"}, Authenticated, nil, 49667820413}, - {api.Version{Major: 4, Minor: 0}, http.MethodDelete, `jobs/?$`, invalidationjobs.Delete, auth.PrivLevelPortal, []string{"JOB:DELETE", "JOB:READ", "DELIVERY-SERVICE:UPDATE", "DELIVERY-SERVICE:READ"}, Authenticated, nil, 4167807763}, - {api.Version{Major: 4, Minor: 0}, http.MethodPut, `jobs/?$`, invalidationjobs.Update, auth.PrivLevelPortal, []string{"JOB:UPDATE", "DELIVERY-SERVICE:UPDATE", "JOB:READ", "DELIVERY-SERVICE:READ"}, Authenticated, nil, 4861342263}, - {api.Version{Major: 4, Minor: 0}, http.MethodPost, `jobs/?`, invalidationjobs.Create, auth.PrivLevelPortal, []string{"JOB:CREATE", "JOB:READ", "DELIVERY-SERVICE:READ", "DELIVERY-SERVICE:UPDATE"}, Authenticated, nil, 404509553}, + {api.Version{Major: 4, Minor: 0}, http.MethodGet, `jobs/?$`, api.ReadHandler(&invalidationjobs.InvalidationJobV4{}), auth.PrivLevelReadOnly, nil, Authenticated, nil, 49667820413}, + {api.Version{Major: 4, Minor: 0}, http.MethodDelete, `jobs/?$`, invalidationjobs.DeleteV40, auth.PrivLevelPortal, nil, Authenticated, nil, 4167807763}, + {api.Version{Major: 4, Minor: 0}, http.MethodPut, `jobs/?$`, invalidationjobs.UpdateV40, auth.PrivLevelPortal, nil, Authenticated, nil, 4861342263}, + {api.Version{Major: 4, Minor: 0}, http.MethodPost, `jobs/?`, invalidationjobs.CreateV40, auth.PrivLevelPortal, nil, Authenticated, nil, 404509553}, //Login {api.Version{Major: 4, Minor: 0}, http.MethodPost, `user/login/?$`, login.LoginHandler(d.DB, d.Config), 0, nil, NoAuth, nil, 43926708213}, diff --git a/traffic_ops/v4-client/job.go b/traffic_ops/v4-client/job.go index 54f612617c..5898f8a115 100644 --- a/traffic_ops/v4-client/job.go +++ b/traffic_ops/v4-client/job.go @@ -16,7 +16,6 @@ package client */ import ( - "errors" "net/url" "strconv" @@ -28,7 +27,7 @@ import ( const apiJobs = "/jobs" // CreateInvalidationJob creates the passed Content Invalidation Job. -func (to *Session) CreateInvalidationJob(job tc.InvalidationJobInput, opts RequestOptions) (tc.Alerts, toclientlib.ReqInf, error) { +func (to *Session) CreateInvalidationJob(job tc.InvalidationJobCreateV4, opts RequestOptions) (tc.Alerts, toclientlib.ReqInf, error) { var alerts tc.Alerts reqInf, err := to.post(apiJobs, opts, job, &alerts) return alerts, reqInf, err @@ -48,23 +47,20 @@ func (to *Session) DeleteInvalidationJob(jobID uint64, opts RequestOptions) (tc. // UpdateInvalidationJob updates the passed Content Invalidation Job (it is // expected to have an ID). -func (to *Session) UpdateInvalidationJob(job tc.InvalidationJob, opts RequestOptions) (tc.Alerts, toclientlib.ReqInf, error) { +func (to *Session) UpdateInvalidationJob(job tc.InvalidationJobV4, opts RequestOptions) (tc.Alerts, toclientlib.ReqInf, error) { var alerts tc.Alerts - if job.ID == nil { - return alerts, toclientlib.ReqInf{}, errors.New("job has a nil ID") - } if opts.QueryParameters == nil { opts.QueryParameters = url.Values{} } - opts.QueryParameters.Set("id", strconv.FormatUint(*job.ID, 10)) + opts.QueryParameters.Set("id", strconv.FormatUint(job.ID, 10)) reqInf, err := to.put(apiJobs, opts, job, &alerts) return alerts, reqInf, err } // GetInvalidationJobs returns a list of Content Invalidation Jobs visible to // your Tenant. -func (to *Session) GetInvalidationJobs(opts RequestOptions) (tc.InvalidationJobsResponse, toclientlib.ReqInf, error) { - var data tc.InvalidationJobsResponse +func (to *Session) GetInvalidationJobs(opts RequestOptions) (tc.InvalidationJobsResponseV4, toclientlib.ReqInf, error) { + var data tc.InvalidationJobsResponseV4 reqInf, err := to.get(apiJobs, opts, &data) return data, reqInf, err } diff --git a/traffic_ops_db/test/docker/run-db-test.sh b/traffic_ops_db/test/docker/run-db-test.sh index a4b712b453..befdde1876 100755 --- a/traffic_ops_db/test/docker/run-db-test.sh +++ b/traffic_ops_db/test/docker/run-db-test.sh @@ -97,6 +97,7 @@ if [[ "$old_db_version" -eq 0 ]]; then ./db/admin --env=production reset || { echo "DB reset failed!"; exit 1; } fi +# applies migrations then performs seeding and patching ./db/admin --env=production upgrade || { echo "DB upgrade failed!"; exit 1; } new_db_version=$(get_current_db_version) diff --git a/traffic_portal/app/src/common/api/JobService.js b/traffic_portal/app/src/common/api/JobService.js index 39155f9552..4db2d6161d 100644 --- a/traffic_portal/app/src/common/api/JobService.js +++ b/traffic_portal/app/src/common/api/JobService.js @@ -20,7 +20,7 @@ var JobService = function($http, ENV) { this.getJobs = function(queryParams) { - return $http.get(ENV.api.unstable + 'jobs', {params: queryParams}).then( + return $http.get(ENV.api.stable + 'jobs', {params: queryParams}).then( function(result) { return result.data.response; }, @@ -31,7 +31,7 @@ var JobService = function($http, ENV) { }; this.createJob = function(job) { - return $http.post(ENV.api.unstable + 'jobs', job).then( + return $http.post(ENV.api.stable + 'jobs', job).then( function (result) { return result; }, @@ -42,7 +42,7 @@ var JobService = function($http, ENV) { }; this.deleteJob = function(id) { - return $http.delete(ENV.api.unstable + 'jobs', {params: {id: id}}).then( + return $http.delete(ENV.api.stable + 'jobs', {params: {id: id}}).then( function(result) { return result; },