From 495538ca5e83f936f4e452c219f3cfb2d14fc926 Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Wed, 28 Jul 2021 18:38:50 -0600 Subject: [PATCH 01/35] Modify existing endpoints to use new db schema for backwards compatibility --- lib/go-tc/invalidationjobs.go | 12 +- .../invalidationjobs/invalidationjobs.go | 153 +++++++++--------- 2 files changed, 82 insertions(+), 83 deletions(-) diff --git a/lib/go-tc/invalidationjobs.go b/lib/go-tc/invalidationjobs.go index 6239f27a8f..2014a97e48 100644 --- a/lib/go-tc/invalidationjobs.go +++ b/lib/go-tc/invalidationjobs.go @@ -255,17 +255,17 @@ func ValidateJobUniqueness(tx *sql.Tx, dsID uint, startTime time.Time, assetURL var errs []string const readQuery = ` -SELECT job.id, - keyword, - parameters, +SELECT invalidation.id, + 'PURGE' as keyword, + CONCAT('TTL:', ttl_hr, 'h') AS parameters, asset_url, start_time -FROM job -WHERE job.job_deliveryservice = $1 +FROM invalidation +WHERE invalidation.job_deliveryservice = $1 ` rows, err := tx.Query(readQuery, dsID) if err != nil { - errs = append(errs, fmt.Sprintf("unable to query for other invalidation jobs")) + errs = append(errs, "unable to query for other invalidation jobs") } else { defer rows.Close() jobStart := startTime diff --git a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go index 6124863387..213a7f944a 100644 --- a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go +++ b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go @@ -47,47 +47,28 @@ type InvalidationJob struct { tc.InvalidationJob } -const readQuery = ` -SELECT job.id, - keyword, - parameters, - asset_url, - start_time, - u.username AS createdBy, - ds.xml_id AS dsId -FROM job -JOIN tm_user u ON job.job_user = u.id -JOIN deliveryservice ds ON job.job_deliveryservice = ds.id -` - const insertQuery = ` -INSERT INTO job ( - agent, - asset_type, +INSERT INTO invalidation ( + ttl_hr, asset_url, + start_time, entered_time, - job_deliveryservice, job_user, - keyword, - parameters, - start_time, - status) + job_deliveryservice, + invalidation_type) VALUES ( - 1::bigint, - 'file', + $1, ( SELECT o.protocol::text || '://' || o.fqdn || rtrim(concat(':', o.port::text), ':') FROM origin o - WHERE o.deliveryservice = $1 + WHERE o.deliveryservice = $2 AND o.is_primary - ) || $2, - $3, + ) || $3, $4, $5, - 'PURGE', $6, $7, - 1::bigint + $8 ) RETURNING asset_url, @@ -98,8 +79,8 @@ RETURNING (SELECT tm_user.username FROM tm_user WHERE tm_user.id=job_user) AS createdBy, - keyword, - parameters, + 'PURGE' AS keyword, + CONCAT('TTL:', ttl_hr, 'h') AS parameters, start_time ` @@ -128,65 +109,64 @@ WHERE server.status NOT IN ( ` const updateQuery = ` -UPDATE job +UPDATE invalidation SET asset_url=$1, - keyword=$2, - parameters=$3, - start_time=$4 -WHERE job.id=$5 -RETURNING job.asset_url, + ttl_hr=$2, + start_time=$3 +WHERE invalidation.id=$4 +RETURNING asset_url, ( SELECT tm_user.username FROM tm_user - WHERE tm_user.id=job.job_user + WHERE tm_user.id=invalidation.job_user ) AS created_by, ( SELECT deliveryservice.xml_id FROM deliveryservice - WHERE deliveryservice.id=job.job_deliveryservice + WHERE deliveryservice.id=invalidation.job_deliveryservice ) AS delivery_service, - job.id, - job.keyword, - job.parameters, - job.start_time + invalidation.id, + 'PURGE' as keyword, + CONCAT('TTL:', ttl_hr, 'h') AS parameters, + start_time ` const putInfoQuery = ` -SELECT job.id AS id, +SELECT invalidation.id AS id, tm_user.username AS createdBy, - job.job_user AS createdByID, - job.job_deliveryservice AS dsid, + invalidation.job_user AS createdByID, + invalidation.job_deliveryservice AS dsid, deliveryservice.xml_id AS dsxmlid, - job.asset_url AS assetURL, - job.parameters, - job.start_time AS start_time, + invalidation.asset_url AS assetURL, + CONCAT('TTL:', ttl_hr, 'h') AS parameters, + invalidation.start_time AS start_time, origin.protocol || '://' || origin.fqdn || rtrim(concat(':', origin.port), ':') AS OFQDN -FROM job -INNER JOIN origin ON origin.deliveryservice=job.job_deliveryservice AND origin.is_primary -INNER JOIN tm_user ON tm_user.id=job.job_user -INNER JOIN deliveryservice ON deliveryservice.id=job.job_deliveryservice -WHERE job.id=$1 +FROM invalidation +INNER JOIN origin ON origin.deliveryservice=invalidation.job_deliveryservice AND origin.is_primary +INNER JOIN tm_user ON tm_user.id=invalidation.job_user +INNER JOIN deliveryservice ON deliveryservice.id=invalidation.job_deliveryservice +WHERE invalidation.id=$1 ` const deleteQuery = ` DELETE -FROM job -WHERE job.id=$1 -RETURNING job.asset_url, +FROM invalidation +WHERE invalidation.id=$1 +RETURNING invalidation.asset_url, ( SELECT tm_user.username FROM tm_user - WHERE tm_user.id=job.job_user + WHERE tm_user.id=invalidation.job_user ) AS created_by, ( SELECT deliveryservice.xml_id FROM deliveryservice - WHERE deliveryservice.id=job.job_deliveryservice + WHERE deliveryservice.id=invalidation.job_deliveryservice ) AS deliveryservice, - job.id, - job.keyword, - job.parameters, - job.start_time + invalidation.id, + 'PURGE' as keyword, + CONCAT('TTL:', ttl_hr, 'h') AS parameters, + invalidation.start_time ` type apiResponse struct { @@ -203,20 +183,33 @@ func selectMaxLastUpdatedQuery(where string) string { select max(last_updated) as t from last_deleted l where l.table_name='job') as res` } +const readQuery = ` +SELECT invalidation.id, + 'PURGE' AS keyword, + ttl_hr, + asset_url, + start_time, + u.username as createdBy, + ds.xml_id as dsId +FROM invalidation +JOIN tm_user u ON invalidation.job_user = u.id +JOIN deliveryservice ds ON invalidation.job_deliveryservice = ds.id +` + // Used by GET requests to `/jobs`, simply returns a filtered list of // content invalidation jobs according to the provided query parameters. func (job *InvalidationJob) Read(h http.Header, useIMS bool) ([]interface{}, error, error, int, *time.Time) { var maxTime time.Time var runSecond bool queryParamsToSQLCols := map[string]dbhelpers.WhereColumnInfo{ - "id": dbhelpers.WhereColumnInfo{Column: "job.id", Checker: api.IsInt}, - "keyword": dbhelpers.WhereColumnInfo{Column: "job.keyword"}, - "assetUrl": dbhelpers.WhereColumnInfo{Column: "job.asset_url"}, - "startTime": dbhelpers.WhereColumnInfo{Column: "job.start_time"}, - "userId": dbhelpers.WhereColumnInfo{Column: "job.job_user", Checker: api.IsInt}, - "createdBy": dbhelpers.WhereColumnInfo{Column: `(SELECT tm_user.username FROM tm_user WHERE tm_user.id=job.job_user)`}, - "deliveryService": dbhelpers.WhereColumnInfo{Column: `(SELECT deliveryservice.xml_id FROM deliveryservice WHERE deliveryservice.id=job.job_deliveryservice)`}, - "dsId": dbhelpers.WhereColumnInfo{Column: "job.job_deliveryservice", Checker: api.IsInt}, + "id": dbhelpers.WhereColumnInfo{Column: "invalidation.id", Checker: api.IsInt}, + "keyword": dbhelpers.WhereColumnInfo{Column: "keyword"}, + "assetUrl": dbhelpers.WhereColumnInfo{Column: "asset_url"}, + "startTime": dbhelpers.WhereColumnInfo{Column: "start_time"}, + "userId": dbhelpers.WhereColumnInfo{Column: "job_user", Checker: api.IsInt}, + "createdBy": dbhelpers.WhereColumnInfo{Column: `(SELECT tm_user.username FROM tm_user WHERE tm_user.id=invalidation.job_user)`}, + "deliveryService": dbhelpers.WhereColumnInfo{Column: `(SELECT deliveryservice.xml_id FROM deliveryservice WHERE deliveryservice.id=invalidation.job_deliveryservice)`}, + "dsId": dbhelpers.WhereColumnInfo{Column: "invalidation.job_deliveryservice", Checker: api.IsInt}, } where, orderBy, pagination, queryValues, errs := dbhelpers.BuildWhereAndOrderByAndPagination(job.APIInfo().Params, queryParamsToSQLCols) @@ -288,6 +281,12 @@ func (job *InvalidationJob) Read(h http.Header, useIMS bool) ([]interface{}, err return nil, nil, fmt.Errorf("parsing db response: %v", err), http.StatusInternalServerError, nil } + // NamedQuery (nor prepare statements) cannot handle string functions in SELECT such as: + // CONCAT('TTL:', ttl_hr, 'h') AS parameters + // So it is necessary to modify the value after it's been queried. + fmtTTL := fmt.Sprintf("TTL:%sh", *j.Parameters) + j.Parameters = &fmtTTL + returnable = append(returnable, j) } @@ -377,13 +376,14 @@ func Create(w http.ResponseWriter, r *http.Request) { return } row := inf.Tx.Tx.QueryRow(insertQuery, - dsid, + ttl, + dsid, // Used in inner select for deliveryservice *job.Regex, + (*job.StartTime).Time, time.Now(), - dsid, inf.User.ID, - fmt.Sprintf("TTL:%dh", ttl), - (*job.StartTime).Time) + dsid, + "REFRESH") result := tc.InvalidationJob{} err = row.Scan(&result.AssetURL, @@ -568,8 +568,7 @@ func Update(w http.ResponseWriter, r *http.Request) { row = inf.Tx.Tx.QueryRow(updateQuery, input.AssetURL, - input.Keyword, - input.Parameters, + strings.TrimSuffix(strings.TrimPrefix(*input.Parameters, "TTL:"), "h"), // Strip TTL: and h from 'TTL:##h' input.StartTime.Time, *job.ID) err = row.Scan(&job.AssetURL, @@ -634,7 +633,7 @@ func Delete(w http.ResponseWriter, r *http.Request) { var dsid uint var createdBy uint - row := inf.Tx.Tx.QueryRow(`SELECT job_deliveryservice, job_user FROM job WHERE id=$1`, inf.Params["id"]) + row := inf.Tx.Tx.QueryRow(`SELECT job_deliveryservice, job_user FROM invalidation WHERE id=$1`, inf.Params["id"]) if err := row.Scan(&dsid, &createdBy); err != nil { if err == sql.ErrNoRows { userErr = fmt.Errorf("No job by id '%s'!", inf.Params["id"]) From 7b3d32f28e18304d2d2bada81be5dad4e0bdbdff Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Thu, 29 Jul 2021 09:57:02 -0600 Subject: [PATCH 02/35] Revert table name change from 'invalidation' back to 'job' --- lib/go-tc/invalidationjobs.go | 6 +- .../invalidationjobs/invalidationjobs.go | 64 +++++++++---------- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/lib/go-tc/invalidationjobs.go b/lib/go-tc/invalidationjobs.go index 2014a97e48..f1f3a143bb 100644 --- a/lib/go-tc/invalidationjobs.go +++ b/lib/go-tc/invalidationjobs.go @@ -255,13 +255,13 @@ func ValidateJobUniqueness(tx *sql.Tx, dsID uint, startTime time.Time, assetURL var errs []string const readQuery = ` -SELECT invalidation.id, +SELECT job.id, 'PURGE' as keyword, CONCAT('TTL:', ttl_hr, 'h') AS parameters, asset_url, start_time -FROM invalidation -WHERE invalidation.job_deliveryservice = $1 +FROM job +WHERE job.job_deliveryservice = $1 ` rows, err := tx.Query(readQuery, dsID) if err != nil { diff --git a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go index 213a7f944a..fbe2e19476 100644 --- a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go +++ b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go @@ -48,7 +48,7 @@ type InvalidationJob struct { } const insertQuery = ` -INSERT INTO invalidation ( +INSERT INTO job ( ttl_hr, asset_url, start_time, @@ -109,64 +109,64 @@ WHERE server.status NOT IN ( ` const updateQuery = ` -UPDATE invalidation +UPDATE job SET asset_url=$1, ttl_hr=$2, start_time=$3 -WHERE invalidation.id=$4 +WHERE job.id=$4 RETURNING asset_url, ( SELECT tm_user.username FROM tm_user - WHERE tm_user.id=invalidation.job_user + WHERE tm_user.id=job.job_user ) AS created_by, ( SELECT deliveryservice.xml_id FROM deliveryservice - WHERE deliveryservice.id=invalidation.job_deliveryservice + WHERE deliveryservice.id=job.job_deliveryservice ) AS delivery_service, - invalidation.id, + job.id, 'PURGE' as keyword, CONCAT('TTL:', ttl_hr, 'h') AS parameters, start_time ` const putInfoQuery = ` -SELECT invalidation.id AS id, +SELECT job.id AS id, tm_user.username AS createdBy, - invalidation.job_user AS createdByID, - invalidation.job_deliveryservice AS dsid, + job.job_user AS createdByID, + job.job_deliveryservice AS dsid, deliveryservice.xml_id AS dsxmlid, - invalidation.asset_url AS assetURL, + job.asset_url AS assetURL, CONCAT('TTL:', ttl_hr, 'h') AS parameters, - invalidation.start_time AS start_time, + job.start_time AS start_time, origin.protocol || '://' || origin.fqdn || rtrim(concat(':', origin.port), ':') AS OFQDN -FROM invalidation -INNER JOIN origin ON origin.deliveryservice=invalidation.job_deliveryservice AND origin.is_primary -INNER JOIN tm_user ON tm_user.id=invalidation.job_user -INNER JOIN deliveryservice ON deliveryservice.id=invalidation.job_deliveryservice -WHERE invalidation.id=$1 +FROM job +INNER JOIN origin ON origin.deliveryservice=job.job_deliveryservice AND origin.is_primary +INNER JOIN tm_user ON tm_user.id=job.job_user +INNER JOIN deliveryservice ON deliveryservice.id=job.job_deliveryservice +WHERE job.id=$1 ` const deleteQuery = ` DELETE -FROM invalidation -WHERE invalidation.id=$1 -RETURNING invalidation.asset_url, +FROM job +WHERE job.id=$1 +RETURNING job.asset_url, ( SELECT tm_user.username FROM tm_user - WHERE tm_user.id=invalidation.job_user + WHERE tm_user.id=job.job_user ) AS created_by, ( SELECT deliveryservice.xml_id FROM deliveryservice - WHERE deliveryservice.id=invalidation.job_deliveryservice + WHERE deliveryservice.id=job.job_deliveryservice ) AS deliveryservice, - invalidation.id, + job.id, 'PURGE' as keyword, CONCAT('TTL:', ttl_hr, 'h') AS parameters, - invalidation.start_time + job.start_time ` type apiResponse struct { @@ -184,16 +184,16 @@ func selectMaxLastUpdatedQuery(where string) string { } const readQuery = ` -SELECT invalidation.id, +SELECT job.id, 'PURGE' AS keyword, ttl_hr, asset_url, start_time, u.username as createdBy, ds.xml_id as dsId -FROM invalidation -JOIN tm_user u ON invalidation.job_user = u.id -JOIN deliveryservice ds ON invalidation.job_deliveryservice = ds.id +FROM job +JOIN tm_user u ON job.job_user = u.id +JOIN deliveryservice ds ON job.job_deliveryservice = ds.id ` // Used by GET requests to `/jobs`, simply returns a filtered list of @@ -202,14 +202,14 @@ func (job *InvalidationJob) Read(h http.Header, useIMS bool) ([]interface{}, err var maxTime time.Time var runSecond bool queryParamsToSQLCols := map[string]dbhelpers.WhereColumnInfo{ - "id": dbhelpers.WhereColumnInfo{Column: "invalidation.id", Checker: api.IsInt}, + "id": dbhelpers.WhereColumnInfo{Column: "job.id", Checker: api.IsInt}, "keyword": dbhelpers.WhereColumnInfo{Column: "keyword"}, "assetUrl": dbhelpers.WhereColumnInfo{Column: "asset_url"}, "startTime": dbhelpers.WhereColumnInfo{Column: "start_time"}, "userId": dbhelpers.WhereColumnInfo{Column: "job_user", Checker: api.IsInt}, - "createdBy": dbhelpers.WhereColumnInfo{Column: `(SELECT tm_user.username FROM tm_user WHERE tm_user.id=invalidation.job_user)`}, - "deliveryService": dbhelpers.WhereColumnInfo{Column: `(SELECT deliveryservice.xml_id FROM deliveryservice WHERE deliveryservice.id=invalidation.job_deliveryservice)`}, - "dsId": dbhelpers.WhereColumnInfo{Column: "invalidation.job_deliveryservice", Checker: api.IsInt}, + "createdBy": dbhelpers.WhereColumnInfo{Column: `(SELECT tm_user.username FROM tm_user WHERE tm_user.id=job.job_user)`}, + "deliveryService": dbhelpers.WhereColumnInfo{Column: `(SELECT deliveryservice.xml_id FROM deliveryservice WHERE deliveryservice.id=job.job_deliveryservice)`}, + "dsId": dbhelpers.WhereColumnInfo{Column: "job.job_deliveryservice", Checker: api.IsInt}, } where, orderBy, pagination, queryValues, errs := dbhelpers.BuildWhereAndOrderByAndPagination(job.APIInfo().Params, queryParamsToSQLCols) @@ -633,7 +633,7 @@ func Delete(w http.ResponseWriter, r *http.Request) { var dsid uint var createdBy uint - row := inf.Tx.Tx.QueryRow(`SELECT job_deliveryservice, job_user FROM invalidation WHERE id=$1`, inf.Params["id"]) + row := inf.Tx.Tx.QueryRow(`SELECT job_deliveryservice, job_user FROM job WHERE id=$1`, inf.Params["id"]) if err := row.Scan(&dsid, &createdBy); err != nil { if err == sql.ErrNoRows { userErr = fmt.Errorf("No job by id '%s'!", inf.Params["id"]) From 8f3719eb5833c9165d6c0c2779756dec9ab83df6 Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Fri, 13 Aug 2021 15:14:41 -0600 Subject: [PATCH 03/35] Add Create ability for REFETCH --- lib/go-tc/invalidationjobs.go | 156 +++++++++++++- .../invalidationjobs/invalidationjobs.go | 203 +++++++++++++++++- .../traffic_ops_golang/routing/routes.go | 8 +- 3 files changed, 347 insertions(+), 20 deletions(-) diff --git a/lib/go-tc/invalidationjobs.go b/lib/go-tc/invalidationjobs.go index f1f3a143bb..4019c07187 100644 --- a/lib/go-tc/invalidationjobs.go +++ b/lib/go-tc/invalidationjobs.go @@ -59,6 +59,17 @@ type InvalidationJob struct { StartTime *Time `json:"startTime"` } +// InvalidationJobV40 represents a content invalidation job as returned by the API. +type InvalidationJobV40 struct { + ID *uint64 `json:"id"` + AssetURL *string `json:"assetUrl"` + CreatedBy *string `json:"createdBy"` + DeliveryServiceXMLID *string `json:"deliveryService"` + TTLHours *uint `json:"ttlHours"` + InvalidationType *string `json:"invalidationType"` + StartTime *time.Time `json:"startTime"` +} + // InvalidationJobsResponse is the type of a response from Traffic Ops to a // request made to its /jobs API endpoint. type InvalidationJobsResponse struct { @@ -246,6 +257,12 @@ func (job *InvalidationJobInput) Validate(tx *sql.Tx) error { return nil } +type compareJob struct { + AssetURL *string + TTLHours *uint + StartTime *time.Time +} + // ValidateJobUniqueness returns a message describing each overlap between // existing content invalidation jobs for the same assetURL as the one passed. // @@ -255,35 +272,35 @@ func ValidateJobUniqueness(tx *sql.Tx, dsID uint, startTime time.Time, assetURL var errs []string const readQuery = ` -SELECT job.id, - 'PURGE' as keyword, - CONCAT('TTL:', ttl_hr, 'h') AS parameters, - asset_url, +SELECT asset_url, + ttl_hr, start_time FROM job WHERE job.job_deliveryservice = $1 ` rows, err := tx.Query(readQuery, dsID) if err != nil { - errs = append(errs, "unable to query for other invalidation jobs") + errs = append(errs, "unable to query for invalidation jobs while validating job uniqueness") } else { defer rows.Close() jobStart := startTime for rows.Next() { - testJob := InvalidationJob{} - err = rows.Scan(&testJob.ID, &testJob.Keyword, &testJob.Parameters, &testJob.AssetURL, &testJob.StartTime) + testJob := compareJob{} + err = rows.Scan( + &testJob.AssetURL, + &testJob.TTLHours, + &testJob.StartTime) if err != nil { continue } if !strings.HasSuffix(*testJob.AssetURL, assetURL) { continue } - testJobTTL := testJob.TTLHours() - if testJobTTL == 0 { + if *testJob.TTLHours == 0 { continue } - testJobStart := testJob.StartTime.Time - testJobEnd := testJobStart.Add(time.Hour * time.Duration(testJobTTL)) + testJobStart := testJob.StartTime + testJobEnd := testJobStart.Add(time.Hour * time.Duration(*testJob.TTLHours)) jobEnd := jobStart.Add(time.Hour * time.Duration(ttlHours)) // jobStart in testJob range if (testJobStart.Before(jobStart) && jobStart.Before(testJobEnd)) || @@ -456,3 +473,120 @@ func (job *UserInvalidationJobInput) Validate(tx *sql.Tx) error { } return nil } + +const REFRESH = "REFRESH" +const REFETCH = "REFETCH" + +// InvalidationJobCreateV40 represents user input intending to create a content invalidation job. +type InvalidationJobCreateV40 struct { + // The Delivery Service XML-ID for which the Invalidation Job is to be applied. + DeliveryService string `json:"deliveryService"` + + // Regex is a regular expression which not only must be valid, but should also start with '/' + // (or escaped: '\/') + Regex string `json:"regex"` + + // StartTime is the time at which the job will come into effect. Must be in the future. + StartTime time.Time `json:"startTime"` + + // TTLHours indicates the Time-to-Live of the job in hours. Must be a positive integer value. + TTLHours uint32 `json:"ttlHours"` + + // InvalidationType must be either REFRESH (default behavior) or REFETCH. If REFETCH, must + // also comply with global parameter setting + InvalidationType string `json:"invalidationType"` +} + +// Validates the fields submitted for an InvalidationJobCreateV40. These errors +// are ultimately returned to the user +func (job *InvalidationJobCreateV40) Validate(tx *sql.Tx) error { + errs := []string{} + err := validation.ValidateStruct(job, + validation.Field(&job.DeliveryService, validation.Required), + validation.Field(&job.Regex, validation.Required, validation.NewStringRule(func(s string) bool { + return strings.HasPrefix(s, `\/`) || strings.HasPrefix(s, "/") + }, `must start with '/' (or '\/')`)), + validation.Field(&job.StartTime, validation.Required), + validation.Field(&job.TTLHours, validation.Required), + validation.Field(&job.InvalidationType, validation.Required, validation.NewStringRule(func(s string) bool { + return s == REFRESH || s == REFETCH + }, fmt.Sprintf("must be either %s or %s (case sensitive)", REFRESH, REFETCH))), + ) + if err != nil { + errs = append(errs, err.Error()) + } + + if err := job.validateDeliveryService(tx); err != nil { + errs = append(errs, "Delivery Service is invalid: "+err.Error()) + } + + if _, err := regexp.Compile(job.Regex); err != nil { + errs = append(errs, "regex: is not a valid Regular Expression: "+err.Error()) + } + + if job.StartTime.Before(time.Now()) { + errs = append(errs, "startTime: must be in the future") + } + + if err := job.validateTLLHours(tx); err != nil { + errs = append(errs, "TTL is invalid: "+err.Error()) + } + + if err := job.validateRefetch(tx); err != nil { + errs = append(errs, "InvalidationType is invalid: "+err.Error()) + } + + if len(errs) > 0 { + return errors.New(strings.Join(errs, ", ")) + } + + return nil +} + +// validateDeliveryService ensures the supplied (required) Delivery Service XML ID exists +func (job *InvalidationJobCreateV40) validateDeliveryService(tx *sql.Tx) error { + var exists bool + row := tx.QueryRow(`SELECT EXISTS(SELECT * FROM deliveryservice WHERE xml_id=$1)`, job.DeliveryService) + if err := row.Scan(&exists); err != nil { + if err == sql.ErrNoRows { + return fmt.Errorf("No DeliveryService exists matching identifier: %v", job.DeliveryService) + } + return errors.New("Unknown error occurred") + } + return nil +} + +// validateDeliveryService ensures the supplied (required) Delivery Service XML ID exists +func (job *InvalidationJobCreateV40) validateTLLHours(tx *sql.Tx) error { + var maxDays uint + err := tx.QueryRow(`SELECT value FROM parameter WHERE name='maxRevalDurationDays' AND config_file='regex_revalidate.config'`).Scan(&maxDays) + maxHours := maxDays * 24 + if err == nil && uint(job.TTLHours) > maxHours { // silently ignore other errors too + return fmt.Errorf("cannot exceed %s", strconv.FormatUint(uint64(maxHours), 10)) + } + return nil +} + +// validateDeliveryService ensures the supplied (required) Delivery Service XML ID exists +func (job *InvalidationJobCreateV40) validateRefetch(tx *sql.Tx) error { + if job.InvalidationType == REFETCH { + var refetchEnabled bool + err := tx.QueryRow(`SELECT value FROM parameter WHERE name='refetch_enabled' AND config_file='global'`).Scan(&refetchEnabled) + if err == nil && !refetchEnabled { // silently ignore other errors too + return fmt.Errorf("%s has not been enabled for this CDN", REFETCH) + } + } + return nil +} + +func (job *InvalidationJobCreateV40) GetDSIDfromDSXMLID(tx *sql.Tx) (uint, error) { + var dsID uint + row := tx.QueryRow(`SELECT id FROM deliveryservice WHERE xml_id=$1`, job.DeliveryService) + if err := row.Scan(&dsID); err != nil { + if err == sql.ErrNoRows { + return 0, fmt.Errorf("No DeliveryService exists matching identifier: %v", job.DeliveryService) + } + return 0, errors.New("Unknown error occurred") + } + return dsID, nil +} diff --git a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go index fbe2e19476..52c91a39fb 100644 --- a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go +++ b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go @@ -47,6 +47,7 @@ type InvalidationJob struct { tc.InvalidationJob } +// Deprecated, only to be used with versions below 4.0 const insertQuery = ` INSERT INTO job ( ttl_hr, @@ -84,6 +85,44 @@ RETURNING start_time ` +// Almost the same as the above insert, but returns appropriate values for API 4.0+ +const insertQueryV40 = ` +INSERT INTO job ( + ttl_hr, + asset_url, + start_time, + entered_time, + job_user, + job_deliveryservice, + invalidation_type) +VALUES ( + $1, + ( + SELECT o.protocol::text || '://' || o.fqdn || rtrim(concat(':', o.port::text), ':') + FROM origin o + WHERE o.deliveryservice = $2 + AND o.is_primary + ) || $3, + $4, + $5, + $6, + $7, + $8 +) +RETURNING + id, + asset_url, + (SELECT tm_user.username + FROM tm_user + WHERE tm_user.id=job_user) AS createdBy, + (SELECT deliveryservice.xml_id + FROM deliveryservice + WHERE deliveryservice.id=job_deliveryservice) AS deliveryServiceXML, + ttl_hr as ttlHrs, + invalidation_type as invalidationType, + start_time as startTime +` + const revalQuery = ` UPDATE server SET %s=TRUE WHERE server.status NOT IN ( @@ -174,6 +213,11 @@ type apiResponse struct { Response tc.InvalidationJob `json:"response,omitempty"` } +type apiResponseV40 struct { + Alerts []tc.Alert `json:"alerts,omitempty"` + Response tc.InvalidationJobV40 `json:"response,omitempty"` +} + func selectMaxLastUpdatedQuery(where string) string { return `SELECT max(t) from ( SELECT max(job.last_updated) as t FROM job @@ -299,6 +343,152 @@ func (job *InvalidationJob) Read(h http.Header, useIMS bool) ([]interface{}, err // Used by POST requests to `/jobs`, creates a new content invalidation job // from the provided request body. +func CreateV40(w http.ResponseWriter, r *http.Request) { + inf, userErr, sysErr, errCode := api.NewInfo(r, nil, nil) + if userErr != nil || sysErr != nil { + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr) + return + } + defer inf.Close() + + job := tc.InvalidationJobCreateV40{} + if err := json.NewDecoder(r.Body).Decode(&job); err != nil { + api.HandleErr(w, r, inf.Tx.Tx, http.StatusBadRequest, errors.New("Unable to parse Invalidation Job"), fmt.Errorf("parsing jobs/ POST: %v", err)) + return + } + + // Check if request object is valid + w.Header().Set(rfc.ContentType, rfc.ApplicationJSON) + if err := job.Validate(inf.Tx.Tx); err != nil { + response := tc.Alerts{ + Alerts: []tc.Alert{ + { + Text: err.Error(), + Level: tc.ErrorLevel.String(), + }, + }, + } + + resp, err := json.Marshal(response) + if err != nil { + api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, fmt.Errorf("Encoding bad request response: %v", err)) + return + } + + w.WriteHeader(http.StatusBadRequest) + api.WriteAndLogErr(w, r, append(resp, '\n')) + return + } + + // Check if authorized + if ok, err := IsUserAuthorizedToModifyDSXMLID(inf, job.DeliveryService); err != nil { + sysErr = fmt.Errorf("failed checking current user permissions for DS #%s: %v", job.DeliveryService, err) + errCode = http.StatusInternalServerError + api.HandleErr(w, r, inf.Tx.Tx, errCode, nil, sysErr) + return + } else if !ok { + userErr = fmt.Errorf("failed to authorize based on tenancy") + errCode = http.StatusNotFound + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) + return + } + + // DS existence was already verified in the Validate() function + dsid, err := job.GetDSIDfromDSXMLID(inf.Tx.Tx) + if err != nil { + sysErr = fmt.Errorf("failed to match XML ID to int ID for Delivery Service %s: %v", job.DeliveryService, err) + errCode = http.StatusInternalServerError + api.HandleErr(w, r, inf.Tx.Tx, errCode, nil, sysErr) + } + + row := inf.Tx.Tx.QueryRow(insertQueryV40, + job.TTLHours, + dsid, // Used in inner select for deliveryservice + job.Regex, + job.StartTime, + time.Now(), + inf.User.ID, + dsid, + job.InvalidationType) // Defaults for all api versions below 4.0 + + result := tc.InvalidationJobV40{} + err = row.Scan( + &result.ID, + &result.AssetURL, + &result.CreatedBy, + &result.DeliveryServiceXMLID, + &result.TTLHours, + &result.InvalidationType, + &result.StartTime) + if err != nil { + userErr, sysErr, errCode = api.ParseDBError(err) + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr) + return + } + + if err := setRevalFlags(dsid, inf.Tx.Tx); err != nil { + api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, fmt.Errorf("setting reval flags: %v", err)) + return + } + + conflicts := tc.ValidateJobUniqueness(inf.Tx.Tx, dsid, *result.StartTime, *result.AssetURL, *result.TTLHours) + response := apiResponseV40{ + make([]tc.Alert, len(conflicts)+1), + result, + } + for i, conflict := range conflicts { + response.Alerts[i] = tc.Alert{ + Text: conflict, + Level: tc.WarnLevel.String(), + } + } + response.Alerts[len(conflicts)] = tc.Alert{ + Text: fmt.Sprintf("Invalidation (%s) request created for %v, start:%v end %v", + *result.InvalidationType, + *result.AssetURL, + *result.StartTime, + result.StartTime.Add(time.Hour*time.Duration(job.TTLHours))), + Level: tc.SuccessLevel.String(), + } + resp, err := json.Marshal(response) + + if err != nil { + api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, fmt.Errorf("Marshaling JSON: %v", err)) + return + } + + if inf.Version == nil { + api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, errors.New("nil API version")) + return + } + + w.Header().Set(http.CanonicalHeaderKey("location"), fmt.Sprintf("%s://%s/api/%d.%d/jobs?id=%d", inf.Config.URL.Scheme, r.Host, inf.Version.Major, inf.Version.Minor, *result.ID)) + w.WriteHeader(http.StatusOK) + api.WriteAndLogErr(w, r, append(resp, '\n')) + + duplicate := "" + if len(conflicts) > 0 { + duplicate = "(duplicate) " + } + changeLogMsg := fmt.Sprintf("%s content invalidation job %s- ID: %d DSXMLID: %s ASSET_URL: %s TTLHRs: %d INVALIDATION: %s", + api.Created, + duplicate, + *result.ID, + *result.DeliveryServiceXMLID, + *result.AssetURL, + *result.TTLHours, + *result.InvalidationType, + ) + api.CreateChangeLogRawTx(api.ApiChange, + changeLogMsg, + inf.User, + inf.Tx.Tx) +} + +// Used by POST requests to `/jobs`, creates a new content invalidation job +// from the provided request body. +// +// Deprecated. To be used only with versions less than 4.0 func Create(w http.ResponseWriter, r *http.Request) { inf, userErr, sysErr, errCode := api.NewInfo(r, nil, nil) if userErr != nil || sysErr != nil { @@ -383,7 +573,7 @@ func Create(w http.ResponseWriter, r *http.Request) { time.Now(), inf.User.ID, dsid, - "REFRESH") + tc.REFRESH) // Defaults for all api versions below 4.0 result := tc.InvalidationJob{} err = row.Scan(&result.AssetURL, @@ -719,6 +909,9 @@ func Delete(w http.ResponseWriter, r *http.Request) { api.CreateChangeLogRawTx(api.ApiChange, api.Deleted+" content invalidation job - ID: "+strconv.FormatUint(*result.ID, 10)+" DS: "+*result.DeliveryService+" URL: '"+*result.AssetURL+"' Params: '"+*result.Parameters+"'", inf.User, inf.Tx.Tx) } +// API versions below 4.0 allowed for either the Delivery Service ID (uint) OR Delivery Service XML-ID (string). +// This can be refactored once api versions below 4.0 are removed to take a Delivery Service XML-ID (string), rather +// than an empty interface {}. func setRevalFlags(d interface{}, tx *sql.Tx) error { var useReval string row := tx.QueryRow(`SELECT value FROM parameter WHERE name=$1 AND config_file=$2`, tc.UseRevalPendingParameterName, tc.GlobalConfigFileName) @@ -764,7 +957,7 @@ func setRevalFlags(d interface{}, tx *sql.Tx) error { // user isn't authorized. func IsUserAuthorizedToModifyDSID(inf *api.APIInfo, ds uint) (bool, error) { var t uint - row := inf.Tx.Tx.QueryRow(`SELECT tenant_id FROM deliveryservice where id=$1`, ds) + row := inf.Tx.Tx.QueryRow(`SELECT tenant_id FROM deliveryservice WHERE id=$1`, ds) if err := row.Scan(&t); err != nil { if err == sql.ErrNoRows { return false, nil //I do this to conceal the existence of DSes for which the user has no permission to see @@ -788,7 +981,7 @@ func IsUserAuthorizedToModifyDSID(inf *api.APIInfo, ds uint) (bool, error) { // user isn't authorized. func IsUserAuthorizedToModifyDSXMLID(inf *api.APIInfo, ds string) (bool, error) { var t uint - row := inf.Tx.Tx.QueryRow(`SELECT tenant_id FROM deliveryservice where xml_id=$1`, ds) + row := inf.Tx.Tx.QueryRow(`SELECT tenant_id FROM deliveryservice WHERE xml_id=$1`, ds) if err := row.Scan(&t); err != nil { if err == sql.ErrNoRows { return false, nil //I do this to conceal the existence of DSes for which the user has no permission to see @@ -812,7 +1005,7 @@ func IsUserAuthorizedToModifyDSXMLID(inf *api.APIInfo, ds string) (bool, error) // user isn't authorized. func IsUserAuthorizedToModifyJobsMadeByUserID(inf *api.APIInfo, u uint) (bool, error) { var t uint - row := inf.Tx.Tx.QueryRow(`SELECT tenant_id FROM tm_user where id=$1`, u) + row := inf.Tx.Tx.QueryRow(`SELECT tenant_id FROM tm_user WHERE id=$1`, u) if err := row.Scan(&t); err != nil { if err == sql.ErrNoRows { return false, nil //I do this to conceal the existence of DSes for which the user has no permission to see @@ -836,7 +1029,7 @@ func IsUserAuthorizedToModifyJobsMadeByUserID(inf *api.APIInfo, u uint) (bool, e // user isn't authorized. func IsUserAuthorizedToModifyJobsMadeByUsername(inf *api.APIInfo, u string) (bool, error) { var t uint - row := inf.Tx.Tx.QueryRow(`SELECT tenant_id FROM tm_user where username=$1`, u) + row := inf.Tx.Tx.QueryRow(`SELECT tenant_id FROM tm_user WHERE username=$1`, u) if err := row.Scan(&t); err != nil { if err == sql.ErrNoRows { return false, nil //I do this to conceal the existence of DSes for which the user has no permission to see diff --git a/traffic_ops/traffic_ops_golang/routing/routes.go b/traffic_ops/traffic_ops_golang/routing/routes.go index 7bf4435640..71eb5afb08 100644 --- a/traffic_ops/traffic_ops_golang/routing/routes.go +++ b/traffic_ops/traffic_ops_golang/routing/routes.go @@ -216,10 +216,10 @@ func Routes(d ServerData) ([]Route, http.Handler, error) { {api.Version{Major: 4, Minor: 0}, http.MethodGet, `logs/newcount/?$`, logs.GetNewCount, auth.PrivLevelReadOnly, []string{"LOG:READ"}, Authenticated, nil, 44058330123}, //Content invalidation jobs - {api.Version{Major: 4, Minor: 0}, http.MethodGet, `jobs/?$`, api.ReadHandler(&invalidationjobs.InvalidationJob{}), auth.PrivLevelReadOnly, []string{"JOB:READ", "DELIVERY-SERVICE:READ"}, Authenticated, nil, 49667820413}, - {api.Version{Major: 4, Minor: 0}, http.MethodDelete, `jobs/?$`, invalidationjobs.Delete, auth.PrivLevelPortal, []string{"JOB:DELETE", "JOB:READ", "DELIVERY-SERVICE:UPDATE", "DELIVERY-SERVICE:READ"}, Authenticated, nil, 4167807763}, - {api.Version{Major: 4, Minor: 0}, http.MethodPut, `jobs/?$`, invalidationjobs.Update, auth.PrivLevelPortal, []string{"JOB:UPDATE", "DELIVERY-SERVICE:UPDATE", "JOB:READ", "DELIVERY-SERVICE:READ"}, Authenticated, nil, 4861342263}, - {api.Version{Major: 4, Minor: 0}, http.MethodPost, `jobs/?`, invalidationjobs.Create, auth.PrivLevelPortal, []string{"JOB:CREATE", "JOB:READ", "DELIVERY-SERVICE:READ", "DELIVERY-SERVICE:UPDATE"}, Authenticated, nil, 404509553}, + {api.Version{Major: 4, Minor: 0}, http.MethodGet, `jobs/?$`, api.ReadHandler(&invalidationjobs.InvalidationJob{}), auth.PrivLevelReadOnly, nil, Authenticated, nil, 49667820413}, + {api.Version{Major: 4, Minor: 0}, http.MethodDelete, `jobs/?$`, invalidationjobs.Delete, auth.PrivLevelPortal, nil, Authenticated, nil, 4167807763}, + {api.Version{Major: 4, Minor: 0}, http.MethodPut, `jobs/?$`, invalidationjobs.Update, auth.PrivLevelPortal, nil, Authenticated, nil, 4861342263}, + {api.Version{Major: 4, Minor: 0}, http.MethodPost, `jobs/?`, invalidationjobs.CreateV40, auth.PrivLevelPortal, nil, Authenticated, nil, 404509553}, //Login {api.Version{Major: 4, Minor: 0}, http.MethodPost, `user/login/?$`, login.LoginHandler(d.DB, d.Config), 0, nil, NoAuth, nil, 43926708213}, From 0e47d98b479d4a9a39a1784bc4b28323d69c4898 Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Fri, 13 Aug 2021 16:36:39 -0600 Subject: [PATCH 04/35] Add Update ability for REFETCH --- lib/go-tc/invalidationjobs.go | 62 ++++- .../invalidationjobs/invalidationjobs.go | 240 +++++++++++++++++- .../traffic_ops_golang/routing/routes.go | 3 +- 3 files changed, 289 insertions(+), 16 deletions(-) diff --git a/lib/go-tc/invalidationjobs.go b/lib/go-tc/invalidationjobs.go index 4019c07187..ba9e905314 100644 --- a/lib/go-tc/invalidationjobs.go +++ b/lib/go-tc/invalidationjobs.go @@ -60,6 +60,7 @@ type InvalidationJob struct { } // InvalidationJobV40 represents a content invalidation job as returned by the API. +// Also used for Update calls. type InvalidationJobV40 struct { ID *uint64 `json:"id"` AssetURL *string `json:"assetUrl"` @@ -70,6 +71,45 @@ type InvalidationJobV40 struct { StartTime *time.Time `json:"startTime"` } +// Validate checks that the InvalidationJob is valid, by ensuring all of its fields are well-defined. +// +// This returns an error describing any and all problematic fields encountered during validation. +func (job *InvalidationJobV40) Validate() error { + errs := []string{} + err := validation.ValidateStruct(job, + validation.Field(&job.DeliveryServiceXMLID, validation.Required), + validation.Field(&job.AssetURL, validation.Required, is.URL), + validation.Field(&job.CreatedBy, validation.Required), + validation.Field(&job.ID, validation.Required), + validation.Field(&job.TTLHours, validation.Required), + validation.Field(&job.InvalidationType, validation.Required, validation.NewStringRule(func(s string) bool { + return s == REFRESH || s == REFETCH + }, fmt.Sprintf("must be either %s or %s (case sensitive)", REFRESH, REFETCH))), + ) + + if err != nil { + errs = append(errs, err.Error()) + } + + if job.StartTime == nil { + return errors.New(strings.Join(append(errs, "startTime: cannot be blank"), ", ")) + } + + if job.StartTime.After(time.Now().Add(twoDays)) { + errs = append(errs, "startTime: must be within two days from now") + } + + if job.StartTime.Before(time.Now()) { + errs = append(errs, "startTime: cannot be in the past") + } + + if len(errs) > 0 { + return errors.New(strings.Join(errs, ", ")) + } + + return nil +} + // InvalidationJobsResponse is the type of a response from Traffic Ops to a // request made to its /jobs API endpoint. type InvalidationJobsResponse struct { @@ -532,8 +572,8 @@ func (job *InvalidationJobCreateV40) Validate(tx *sql.Tx) error { errs = append(errs, "TTL is invalid: "+err.Error()) } - if err := job.validateRefetch(tx); err != nil { - errs = append(errs, "InvalidationType is invalid: "+err.Error()) + if job.InvalidationType == REFETCH && !RefetchAllowed(tx) { + errs = append(errs, "InvalidationType is invalid") } if len(errs) > 0 { @@ -556,7 +596,7 @@ func (job *InvalidationJobCreateV40) validateDeliveryService(tx *sql.Tx) error { return nil } -// validateDeliveryService ensures the supplied (required) Delivery Service XML ID exists +// validateTLLHours ensures the supplied TTL hours is within acceptable limits func (job *InvalidationJobCreateV40) validateTLLHours(tx *sql.Tx) error { var maxDays uint err := tx.QueryRow(`SELECT value FROM parameter WHERE name='maxRevalDurationDays' AND config_file='regex_revalidate.config'`).Scan(&maxDays) @@ -567,16 +607,12 @@ func (job *InvalidationJobCreateV40) validateTLLHours(tx *sql.Tx) error { return nil } -// validateDeliveryService ensures the supplied (required) Delivery Service XML ID exists -func (job *InvalidationJobCreateV40) validateRefetch(tx *sql.Tx) error { - if job.InvalidationType == REFETCH { - var refetchEnabled bool - err := tx.QueryRow(`SELECT value FROM parameter WHERE name='refetch_enabled' AND config_file='global'`).Scan(&refetchEnabled) - if err == nil && !refetchEnabled { // silently ignore other errors too - return fmt.Errorf("%s has not been enabled for this CDN", REFETCH) - } - } - return nil +// RefetchAllowed checks whether Refetch is allowed and enabled as a global paramter +func RefetchAllowed(tx *sql.Tx) bool { + refetchEnabled := false + // Silently ignore paramter error + tx.QueryRow(`SELECT value FROM parameter WHERE name='refetch_enabled' AND config_file='global'`).Scan(&refetchEnabled) + return refetchEnabled } func (job *InvalidationJobCreateV40) GetDSIDfromDSXMLID(tx *sql.Tx) (uint, error) { diff --git a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go index 52c91a39fb..9f133bbd67 100644 --- a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go +++ b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go @@ -85,7 +85,7 @@ RETURNING start_time ` -// Almost the same as the above insert, but returns appropriate values for API 4.0+ +// Almost the same as insertQuery, but returns appropriate values for API 4.0+ const insertQueryV40 = ` INSERT INTO job ( ttl_hr, @@ -170,6 +170,31 @@ RETURNING asset_url, start_time ` +// Almost the same as putInfoQuery, but returns appropriate values for API 4.0+ +const updateQueryV40 = ` +UPDATE job +SET asset_url=$1, + ttl_hr=$2, + start_time=$3, + invalidation_type=$4 +WHERE job.id=$5 +RETURNING asset_url, + ( + SELECT tm_user.username + FROM tm_user + WHERE tm_user.id=job.job_user + ) AS created_by, + ( + SELECT deliveryservice.xml_id + FROM deliveryservice + WHERE deliveryservice.id=job.job_deliveryservice + ) AS delivery_service, + job.id, + ttl_hr, + start_time, + invalidation_type +` + const putInfoQuery = ` SELECT job.id AS id, tm_user.username AS createdBy, @@ -187,6 +212,25 @@ INNER JOIN deliveryservice ON deliveryservice.id=job.job_deliveryservice WHERE job.id=$1 ` +// Almost the same as putInfoQuery, but returns appropriate values for API 4.0+ +const putInfoQueryV40 = ` +SELECT job.id AS id, + tm_user.username AS createdBy, + job.job_user AS createdByID, + job.job_deliveryservice AS dsid, + deliveryservice.xml_id AS dsxmlid, + job.asset_url AS assetURL, + job.ttl_hr AS ttlhrs, + job.start_time AS start_time, + job.invalidation_type as invalidationType, + origin.protocol || '://' || origin.fqdn || rtrim(concat(':', origin.port), ':') AS OFQDN +FROM job +INNER JOIN origin ON origin.deliveryservice=job.job_deliveryservice AND origin.is_primary +INNER JOIN tm_user ON tm_user.id=job.job_user +INNER JOIN deliveryservice ON deliveryservice.id=job.job_deliveryservice +WHERE job.id=$1 +` + const deleteQuery = ` DELETE FROM job @@ -470,7 +514,7 @@ func CreateV40(w http.ResponseWriter, r *http.Request) { if len(conflicts) > 0 { duplicate = "(duplicate) " } - changeLogMsg := fmt.Sprintf("%s content invalidation job %s- ID: %d DSXMLID: %s ASSET_URL: %s TTLHRs: %d INVALIDATION: %s", + changeLogMsg := fmt.Sprintf("%s content invalidation job %s- ID: %d DSXMLID: %s ASSET_URL: '%s' TTLHRs: %d INVALIDATION: %s", api.Created, duplicate, *result.ID, @@ -637,6 +681,198 @@ func Create(w http.ResponseWriter, r *http.Request) { // Used by PUT requests to `/jobs`, replaces an existing content invalidation job // with the provided request body. +func UpdateV40(w http.ResponseWriter, r *http.Request) { + inf, userErr, sysErr, errCode := api.NewInfo(r, nil, nil) + if userErr != nil || sysErr != nil { + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr) + return + } + defer inf.Close() + + var oFQDN string + var dsid uint + var uid uint + job := tc.InvalidationJobV40{} + row := inf.Tx.Tx.QueryRow(putInfoQueryV40, inf.Params["id"]) + err := row.Scan(&job.ID, + &job.CreatedBy, + &uid, + &dsid, + &job.DeliveryServiceXMLID, + &job.AssetURL, + &job.TTLHours, + &job.StartTime, + &job.InvalidationType, + &oFQDN) + if err != nil { + if err == sql.ErrNoRows { + userErr = fmt.Errorf("No job by id '%s'!", inf.Params["id"]) + errCode = http.StatusNotFound + } else { + sysErr = fmt.Errorf("fetching job update info: %v", err) + errCode = http.StatusInternalServerError + } + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr) + return + } + + if ok, err := IsUserAuthorizedToModifyDSID(inf, dsid); err != nil { + sysErr = fmt.Errorf("Checking user permissions on DS #%d: %v", dsid, err) + errCode = http.StatusInternalServerError + api.HandleErr(w, r, inf.Tx.Tx, errCode, nil, sysErr) + return + } else if !ok { + userErr = errors.New("No such Delivery Service!") + errCode = http.StatusNotFound + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) + return + } + + if ok, err := IsUserAuthorizedToModifyJobsMadeByUsername(inf, *job.CreatedBy); err != nil { + sysErr = fmt.Errorf("Checking user permissions against user %s: %v", *job.CreatedBy, err) + errCode = http.StatusInternalServerError + api.HandleErr(w, r, inf.Tx.Tx, errCode, nil, sysErr) + return + } else if !ok { + userErr = fmt.Errorf("No job by id '%s'!", inf.Params["id"]) + errCode = http.StatusNotFound + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) + return + } + + input := tc.InvalidationJobV40{} + if err := json.NewDecoder(r.Body).Decode(&input); err != nil { + userErr = fmt.Errorf("Unable to parse input: %v", err) + sysErr = fmt.Errorf("parsing input to PUT jobs?id=%s: %v", inf.Params["id"], err) + errCode = http.StatusBadRequest + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr) + return + } + + if err := input.Validate(); err != nil { + api.HandleErr(w, r, inf.Tx.Tx, http.StatusBadRequest, err, nil) + return + } + + if !strings.HasPrefix(*input.AssetURL, oFQDN) { + userErr = fmt.Errorf("Cannot set asset URL that does not start with Delivery Service origin URL: %s", oFQDN) + errCode = http.StatusBadRequest + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) + return + } + + if job.StartTime.Before(time.Now()) { + userErr = errors.New("Cannot modify a job that has already started!") + errCode = http.StatusMethodNotAllowed + w.Header().Set(http.CanonicalHeaderKey("allow"), "GET,HEAD,DELETE") + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) + return + } + + if *job.DeliveryServiceXMLID != *input.DeliveryServiceXMLID { + userErr = errors.New("Cannot change 'deliveryService' of existing invalidation job!") + errCode = http.StatusConflict + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) + return + } + + if *job.CreatedBy != *input.CreatedBy { + userErr = errors.New("Cannot change 'createdBy' of existing invalidation jobs!") + errCode = http.StatusConflict + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) + return + } + + if *job.ID != *input.ID { + userErr = errors.New("Cannot change an invalidation job 'id'!") + errCode = http.StatusConflict + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) + return + } + + if *job.InvalidationType != *input.InvalidationType { + if *input.InvalidationType == tc.REFETCH && !tc.RefetchAllowed(inf.Tx.Tx) { + userErr = errors.New("Invalidation Type REFRESH is disallowed") + errCode = http.StatusBadRequest + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) + return + } + } + + row = inf.Tx.Tx.QueryRow(updateQueryV40, + input.AssetURL, + input.TTLHours, + input.StartTime, + input.InvalidationType, + *job.ID) + err = row.Scan(&job.AssetURL, + &job.CreatedBy, + &job.DeliveryServiceXMLID, + &job.ID, + &job.TTLHours, + &job.StartTime, + &job.InvalidationType) + if err != nil { + sysErr = fmt.Errorf("Updating a job: %v", err) + errCode = http.StatusInternalServerError + api.HandleErr(w, r, inf.Tx.Tx, errCode, nil, sysErr) + return + } + + if err = setRevalFlags(*job.DeliveryServiceXMLID, inf.Tx.Tx); err != nil { + api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, fmt.Errorf("Setting reval flags: %v", err)) + return + } + + conflicts := tc.ValidateJobUniqueness(inf.Tx.Tx, dsid, *input.StartTime, *input.AssetURL, *input.TTLHours) + response := apiResponseV40{ + make([]tc.Alert, len(conflicts)+1), + job, + } + for i, conflict := range conflicts { + response.Alerts[i] = tc.Alert{ + Text: conflict, + Level: tc.WarnLevel.String(), + } + } + response.Alerts[len(conflicts)] = tc.Alert{ + Text: fmt.Sprintf("Invalidation request created for %s, start: %v end: %v invalidation type: %v", + *job.AssetURL, + job.StartTime, + job.StartTime.Add(time.Hour*time.Duration(*job.TTLHours)), + *job.InvalidationType), + Level: tc.SuccessLevel.String(), + } + + resp, err := json.Marshal(response) + if err != nil { + sysErr = fmt.Errorf("encoding response: %v", err) + errCode = http.StatusInternalServerError + api.HandleErr(w, r, inf.Tx.Tx, errCode, nil, sysErr) + return + } + + w.Header().Set(http.CanonicalHeaderKey("content-type"), rfc.ApplicationJSON) + api.WriteAndLogErr(w, r, append(resp, '\n')) + + changeLogMsg := fmt.Sprintf("%s content invalidation job - ID: %d DSXMLID: %s ASSET_URL: '%s' TTLHRs: %d INVALIDATION: %s", + api.Updated, + *input.ID, + *input.DeliveryServiceXMLID, + *input.AssetURL, + *input.TTLHours, + *input.InvalidationType, + ) + api.CreateChangeLogRawTx(api.ApiChange, + changeLogMsg, + inf.User, + inf.Tx.Tx) +} + +// Used by PUT requests to `/jobs`, replaces an existing content invalidation job +// with the provided request body. +// +// Deprecated. To be used only with versions less than 4.0 func Update(w http.ResponseWriter, r *http.Request) { inf, userErr, sysErr, errCode := api.NewInfo(r, nil, nil) if userErr != nil || sysErr != nil { diff --git a/traffic_ops/traffic_ops_golang/routing/routes.go b/traffic_ops/traffic_ops_golang/routing/routes.go index 71eb5afb08..adaf0220ed 100644 --- a/traffic_ops/traffic_ops_golang/routing/routes.go +++ b/traffic_ops/traffic_ops_golang/routing/routes.go @@ -216,9 +216,10 @@ func Routes(d ServerData) ([]Route, http.Handler, error) { {api.Version{Major: 4, Minor: 0}, http.MethodGet, `logs/newcount/?$`, logs.GetNewCount, auth.PrivLevelReadOnly, []string{"LOG:READ"}, Authenticated, nil, 44058330123}, //Content invalidation jobs + {api.Version{Major: 4, Minor: 0}, http.MethodGet, `jobs/?$`, api.ReadHandler(&invalidationjobs.InvalidationJob{}), auth.PrivLevelReadOnly, nil, Authenticated, nil, 49667820413}, {api.Version{Major: 4, Minor: 0}, http.MethodDelete, `jobs/?$`, invalidationjobs.Delete, auth.PrivLevelPortal, nil, Authenticated, nil, 4167807763}, - {api.Version{Major: 4, Minor: 0}, http.MethodPut, `jobs/?$`, invalidationjobs.Update, auth.PrivLevelPortal, nil, Authenticated, nil, 4861342263}, + {api.Version{Major: 4, Minor: 0}, http.MethodPut, `jobs/?$`, invalidationjobs.UpdateV40, auth.PrivLevelPortal, nil, Authenticated, nil, 4861342263}, {api.Version{Major: 4, Minor: 0}, http.MethodPost, `jobs/?`, invalidationjobs.CreateV40, auth.PrivLevelPortal, nil, Authenticated, nil, 404509553}, //Login From 087a560b32878ce02ac929e2e541f522754e3b39 Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Fri, 13 Aug 2021 19:25:14 -0600 Subject: [PATCH 05/35] Add Delete ability for REFETCH --- .../invalidationjobs/invalidationjobs.go | 132 +++++++++++++++++- .../traffic_ops_golang/routing/routes.go | 3 +- 2 files changed, 130 insertions(+), 5 deletions(-) diff --git a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go index 9f133bbd67..b213d038b8 100644 --- a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go +++ b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go @@ -170,7 +170,7 @@ RETURNING asset_url, start_time ` -// Almost the same as putInfoQuery, but returns appropriate values for API 4.0+ +// Almost the same as updateQuery, but returns appropriate values for API 4.0+ const updateQueryV40 = ` UPDATE job SET asset_url=$1, @@ -236,8 +236,7 @@ DELETE FROM job WHERE job.id=$1 RETURNING job.asset_url, - ( - SELECT tm_user.username + ( SELECT tm_user.username FROM tm_user WHERE tm_user.id=job.job_user ) AS created_by, @@ -252,6 +251,29 @@ RETURNING job.asset_url, job.start_time ` +// Almost the same as deleteQuery, but returns appropriate values for API 4.0+ +const deleteQueryV40 = ` +DELETE +FROM job +WHERE job.id=$1 +RETURNING + job.id, + job.asset_url, + ( + SELECT tm_user.username + FROM tm_user + WHERE tm_user.id=job.job_user + ) AS created_by, + ( + SELECT deliveryservice.xml_id + FROM deliveryservice + WHERE deliveryservice.id=job.job_deliveryservice + ) AS deliveryservice, + ttl_hr, + job.invalidation_type, + job.start_time +` + type apiResponse struct { Alerts []tc.Alert `json:"alerts,omitempty"` Response tc.InvalidationJob `json:"response,omitempty"` @@ -1049,6 +1071,110 @@ func Update(w http.ResponseWriter, r *http.Request) { } // Used by DELETE requests to `/jobs`, deletes an existing content invalidation job +func DeleteV40(w http.ResponseWriter, r *http.Request) { + inf, userErr, sysErr, errCode := api.NewInfo(r, []string{"id"}, []string{"id"}) + if userErr != nil || sysErr != nil { + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr) + return + } + defer inf.Close() + + var dsid uint + var createdBy uint + row := inf.Tx.Tx.QueryRow(`SELECT job_deliveryservice, job_user FROM job WHERE id=$1`, inf.Params["id"]) + if err := row.Scan(&dsid, &createdBy); err != nil { + if err == sql.ErrNoRows { + userErr = fmt.Errorf("No job by id '%s'!", inf.Params["id"]) + errCode = http.StatusNotFound + } else { + sysErr = fmt.Errorf("Getting info for job #%s: %v", inf.Params["id"], err) + errCode = http.StatusInternalServerError + } + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr) + return + } + + if ok, err := IsUserAuthorizedToModifyDSID(inf, dsid); err != nil { + sysErr = fmt.Errorf("Checking user permissions on DS #%d: %v", dsid, err) + errCode = http.StatusInternalServerError + api.HandleErr(w, r, inf.Tx.Tx, errCode, nil, sysErr) + return + } else if !ok { + userErr = errors.New("No such Delivery Service!") + errCode = http.StatusNotFound + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) + return + } + + if ok, err := IsUserAuthorizedToModifyJobsMadeByUserID(inf, createdBy); err != nil { + sysErr = fmt.Errorf("Checking user permissions against user %v: %v", createdBy, err) + errCode = http.StatusInternalServerError + api.HandleErr(w, r, inf.Tx.Tx, errCode, nil, sysErr) + return + } else if !ok { + userErr = fmt.Errorf("No job by id '%s'!", inf.Params["id"]) + errCode = http.StatusNotFound + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) + return + } + + result := tc.InvalidationJobV40{} + row = inf.Tx.Tx.QueryRow(deleteQueryV40, inf.Params["id"]) + err := row.Scan( + &result.ID, + &result.AssetURL, + &result.CreatedBy, + &result.DeliveryServiceXMLID, + &result.TTLHours, + &result.InvalidationType, + &result.StartTime) + if err != nil { + sysErr = fmt.Errorf("deleting job #%s: %v", inf.Params["id"], err) + errCode = http.StatusInternalServerError + api.HandleErr(w, r, inf.Tx.Tx, errCode, nil, sysErr) + return + } + + if err = setRevalFlags(dsid, inf.Tx.Tx); err != nil { + sysErr = fmt.Errorf("setting reval_pending after deleting job #%s: %v", inf.Params["id"], err) + errCode = http.StatusInternalServerError + api.HandleErr(w, r, inf.Tx.Tx, errCode, nil, sysErr) + return + } + + response := apiResponseV40{[]tc.Alert{ + {Text: "Content invalidation job was deleted", Level: tc.SuccessLevel.String()}, + }, + result, + } + resp, err := json.Marshal(response) + if err != nil { + sysErr = fmt.Errorf("encoding response: %v", err) + errCode = http.StatusInternalServerError + api.HandleErr(w, r, inf.Tx.Tx, errCode, nil, sysErr) + return + } + + w.Header().Set(http.CanonicalHeaderKey("content-type"), rfc.ApplicationJSON) + api.WriteAndLogErr(w, r, append(resp, '\n')) + + changeLogMsg := fmt.Sprintf("%s content invalidation job - ID: %d DSXMLID: %s ASSET_URL: '%s' TTLHRs: %d INVALIDATION: %s", + api.Deleted, + *result.ID, + *result.DeliveryServiceXMLID, + *result.AssetURL, + *result.TTLHours, + *result.InvalidationType, + ) + api.CreateChangeLogRawTx(api.ApiChange, + changeLogMsg, + inf.User, + inf.Tx.Tx) +} + +// Used by DELETE requests to `/jobs`, deletes an existing content invalidation job +// +// Deprecated. To be used only with versions less than 4.0 func Delete(w http.ResponseWriter, r *http.Request) { inf, userErr, sysErr, errCode := api.NewInfo(r, []string{"id"}, []string{"id"}) if userErr != nil || sysErr != nil { diff --git a/traffic_ops/traffic_ops_golang/routing/routes.go b/traffic_ops/traffic_ops_golang/routing/routes.go index adaf0220ed..2d2f98f015 100644 --- a/traffic_ops/traffic_ops_golang/routing/routes.go +++ b/traffic_ops/traffic_ops_golang/routing/routes.go @@ -216,9 +216,8 @@ func Routes(d ServerData) ([]Route, http.Handler, error) { {api.Version{Major: 4, Minor: 0}, http.MethodGet, `logs/newcount/?$`, logs.GetNewCount, auth.PrivLevelReadOnly, []string{"LOG:READ"}, Authenticated, nil, 44058330123}, //Content invalidation jobs - {api.Version{Major: 4, Minor: 0}, http.MethodGet, `jobs/?$`, api.ReadHandler(&invalidationjobs.InvalidationJob{}), auth.PrivLevelReadOnly, nil, Authenticated, nil, 49667820413}, - {api.Version{Major: 4, Minor: 0}, http.MethodDelete, `jobs/?$`, invalidationjobs.Delete, auth.PrivLevelPortal, nil, Authenticated, nil, 4167807763}, + {api.Version{Major: 4, Minor: 0}, http.MethodDelete, `jobs/?$`, invalidationjobs.DeleteV40, auth.PrivLevelPortal, nil, Authenticated, nil, 4167807763}, {api.Version{Major: 4, Minor: 0}, http.MethodPut, `jobs/?$`, invalidationjobs.UpdateV40, auth.PrivLevelPortal, nil, Authenticated, nil, 4861342263}, {api.Version{Major: 4, Minor: 0}, http.MethodPost, `jobs/?`, invalidationjobs.CreateV40, auth.PrivLevelPortal, nil, Authenticated, nil, 404509553}, From 8a2416d751667073ed044842da279d8d5612c93e Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Fri, 13 Aug 2021 19:54:44 -0600 Subject: [PATCH 06/35] Add Read ability for REFETCH --- .../invalidationjobs/invalidationjobs.go | 102 ++++++++++++++++++ .../traffic_ops_golang/routing/routes.go | 2 +- 2 files changed, 103 insertions(+), 1 deletion(-) diff --git a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go index b213d038b8..1529d6743e 100644 --- a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go +++ b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go @@ -42,11 +42,17 @@ import ( "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/tenant" ) +// Deprecated, only to be used with versions below 4.0 type InvalidationJob struct { api.APIInfoImpl `json:"-"` tc.InvalidationJob } +type InvalidationJobV40 struct { + api.APIInfoImpl `json:"-"` + tc.InvalidationJobV40 +} + // Deprecated, only to be used with versions below 4.0 const insertQuery = ` INSERT INTO job ( @@ -195,6 +201,7 @@ RETURNING asset_url, invalidation_type ` +// Deprecated, only to be used with versions below 4.0 const putInfoQuery = ` SELECT job.id AS id, tm_user.username AS createdBy, @@ -231,6 +238,7 @@ INNER JOIN deliveryservice ON deliveryservice.id=job.job_deliveryservice WHERE job.id=$1 ` +// Deprecated, only to be used with versions below 4.0 const deleteQuery = ` DELETE FROM job @@ -293,6 +301,7 @@ func selectMaxLastUpdatedQuery(where string) string { select max(last_updated) as t from last_deleted l where l.table_name='job') as res` } +// Deprecated, only to be used with versions below 4.0 const readQuery = ` SELECT job.id, 'PURGE' AS keyword, @@ -306,8 +315,101 @@ JOIN tm_user u ON job.job_user = u.id JOIN deliveryservice ds ON job.job_deliveryservice = ds.id ` +// Almost the same as readQuery, but returns appropriate values for API 4.0+ +const readQueryV40 = ` +SELECT job.id, + asset_url, + u.username as createdBy, + ds.xml_id, + ttl_hr, + invalidation_type, + start_time +FROM job +JOIN tm_user u ON job.job_user = u.id +JOIN deliveryservice ds ON job.job_deliveryservice = ds.id +` + // Used by GET requests to `/jobs`, simply returns a filtered list of // content invalidation jobs according to the provided query parameters. +func (job *InvalidationJobV40) Read(h http.Header, useIMS bool) ([]interface{}, error, error, int, *time.Time) { + var maxTime time.Time + var runSecond bool + queryParamsToSQLCols := map[string]dbhelpers.WhereColumnInfo{ + "id": dbhelpers.WhereColumnInfo{Column: "job.id", Checker: api.IsInt}, + "assetUrl": dbhelpers.WhereColumnInfo{Column: "asset_url"}, + "startTime": dbhelpers.WhereColumnInfo{Column: "start_time"}, + "userId": dbhelpers.WhereColumnInfo{Column: "job_user", Checker: api.IsInt}, + "createdBy": dbhelpers.WhereColumnInfo{Column: `(SELECT tm_user.username FROM tm_user WHERE tm_user.id=job.job_user)`}, + "deliveryService": dbhelpers.WhereColumnInfo{Column: `(SELECT deliveryservice.xml_id FROM deliveryservice WHERE deliveryservice.id=job.job_deliveryservice)`}, + "dsId": dbhelpers.WhereColumnInfo{Column: "job.job_deliveryservice", Checker: api.IsInt}, + "invalidationType": dbhelpers.WhereColumnInfo{Column: "invalidation_type"}, + } + + where, orderBy, pagination, queryValues, errs := dbhelpers.BuildWhereAndOrderByAndPagination(job.APIInfo().Params, queryParamsToSQLCols) + if len(errs) > 0 { + return nil, util.JoinErrs(errs), nil, http.StatusBadRequest, nil + } + + accessibleTenants, err := tenant.GetUserTenantIDListTx(job.APIInfo().Tx.Tx, job.APIInfo().User.TenantID) + if err != nil { + return nil, nil, fmt.Errorf("getting accessible tenants for user - %v", err), http.StatusInternalServerError, nil + } + if len(where) > 0 { + where += " AND ds.tenant_id = ANY(:tenants) " + } else { + where = dbhelpers.BaseWhere + " ds.tenant_id = ANY(:tenants) " + } + queryValues["tenants"] = pq.Array(accessibleTenants) + + if useIMS { + runSecond, maxTime = ims.TryIfModifiedSinceQuery(job.APIInfo().Tx, h, queryValues, selectMaxLastUpdatedQuery(where)) + if !runSecond { + log.Debugln("IMS HIT") + return []interface{}{}, nil, nil, http.StatusNotModified, &maxTime + } + log.Debugln("IMS MISS") + } else { + log.Debugln("Non IMS request") + } + + query := readQueryV40 + where + orderBy + pagination + log.Debugln("generated job query: " + query) + log.Debugf("executing with values: %++v\n", queryValues) + + returnable := []interface{}{} + rows, err := job.APIInfo().Tx.NamedQuery(query, queryValues) + if err != nil { + return nil, nil, fmt.Errorf("querying: %v", err), http.StatusInternalServerError, nil + } + defer rows.Close() + + for rows.Next() { + job := tc.InvalidationJobV40{} + err := rows.Scan(&job.ID, + &job.AssetURL, + &job.CreatedBy, + &job.DeliveryServiceXMLID, + &job.TTLHours, + &job.InvalidationType, + &job.StartTime) + if err != nil { + return nil, nil, fmt.Errorf("parsing db response: %v", err), http.StatusInternalServerError, nil + } + + returnable = append(returnable, job) + } + + if err := rows.Err(); err != nil { + return nil, nil, fmt.Errorf("Parsing db responses: %v", err), http.StatusInternalServerError, nil + } + + return returnable, nil, nil, http.StatusOK, &maxTime +} + +// Used by GET requests to `/jobs`, simply returns a filtered list of +// content invalidation jobs according to the provided query parameters. +// +// Deprecated. To be used only with versions less than 4.0 func (job *InvalidationJob) Read(h http.Header, useIMS bool) ([]interface{}, error, error, int, *time.Time) { var maxTime time.Time var runSecond bool diff --git a/traffic_ops/traffic_ops_golang/routing/routes.go b/traffic_ops/traffic_ops_golang/routing/routes.go index 2d2f98f015..78676c1247 100644 --- a/traffic_ops/traffic_ops_golang/routing/routes.go +++ b/traffic_ops/traffic_ops_golang/routing/routes.go @@ -216,7 +216,7 @@ func Routes(d ServerData) ([]Route, http.Handler, error) { {api.Version{Major: 4, Minor: 0}, http.MethodGet, `logs/newcount/?$`, logs.GetNewCount, auth.PrivLevelReadOnly, []string{"LOG:READ"}, Authenticated, nil, 44058330123}, //Content invalidation jobs - {api.Version{Major: 4, Minor: 0}, http.MethodGet, `jobs/?$`, api.ReadHandler(&invalidationjobs.InvalidationJob{}), auth.PrivLevelReadOnly, nil, Authenticated, nil, 49667820413}, + {api.Version{Major: 4, Minor: 0}, http.MethodGet, `jobs/?$`, api.ReadHandler(&invalidationjobs.InvalidationJobV40{}), auth.PrivLevelReadOnly, nil, Authenticated, nil, 49667820413}, {api.Version{Major: 4, Minor: 0}, http.MethodDelete, `jobs/?$`, invalidationjobs.DeleteV40, auth.PrivLevelPortal, nil, Authenticated, nil, 4167807763}, {api.Version{Major: 4, Minor: 0}, http.MethodPut, `jobs/?$`, invalidationjobs.UpdateV40, auth.PrivLevelPortal, nil, Authenticated, nil, 4861342263}, {api.Version{Major: 4, Minor: 0}, http.MethodPost, `jobs/?`, invalidationjobs.CreateV40, auth.PrivLevelPortal, nil, Authenticated, nil, 404509553}, From a57766a64b031662039f74ae556bc03e6f2e93a8 Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Fri, 13 Aug 2021 20:40:23 -0600 Subject: [PATCH 07/35] Change mod file to match master, run go vendor --- go.sum | 9 +++++++++ vendor/github.com/jmoiron/sqlx/go.mod | 7 +++++++ vendor/github.com/jmoiron/sqlx/go.sum | 6 ++++++ vendor/github.com/lib/pq/go.mod | 3 +++ vendor/modules.txt | 8 ++++++++ 5 files changed, 33 insertions(+) create mode 100644 vendor/github.com/jmoiron/sqlx/go.mod create mode 100644 vendor/github.com/jmoiron/sqlx/go.sum create mode 100644 vendor/github.com/lib/pq/go.mod diff --git a/go.sum b/go.sum index 630f3735db..161c75b04a 100644 --- a/go.sum +++ b/go.sum @@ -205,6 +205,7 @@ github.com/hydrogen18/stoppableListener v0.0.0-20151210151943-dadc9ccc400c/go.mo github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/influxdata/influxdb v1.1.1-0.20170104212736-6a94d200c826 h1:F/ZGpGV51zWV9cpTGdCikuptiAwG1NK747a80m0yK+s= github.com/influxdata/influxdb v1.1.1-0.20170104212736-6a94d200c826/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= +<<<<<<< HEAD github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= @@ -230,6 +231,8 @@ github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQ github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +======= +>>>>>>> 7ab736e82... Change mod file to match master, run go vendor github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/json-iterator/go v1.1.6-0.20181024152841-05d041de1043 h1:XWlFVbIXnWhOnWnMMUyVPElTFYoLeuO0TAarw7Jvdco= @@ -260,6 +263,7 @@ github.com/lestrrat-go/jwx v0.9.1-0.20190702045520-e35178ac2b1f/go.mod h1:iEoxlY github.com/lestrrat/go-jwx v0.0.0-20171104074836-2857e17763b6 h1:Cs+QtIlpsE/GOhCr/54qhec3cg9a+IjOgqdhNl1yumA= github.com/lestrrat/go-jwx v0.0.0-20171104074836-2857e17763b6/go.mod h1:T+yHdCP6MJKtzoVQMHvVCeam5VFwX1+rWzn5zZgKYMI= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +<<<<<<< HEAD github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= @@ -272,6 +276,11 @@ github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +======= +github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= +github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +>>>>>>> 7ab736e82... Change mod file to match master, run go vendor github.com/mattn/go-sqlite3 v1.14.5 h1:1IdxlwTNazvbKJQSxoJ5/9ECbEeaTTyeU7sEAZ5KKTQ= github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI= github.com/miekg/dns v1.0.6-0.20180406150955-01d59357d468 h1:ABwS8bIszuZkM3o4nP3Hw3rz9TGkMW9lV4Ifp1hMEaI= diff --git a/vendor/github.com/jmoiron/sqlx/go.mod b/vendor/github.com/jmoiron/sqlx/go.mod new file mode 100644 index 0000000000..66c67561cc --- /dev/null +++ b/vendor/github.com/jmoiron/sqlx/go.mod @@ -0,0 +1,7 @@ +module github.com/jmoiron/sqlx + +require ( + github.com/go-sql-driver/mysql v1.4.0 + github.com/lib/pq v1.0.0 + github.com/mattn/go-sqlite3 v1.9.0 +) diff --git a/vendor/github.com/jmoiron/sqlx/go.sum b/vendor/github.com/jmoiron/sqlx/go.sum new file mode 100644 index 0000000000..a3239ada75 --- /dev/null +++ b/vendor/github.com/jmoiron/sqlx/go.sum @@ -0,0 +1,6 @@ +github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= diff --git a/vendor/github.com/lib/pq/go.mod b/vendor/github.com/lib/pq/go.mod new file mode 100644 index 0000000000..b5a5639ab6 --- /dev/null +++ b/vendor/github.com/lib/pq/go.mod @@ -0,0 +1,3 @@ +module github.com/lib/pq + +go 1.13 diff --git a/vendor/modules.txt b/vendor/modules.txt index 8361571ac6..d80f361857 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -130,12 +130,20 @@ github.com/lestrrat/go-jwx/internal/emap github.com/lestrrat/go-jwx/jwa github.com/lestrrat/go-jwx/jwk # github.com/lib/pq v1.8.0 +<<<<<<< HEAD ## explicit; go 1.13 +======= +## explicit +>>>>>>> 7ab736e82... Change mod file to match master, run go vendor github.com/lib/pq github.com/lib/pq/oid github.com/lib/pq/scram # github.com/mattn/go-sqlite3 v1.14.5 +<<<<<<< HEAD ## explicit; go 1.10 +======= +## explicit +>>>>>>> 7ab736e82... Change mod file to match master, run go vendor # github.com/miekg/dns v1.0.6-0.20180406150955-01d59357d468 ## explicit github.com/miekg/dns From 9e75e9e6df3a2af8cdb3710255d0ad6349f8a00b Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Fri, 13 Aug 2021 20:45:00 -0600 Subject: [PATCH 08/35] Add go sum file to commit --- go.sum | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/go.sum b/go.sum index 161c75b04a..8dd8dacbb0 100644 --- a/go.sum +++ b/go.sum @@ -206,6 +206,9 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/influxdata/influxdb v1.1.1-0.20170104212736-6a94d200c826 h1:F/ZGpGV51zWV9cpTGdCikuptiAwG1NK747a80m0yK+s= github.com/influxdata/influxdb v1.1.1-0.20170104212736-6a94d200c826/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 7d0002a6e... Add go sum file to commit github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= @@ -231,8 +234,11 @@ github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQ github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +<<<<<<< HEAD ======= >>>>>>> 7ab736e82... Change mod file to match master, run go vendor +======= +>>>>>>> 7d0002a6e... Add go sum file to commit github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/json-iterator/go v1.1.6-0.20181024152841-05d041de1043 h1:XWlFVbIXnWhOnWnMMUyVPElTFYoLeuO0TAarw7Jvdco= @@ -263,7 +269,6 @@ github.com/lestrrat-go/jwx v0.9.1-0.20190702045520-e35178ac2b1f/go.mod h1:iEoxlY github.com/lestrrat/go-jwx v0.0.0-20171104074836-2857e17763b6 h1:Cs+QtIlpsE/GOhCr/54qhec3cg9a+IjOgqdhNl1yumA= github.com/lestrrat/go-jwx v0.0.0-20171104074836-2857e17763b6/go.mod h1:T+yHdCP6MJKtzoVQMHvVCeam5VFwX1+rWzn5zZgKYMI= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -<<<<<<< HEAD github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= @@ -276,11 +281,9 @@ github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -======= github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= ->>>>>>> 7ab736e82... Change mod file to match master, run go vendor github.com/mattn/go-sqlite3 v1.14.5 h1:1IdxlwTNazvbKJQSxoJ5/9ECbEeaTTyeU7sEAZ5KKTQ= github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI= github.com/miekg/dns v1.0.6-0.20180406150955-01d59357d468 h1:ABwS8bIszuZkM3o4nP3Hw3rz9TGkMW9lV4Ifp1hMEaI= From 18dc3bfb44b942316245abbf255bc0ad41e5ed0d Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Fri, 13 Aug 2021 21:05:17 -0600 Subject: [PATCH 09/35] Add DB Migration scripts --- .../2021081321025500_refetch.down.sql | 143 ++++++++++++++++++ .../2021081321025500_refetch.up.sql | 65 ++++++++ 2 files changed, 208 insertions(+) create mode 100644 traffic_ops/app/db/migrations/2021081321025500_refetch.down.sql create mode 100644 traffic_ops/app/db/migrations/2021081321025500_refetch.up.sql diff --git a/traffic_ops/app/db/migrations/2021081321025500_refetch.down.sql b/traffic_ops/app/db/migrations/2021081321025500_refetch.down.sql new file mode 100644 index 0000000000..f39fc30d83 --- /dev/null +++ b/traffic_ops/app/db/migrations/2021081321025500_refetch.down.sql @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +-- Revert + +/** + * Restore job_agent table with constraints and data. + */ + +CREATE TABLE public.job_agent ( + id bigserial NOT NULL, + "name" text NULL, + description text NULL, + active int4 NOT NULL DEFAULT 0, + last_updated timestamptz NOT NULL DEFAULT now(), + CONSTRAINT idx_job_agent_id_primary PRIMARY KEY (id), + CONSTRAINT job_agent_name_unique UNIQUE (name) +); +CREATE INDEX job_agent_last_updated_idx ON public.job_agent USING btree (last_updated DESC NULLS LAST); + +-- Table Triggers + +create trigger on_delete_current_timestamp after +delete + on + public.job_agent for each row execute function on_delete_current_timestamp_last_updated('job_agent'); +create trigger on_update_current_timestamp before +update + on + public.job_agent for each row execute function on_update_current_timestamp_last_updated(); + +-- Permissions + +ALTER TABLE public.job_agent OWNER TO traffic_ops; +GRANT ALL ON TABLE public.job_agent TO traffic_ops; + +-- Data + +INSERT INTO public.job_agent (name, description, active) +VALUES ('Example', 'Description of Purge Agent', 1) +ON CONFLICT (name) DO NOTHING; + +/** + * Restore job_status table with constraints and data + */ + +CREATE TABLE public.job_status ( + id bigserial NOT NULL, + "name" text NULL, + description text NULL, + last_updated timestamptz NOT NULL DEFAULT now(), + CONSTRAINT idx_job_status_id_primary PRIMARY KEY (id), + CONSTRAINT job_status_name_unique UNIQUE (name) +); +CREATE INDEX job_status_last_updated_idx ON public.job_status USING btree (last_updated DESC NULLS LAST); + +-- Table Triggers + +create trigger on_delete_current_timestamp after +delete + on + public.job_status for each row execute function on_delete_current_timestamp_last_updated('job_status'); +create trigger on_update_current_timestamp before +update + on + public.job_status for each row execute function on_update_current_timestamp_last_updated(); + +-- Permissions + +ALTER TABLE public.job_status OWNER TO traffic_ops; +GRANT ALL ON TABLE public.job_status TO traffic_ops; + +-- Data + +INSERT INTO job_status (name, description) +VALUES ('PENDING', 'Job is queued, but has not been picked up by any agents yet'), +('IN_PROGRESS', 'Job is being processed by agents'), +('COMPLETED', 'Job has finished'), +('CANCELLED', 'Job was cancelled') +ON CONFLICT (name) DO NOTHING; + +/** + * Restore job table with constraints and data + */ + +-- Restore table +ALTER TABLE public.job +ADD COLUMN IF NOT EXISTS agent int8 NULL, +ADD COLUMN IF NOT EXISTS object_type text NULL, +ADD COLUMN IF NOT EXISTS object_name text NULL, +ADD COLUMN IF NOT EXISTS keyword text NULL, +ADD COLUMN IF NOT EXISTS asset_type text NULL, +ADD COLUMN IF NOT EXISTS status int8 NULL; + +ALTER TABLE public.job +DROP COLUMN IF EXISTS invalidation_type CASCADE; + +-- Restore indices and foreign key constraints +CREATE INDEX idx_job_fk_job_agent_id ON public.job USING btree (agent); +ALTER TABLE public.job ADD CONSTRAINT job_fk_job_agent_id FOREIGN KEY (agent) REFERENCES public.job_agent(id) ON DELETE CASCADE; + +CREATE INDEX idx_job_fk_job_status_id ON public.job USING btree (status); +ALTER TABLE public.job ADD CONSTRAINT job_fk_job_status_id FOREIGN KEY (status) REFERENCES public.job_status(id); + +-- Restore data +UPDATE public.job +SET agent = 1; + +UPDATE public.job +SET status = 1; +ALTER TABLE public.job +ALTER COLUMN status SET NOT NULL; + +UPDATE public.job +SET keyword = 'PURGE'; +ALTER TABLE public.job +ALTER COLUMN keyword SET NOT NULL; + +UPDATE public.job +SET asset_type = 'file'; +ALTER TABLE public.job +ALTER COLUMN asset_type SET NOT NULL; + +ALTER TABLE public.job +RENAME COLUMN ttl_hr TO parameters; + +ALTER TABLE public.job +ALTER COLUMN parameters TYPE TEXT +USING CONCAT('TTL:', parameters, 'h'); diff --git a/traffic_ops/app/db/migrations/2021081321025500_refetch.up.sql b/traffic_ops/app/db/migrations/2021081321025500_refetch.up.sql new file mode 100644 index 0000000000..c25c796d64 --- /dev/null +++ b/traffic_ops/app/db/migrations/2021081321025500_refetch.up.sql @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +-- Migrate + +/** + * Alter job_agent table and data + */ + +DROP TABLE IF EXISTS public.job_agent CASCADE; + +/** + * Alter job_status table and data + */ + +DROP TABLE IF EXISTS public.job_status CASCADE; + +/** + * Alter job table and data + */ + +ALTER TABLE public.job +DROP COLUMN IF EXISTS agent CASCADE, +DROP COLUMN IF EXISTS object_type CASCADE, +DROP COLUMN IF EXISTS object_name CASCADE, +DROP COLUMN IF EXISTS keyword CASCADE, +DROP COLUMN IF EXISTS asset_type CASCADE, +DROP COLUMN IF EXISTS status CASCADE; + +ALTER TABLE public.job +ADD COLUMN IF NOT EXISTS invalidation_type text NOT NULL DEFAULT 'REFRESH'; + +/* + * If the asset_url contains the temporary fix for refetch + * (adding ##REFETCH## to the end of the url) then assign the + * correct invalidation_type. + */ +UPDATE public.job +SET invalidation_type = 'REFETCH' +WHERE id IN ( + SELECT id + FROM public.job + WHERE asset_url LIKE '%##REFETCH##' +); + +ALTER TABLE public.job +RENAME COLUMN parameters TO ttl_hr; + +ALTER TABLE public.job +ALTER COLUMN ttl_hr TYPE integer +USING CAST(substring(ttl_hr,'TTL:([0-9]+)h') AS integer); From 2a5640f2017dc4b2f054336159627bbdb3ec5f77 Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Mon, 16 Aug 2021 14:16:22 -0600 Subject: [PATCH 10/35] Update v4 client API tests to utilize modified /jobs endpoints with REFETCH capability --- lib/go-tc/invalidationjobs.go | 109 ++--- traffic_ops/testing/api/v4/jobs_test.go | 419 ++++++++---------- traffic_ops/testing/api/v4/tc-fixtures.json | 39 +- traffic_ops/testing/api/v4/todb_test.go | 46 +- .../testing/api/v4/traffic_control_test.go | 2 +- .../invalidationjobs/invalidationjobs.go | 20 +- traffic_ops/v4-client/job.go | 8 +- 7 files changed, 263 insertions(+), 380 deletions(-) diff --git a/lib/go-tc/invalidationjobs.go b/lib/go-tc/invalidationjobs.go index ba9e905314..39c5425c17 100644 --- a/lib/go-tc/invalidationjobs.go +++ b/lib/go-tc/invalidationjobs.go @@ -59,57 +59,6 @@ type InvalidationJob struct { StartTime *Time `json:"startTime"` } -// InvalidationJobV40 represents a content invalidation job as returned by the API. -// Also used for Update calls. -type InvalidationJobV40 struct { - ID *uint64 `json:"id"` - AssetURL *string `json:"assetUrl"` - CreatedBy *string `json:"createdBy"` - DeliveryServiceXMLID *string `json:"deliveryService"` - TTLHours *uint `json:"ttlHours"` - InvalidationType *string `json:"invalidationType"` - StartTime *time.Time `json:"startTime"` -} - -// Validate checks that the InvalidationJob is valid, by ensuring all of its fields are well-defined. -// -// This returns an error describing any and all problematic fields encountered during validation. -func (job *InvalidationJobV40) Validate() error { - errs := []string{} - err := validation.ValidateStruct(job, - validation.Field(&job.DeliveryServiceXMLID, validation.Required), - validation.Field(&job.AssetURL, validation.Required, is.URL), - validation.Field(&job.CreatedBy, validation.Required), - validation.Field(&job.ID, validation.Required), - validation.Field(&job.TTLHours, validation.Required), - validation.Field(&job.InvalidationType, validation.Required, validation.NewStringRule(func(s string) bool { - return s == REFRESH || s == REFETCH - }, fmt.Sprintf("must be either %s or %s (case sensitive)", REFRESH, REFETCH))), - ) - - if err != nil { - errs = append(errs, err.Error()) - } - - if job.StartTime == nil { - return errors.New(strings.Join(append(errs, "startTime: cannot be blank"), ", ")) - } - - if job.StartTime.After(time.Now().Add(twoDays)) { - errs = append(errs, "startTime: must be within two days from now") - } - - if job.StartTime.Before(time.Now()) { - errs = append(errs, "startTime: cannot be in the past") - } - - if len(errs) > 0 { - return errors.New(strings.Join(errs, ", ")) - } - - return nil -} - // InvalidationJobsResponse is the type of a response from Traffic Ops to a // request made to its /jobs API endpoint. type InvalidationJobsResponse struct { @@ -517,6 +466,13 @@ func (job *UserInvalidationJobInput) Validate(tx *sql.Tx) error { const REFRESH = "REFRESH" const REFETCH = "REFETCH" +// InvalidationJobsResponse is the type of a response from Traffic Ops to a +// request made to its /jobs API endpoint for v 4.0+ +type InvalidationJobsResponseV40 struct { + Response []InvalidationJobV40 `json:"response"` + Alerts +} + // InvalidationJobCreateV40 represents user input intending to create a content invalidation job. type InvalidationJobCreateV40 struct { // The Delivery Service XML-ID for which the Invalidation Job is to be applied. @@ -583,6 +539,57 @@ func (job *InvalidationJobCreateV40) Validate(tx *sql.Tx) error { return nil } +// InvalidationJobV40 represents a content invalidation job as returned by the API. +// Also used for Update calls. +type InvalidationJobV40 struct { + ID *uint64 `json:"id"` + AssetURL *string `json:"assetUrl"` + CreatedBy *string `json:"createdBy"` + DeliveryService *string `json:"deliveryService"` + TTLHours *uint `json:"ttlHours"` + InvalidationType *string `json:"invalidationType"` + StartTime *time.Time `json:"startTime"` +} + +// Validate checks that the InvalidationJob is valid, by ensuring all of its fields are well-defined. +// +// This returns an error describing any and all problematic fields encountered during validation. +func (job *InvalidationJobV40) Validate() error { + errs := []string{} + err := validation.ValidateStruct(job, + validation.Field(&job.DeliveryService, validation.Required), + validation.Field(&job.AssetURL, validation.Required, is.URL), + validation.Field(&job.CreatedBy, validation.Required), + validation.Field(&job.ID, validation.Required), + validation.Field(&job.TTLHours, validation.Required), + validation.Field(&job.InvalidationType, validation.Required, validation.NewStringRule(func(s string) bool { + return s == REFRESH || s == REFETCH + }, fmt.Sprintf("must be either %s or %s (case sensitive)", REFRESH, REFETCH))), + ) + + if err != nil { + errs = append(errs, err.Error()) + } + + if job.StartTime == nil { + return errors.New(strings.Join(append(errs, "startTime: cannot be blank"), ", ")) + } + + if job.StartTime.After(time.Now().Add(twoDays)) { + errs = append(errs, "startTime: must be within two days from now") + } + + if job.StartTime.Before(time.Now()) { + errs = append(errs, "startTime: cannot be in the past") + } + + if len(errs) > 0 { + return errors.New(strings.Join(errs, ", ")) + } + + return nil +} + // validateDeliveryService ensures the supplied (required) Delivery Service XML ID exists func (job *InvalidationJobCreateV40) validateDeliveryService(tx *sql.Tx) error { var exists bool diff --git a/traffic_ops/testing/api/v4/jobs_test.go b/traffic_ops/testing/api/v4/jobs_test.go index 047ddc75dc..29ce604524 100644 --- a/traffic_ops/testing/api/v4/jobs_test.go +++ b/traffic_ops/testing/api/v4/jobs_test.go @@ -23,7 +23,6 @@ import ( "time" "github.com/apache/trafficcontrol/lib/go-tc" - "github.com/apache/trafficcontrol/lib/go-util" client "github.com/apache/trafficcontrol/traffic_ops/v4-client" ) @@ -58,19 +57,17 @@ func CreateTestJobs(t *testing.T) { } for i, job := range testData.InvalidationJobs { - job.StartTime = &tc.Time{ - Time: time.Now().Add(time.Minute).UTC(), - Valid: true, - } + job.StartTime = time.Now().Add(time.Minute).UTC() testData.InvalidationJobs[i] = job } for _, job := range testData.InvalidationJobs { - request := tc.InvalidationJobInput{ - DeliveryService: job.DeliveryService, - Regex: job.Regex, - StartTime: job.StartTime, - TTL: job.TTL, + request := tc.InvalidationJobCreateV40{ + DeliveryService: job.DeliveryService, + Regex: job.Regex, + StartTime: job.StartTime, + TTLHours: job.TTLHours, + InvalidationType: job.InvalidationType, } resp, _, err := TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err != nil { @@ -88,15 +85,12 @@ func JobCollisionWarningTest(t *testing.T) { } xmlID := *testData.DeliveryServices[0].XMLID - startTime := tc.Time{ - Time: time.Now().Add(time.Hour), - Valid: true, - } - firstJob := tc.InvalidationJobInput{ - DeliveryService: util.InterfacePtr(&xmlID), - Regex: util.StrPtr(`/\.*([A-Z]0?)`), - TTL: util.InterfacePtr(16), - StartTime: &startTime, + firstJob := tc.InvalidationJobCreateV40{ + DeliveryService: xmlID, + Regex: `/\.*([A-Z]0?)`, + TTLHours: 16, + StartTime: time.Now().Add(time.Hour), + InvalidationType: tc.REFRESH, } resp, _, err := TOSession.CreateInvalidationJob(firstJob, client.RequestOptions{}) @@ -104,15 +98,12 @@ func JobCollisionWarningTest(t *testing.T) { t.Fatalf("Unexpected error creating a content invalidation Job: %v - alerts: %+v", err, resp.Alerts) } - newTime := tc.Time{ - Time: startTime.Time.Add(time.Hour), - Valid: true, - } - newJob := tc.InvalidationJobInput{ - DeliveryService: firstJob.DeliveryService, - Regex: firstJob.Regex, - TTL: firstJob.TTL, - StartTime: &newTime, + newJob := tc.InvalidationJobCreateV40{ + DeliveryService: firstJob.DeliveryService, + Regex: firstJob.Regex, + TTLHours: firstJob.TTLHours, + StartTime: firstJob.StartTime.Add(time.Hour), + InvalidationType: tc.REFRESH, } alerts, _, err := TOSession.CreateInvalidationJob(newJob, client.RequestOptions{}) @@ -126,7 +117,7 @@ func JobCollisionWarningTest(t *testing.T) { found := false for _, alert := range alerts.Alerts { - if alert.Level == tc.WarnLevel.String() && strings.Contains(alert.Text, *firstJob.Regex) { + if alert.Level == tc.WarnLevel.String() && strings.Contains(alert.Text, firstJob.Regex) { found = true } } @@ -141,13 +132,13 @@ func JobCollisionWarningTest(t *testing.T) { t.Fatalf("unable to get invalidation jobs: %v - alerts: %+v", err, jobs.Alerts) } - var realJob *tc.InvalidationJob + var realJob *tc.InvalidationJobV40 for i, job := range jobs.Response { if job.StartTime == nil || job.DeliveryService == nil || job.CreatedBy == nil { t.Error("Traffic Ops returned a representation of a content invalidation Job that had null or undefined Start Time and/or Delivery Service and/or Created By") continue } - diff := newJob.StartTime.Time.Sub(job.StartTime.Time) + diff := newJob.StartTime.Sub(*job.StartTime) if *job.DeliveryService == xmlID && *job.CreatedBy == "admin" && diff < time.Second { realJob = &jobs.Response[i] break @@ -158,8 +149,8 @@ func JobCollisionWarningTest(t *testing.T) { t.Fatal("could not find new job") } - newTime.Time = startTime.Time.Add(time.Hour * 2) - realJob.StartTime = &newTime + time := firstJob.StartTime.Add(time.Hour * 2) + realJob.StartTime = &time alerts, _, err = TOSession.UpdateInvalidationJob(*realJob, client.RequestOptions{}) if err != nil { t.Fatalf("expected invalidation job update to succeed: %v - alerts: %+v", err, alerts.Alerts) @@ -171,7 +162,7 @@ func JobCollisionWarningTest(t *testing.T) { found = false for _, alert := range alerts.Alerts { - if alert.Level == tc.WarnLevel.String() && strings.Contains(alert.Text, *firstJob.Regex) { + if alert.Level == tc.WarnLevel.String() && strings.Contains(alert.Text, firstJob.Regex) { found = true } } @@ -195,10 +186,7 @@ func CreateTestInvalidationJobs(t *testing.T) { } for _, job := range testData.InvalidationJobs { - if job.DeliveryService == nil { - t.Error("Found a Job in the test data that has null or undefined Delivery Service") - } - _, ok := dsNameIDs[(*job.DeliveryService).(string)] + _, ok := dsNameIDs[job.DeliveryService] if !ok { t.Fatalf("can't create test data job: delivery service '%v' not found in Traffic Ops", job.DeliveryService) } @@ -226,10 +214,7 @@ func CreateTestInvalidJob(t *testing.T) { t.Fatal("Need at least one Invalidation Job to test creating an invalid Job") } job := testData.InvalidationJobs[0] - if job.DeliveryService == nil { - t.Fatal("Found a Job in the testing data that has null or undefined Delivery Service") - } - _, ok := dsNameIDs[(*job.DeliveryService).(string)] + _, ok := dsNameIDs[job.DeliveryService] if !ok { t.Fatalf("can't create test data job: delivery service '%v' not found in Traffic Ops", job.DeliveryService) } @@ -248,8 +233,8 @@ func CreateTestInvalidJob(t *testing.T) { if !foundMaxRevalDays { t.Fatalf("expected: parameter named maxRevalDurationDays, actual: not found") } - tooHigh := interface{}((maxRevalDays * 24) + 1) - job.TTL = &tooHigh + tooHigh := (uint32)(maxRevalDays*24) + 1 + job.TTLHours = tooHigh _, reqInf, err := TOSession.CreateInvalidationJob(job, client.RequestOptions{}) if err == nil { t.Error("creating invalid job (TTL higher than maxRevalDurationDays) - expected: error, actual: nil error") @@ -292,15 +277,8 @@ func GetTestJobs(t *testing.T) { t.Fatalf("cannot get Delivery Services: %v - alerts: %+v", err, toDSes.Alerts) } - for i, testJob := range testData.InvalidationJobs { + for _, testJob := range testData.InvalidationJobs { found := false - if testJob.DeliveryService == nil { - t.Errorf("test job (index %v) has nil delivery service", i) - continue - } else if testJob.Regex == nil { - t.Errorf("test job (index %v) has nil regex", i) - continue - } for j, toJob := range toJobs.Response { if toJob.DeliveryService == nil { t.Errorf("to job (index %v) has nil delivery service", j) @@ -310,23 +288,23 @@ func GetTestJobs(t *testing.T) { t.Errorf("to job (index %v) has nil asset url", j) continue } - if *toJob.DeliveryService != *testJob.DeliveryService { + if *toJob.DeliveryService != testJob.DeliveryService { continue } - if !strings.HasSuffix(*toJob.AssetURL, *testJob.Regex) { + if !strings.HasSuffix(*toJob.AssetURL, testJob.Regex) { continue } toJobTime := toJob.StartTime.Round(time.Minute) testJobTime := testJob.StartTime.Round(time.Minute) if !toJobTime.Equal(testJobTime) { - t.Errorf("test job ds %v regex %s start time expected '%+v' actual '%+v'", *testJob.DeliveryService, *testJob.Regex, testJobTime, toJobTime) + t.Errorf("test job ds %v regex %s start time expected '%+v' actual '%+v'", testJob.DeliveryService, testJob.Regex, testJobTime, toJobTime) continue } found = true break } if !found { - t.Errorf("test job ds %v regex %s expected: exists, actual: not found", *testJob.DeliveryService, *testJob.Regex) + t.Errorf("test job ds %v regex %s expected: exists, actual: not found", testJob.DeliveryService, testJob.Regex) } } } @@ -354,10 +332,10 @@ func GetTestInvalidationJobs(t *testing.T) { for _, testJob := range testData.InvalidationJobs { found := false for _, toJob := range jobs.Response { - if *toJob.DeliveryService != (*testJob.DeliveryService).(string) { + if *toJob.DeliveryService != testJob.DeliveryService { continue } - if !strings.HasSuffix(*toJob.AssetURL, *testJob.Regex) { + if !strings.HasSuffix(*toJob.AssetURL, testJob.Regex) { continue } if !toJob.StartTime.Round(time.Minute).Equal(testJob.StartTime.Round(time.Minute)) { @@ -387,7 +365,7 @@ func GetTestJobsByValidData(t *testing.T) { createdBy := jobs.CreatedBy id := jobs.ID dsName := jobs.DeliveryService - keyword := jobs.Keyword + invalidationType := jobs.InvalidationType //Get Jobs by Asset URL if len(*assetUrl) > 0 { @@ -425,10 +403,10 @@ func GetTestJobsByValidData(t *testing.T) { t.Errorf("ID Field is empty, so can't test get jobs %d", *id) } - //Get Jobs by Keyword - if keyword != nil && len(*keyword) > 1 { + //Get Jobs by Invalidation Type + if len(*invalidationType) > 1 { opts := client.NewRequestOptions() - opts.QueryParameters.Set("Keyword", *keyword) + opts.QueryParameters.Set("InvalidationType", *invalidationType) toJobs, _, _ = TOSession.GetInvalidationJobs(opts) if len(toJobs.Response) < 1 { t.Errorf("Expected atleast one Jobs response for GET Jobs by keyword, but found %d ", len(toJobs.Response)) @@ -508,7 +486,7 @@ func GetTestJobsByValidData(t *testing.T) { t.Errorf("GET /jobs by maxRevalDurationDays - expected at least 1 job") } for _, j := range maxRevalJobs.Response { - if time.Since((*j.StartTime).Time) > time.Duration(maxRevalDurationDays)*24*time.Hour { + if time.Since(*j.StartTime) > time.Duration(maxRevalDurationDays)*24*time.Hour { t.Errorf("GET /jobs by maxRevalDurationDays returned job that is older than %d days: {%s, %s, %v}", maxRevalDurationDays, *j.DeliveryService, *j.AssetURL, *j.StartTime) } } @@ -566,9 +544,9 @@ func GetTestJobsByInvalidData(t *testing.T) { t.Errorf("Expected no response from Get Jobs by Invalid ID, but found %d ", len(toJobs.Response)) } - //Get Jobs by Invalid Keyword + //Get Jobs by Invalid Invalidation Type opts = client.NewRequestOptions() - opts.QueryParameters.Set("keyword", "invalid") + opts.QueryParameters.Set("invalidationType", "invalid") toJobs, _, _ = TOSession.GetInvalidationJobs(opts) if len(toJobs.Response) != 0 { t.Errorf("Expected no response from Get Jobs by Invalid Keyword, but found %d ", len(toJobs.Response)) @@ -604,32 +582,31 @@ func CreateTestJobsInvalidDS(t *testing.T) { t.Error("Need at least one Invalidation Jobs to test invalid ds") } job := testData.InvalidationJobs[0] - job.StartTime = &tc.Time{ - Time: time.Now().Add(time.Minute).UTC(), - Valid: true, - } + job.StartTime = time.Now().Add(time.Minute).UTC() testData.InvalidationJobs[0] = job - //Invalid DS - request := tc.InvalidationJobInput{ - DeliveryService: util.InterfacePtr("invalid"), - Regex: job.Regex, - StartTime: job.StartTime, - TTL: job.TTL, + //Invalid DS XML ID (Does not exist) + request := tc.InvalidationJobCreateV40{ + DeliveryService: "invalid", + Regex: job.Regex, + StartTime: job.StartTime, + TTLHours: job.TTLHours, + InvalidationType: job.InvalidationType, } resp, reqInf, err := TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err == nil { t.Errorf("Expected No DeliveryService exists matching identifier: %v - alerts: %v", request.DeliveryService, resp.Alerts) } - if reqInf.StatusCode != http.StatusBadRequest { - t.Errorf("Expected status code 400, got %v", reqInf.StatusCode) + if reqInf.StatusCode != http.StatusNotFound { + t.Errorf("Expected status code 404, got %v", reqInf.StatusCode) } //Missing DS - request = tc.InvalidationJobInput{ - Regex: job.Regex, - StartTime: job.StartTime, - TTL: job.TTL, + request = tc.InvalidationJobCreateV40{ + Regex: job.Regex, + StartTime: job.StartTime, + TTLHours: job.TTLHours, + InvalidationType: job.InvalidationType, } resp, reqInf, err = TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err == nil { @@ -640,11 +617,12 @@ func CreateTestJobsInvalidDS(t *testing.T) { } //Empty DS - request = tc.InvalidationJobInput{ - DeliveryService: util.InterfacePtr(""), - Regex: job.Regex, - StartTime: job.StartTime, - TTL: job.TTL, + request = tc.InvalidationJobCreateV40{ + DeliveryService: "", + Regex: job.Regex, + StartTime: job.StartTime, + TTLHours: job.TTLHours, + InvalidationType: job.InvalidationType, } resp, reqInf, err = TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err == nil { @@ -660,17 +638,15 @@ func CreateTestJobsAlreadyExistTTL(t *testing.T) { t.Error("Need at least one Invalidation Jobs to create duplicate data") } job := testData.InvalidationJobs[0] - job.StartTime = &tc.Time{ - Time: time.Now().Add(time.Minute).UTC(), - Valid: true, - } + job.StartTime = time.Now().Add(time.Minute).UTC() testData.InvalidationJobs[0] = job - request := tc.InvalidationJobInput{ - DeliveryService: job.DeliveryService, - Regex: job.Regex, - StartTime: job.StartTime, - TTL: job.TTL, + request := tc.InvalidationJobCreateV40{ + DeliveryService: job.DeliveryService, + Regex: job.Regex, + StartTime: job.StartTime, + TTLHours: job.TTLHours, + InvalidationType: job.InvalidationType, } resp, _, err := TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err != nil { @@ -686,16 +662,14 @@ func CreateTestJobsWithPastDate(t *testing.T) { dt := time.Now() dt.Format("2019-06-18 21:28:31") job := testData.InvalidationJobs[0] - job.StartTime = &tc.Time{ - Time: dt.AddDate(0, 0, -1), - Valid: true, - } + job.StartTime = dt.AddDate(0, 0, -1) testData.InvalidationJobs[0] = job - request := tc.InvalidationJobInput{ - DeliveryService: job.DeliveryService, - Regex: job.Regex, - StartTime: job.StartTime, - TTL: job.TTL, + request := tc.InvalidationJobCreateV40{ + DeliveryService: job.DeliveryService, + Regex: job.Regex, + StartTime: job.StartTime, + TTLHours: job.TTLHours, + InvalidationType: job.InvalidationType, } resp, reqInf, err := TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err == nil { @@ -709,16 +683,14 @@ func CreateTestJobsWithPastDate(t *testing.T) { dt = time.Now() dt.Format("2019-10-12T07:20:50.52Z") job = testData.InvalidationJobs[0] - job.StartTime = &tc.Time{ - Time: dt.AddDate(0, 0, -1), - Valid: true, - } + job.StartTime = dt.AddDate(0, 0, -1) testData.InvalidationJobs[0] = job - request = tc.InvalidationJobInput{ - DeliveryService: job.DeliveryService, - Regex: job.Regex, - StartTime: job.StartTime, - TTL: job.TTL, + request = tc.InvalidationJobCreateV40{ + DeliveryService: job.DeliveryService, + Regex: job.Regex, + StartTime: job.StartTime, + TTLHours: job.TTLHours, + InvalidationType: job.InvalidationType, } resp, reqInf, err = TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err == nil { @@ -732,16 +704,14 @@ func CreateTestJobsWithPastDate(t *testing.T) { dt = time.Now() dt.Format("2020-03-11 14:12:20-06") job = testData.InvalidationJobs[0] - job.StartTime = &tc.Time{ - Time: dt.AddDate(0, 0, -5), - Valid: true, - } + job.StartTime = dt.AddDate(0, 0, -5) testData.InvalidationJobs[0] = job - request = tc.InvalidationJobInput{ - DeliveryService: job.DeliveryService, - Regex: job.Regex, - StartTime: job.StartTime, - TTL: job.TTL, + request = tc.InvalidationJobCreateV40{ + DeliveryService: job.DeliveryService, + Regex: job.Regex, + StartTime: job.StartTime, + TTLHours: job.TTLHours, + InvalidationType: job.InvalidationType, } resp, reqInf, err = TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err == nil { @@ -753,16 +723,14 @@ func CreateTestJobsWithPastDate(t *testing.T) { //unix standard format past start date job = testData.InvalidationJobs[0] - job.StartTime = &tc.Time{ - Time: time.Unix(1, 0), - Valid: true, - } + job.StartTime = time.Unix(1, 0) testData.InvalidationJobs[0] = job - request = tc.InvalidationJobInput{ - DeliveryService: job.DeliveryService, - Regex: job.Regex, - StartTime: job.StartTime, - TTL: job.TTL, + request = tc.InvalidationJobCreateV40{ + DeliveryService: job.DeliveryService, + Regex: job.Regex, + StartTime: job.StartTime, + TTLHours: job.TTLHours, + InvalidationType: job.InvalidationType, } resp, reqInf, err = TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err == nil { @@ -781,16 +749,14 @@ func CreateTestJobsWithFutureDate(t *testing.T) { dt := time.Now() dt.Format("2019-10-12T07:20:50.52Z") job := testData.InvalidationJobs[0] - job.StartTime = &tc.Time{ - Time: dt.AddDate(0, 0, 1), - Valid: true, - } + job.StartTime = dt.AddDate(0, 0, 1) testData.InvalidationJobs[0] = job - request := tc.InvalidationJobInput{ - DeliveryService: job.DeliveryService, - Regex: job.Regex, - StartTime: job.StartTime, - TTL: job.TTL, + request := tc.InvalidationJobCreateV40{ + DeliveryService: job.DeliveryService, + Regex: job.Regex, + StartTime: job.StartTime, + TTLHours: job.TTLHours, + InvalidationType: job.InvalidationType, } resp, reqInf, err := TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err != nil { @@ -804,16 +770,14 @@ func CreateTestJobsWithFutureDate(t *testing.T) { dt = time.Now() dt.Format("2020-03-11 14:12:20-06") job = testData.InvalidationJobs[0] - job.StartTime = &tc.Time{ - Time: dt.AddDate(0, 0, 1), - Valid: true, - } + job.StartTime = dt.AddDate(0, 0, 1) testData.InvalidationJobs[0] = job - request = tc.InvalidationJobInput{ - DeliveryService: job.DeliveryService, - Regex: job.Regex, - StartTime: job.StartTime, - TTL: job.TTL, + request = tc.InvalidationJobCreateV40{ + DeliveryService: job.DeliveryService, + Regex: job.Regex, + StartTime: job.StartTime, + TTLHours: job.TTLHours, + InvalidationType: job.InvalidationType, } resp, reqInf, err = TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err != nil { @@ -827,16 +791,14 @@ func CreateTestJobsWithFutureDate(t *testing.T) { dt = time.Now() dt.Format(".000") job = testData.InvalidationJobs[0] - job.StartTime = &tc.Time{ - Time: dt.AddDate(0, 0, 1), - Valid: true, - } + job.StartTime = dt.AddDate(0, 0, 1) testData.InvalidationJobs[0] = job - request = tc.InvalidationJobInput{ - DeliveryService: job.DeliveryService, - Regex: job.Regex, - StartTime: job.StartTime, - TTL: job.TTL, + request = tc.InvalidationJobCreateV40{ + DeliveryService: job.DeliveryService, + Regex: job.Regex, + StartTime: job.StartTime, + TTLHours: job.TTLHours, + InvalidationType: job.InvalidationType, } resp, reqInf, err = TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err != nil { @@ -851,12 +813,13 @@ func CreateJobsMissingDate(t *testing.T) { if len(testData.InvalidationJobs) < 1 { t.Fatal("Need at least one Invalidation Job to test creating an invalid Job") } - //Missing date + //Missing start date job := testData.InvalidationJobs[0] - request := tc.InvalidationJobInput{ - DeliveryService: job.DeliveryService, - Regex: job.Regex, - TTL: job.TTL, + request := tc.InvalidationJobCreateV40{ + DeliveryService: job.DeliveryService, + Regex: job.Regex, + TTLHours: job.TTLHours, + InvalidationType: job.InvalidationType, } resp, reqInf, err := TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err == nil { @@ -876,15 +839,12 @@ func CreateJobsMissingRegex(t *testing.T) { dt := time.Now() dt.Format("2019-10-12T07:20:50.52Z") job := testData.InvalidationJobs[0] - job.StartTime = &tc.Time{ - Time: dt.AddDate(0, 0, 1), - Valid: true, - } + job.StartTime = dt.AddDate(0, 0, 1) testData.InvalidationJobs[0] = job - request := tc.InvalidationJobInput{ - DeliveryService: job.DeliveryService, - TTL: job.TTL, - StartTime: job.StartTime, + request := tc.InvalidationJobCreateV40{ + DeliveryService: job.DeliveryService, + TTLHours: job.TTLHours, + InvalidationType: job.InvalidationType, } resp, reqInf, err := TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err == nil { @@ -895,11 +855,11 @@ func CreateJobsMissingRegex(t *testing.T) { } //Empty Regex - job.Regex = nil - request = tc.InvalidationJobInput{ + job.Regex = "" + request = tc.InvalidationJobCreateV40{ DeliveryService: job.DeliveryService, Regex: job.Regex, - TTL: job.TTL, + TTLHours: job.TTLHours, StartTime: job.StartTime, } resp, _, err = TOSession.CreateInvalidationJob(request, client.RequestOptions{}) @@ -920,15 +880,12 @@ func CreateJobsMissingTtl(t *testing.T) { dt := time.Now() dt.Format("2019-10-12T07:20:50.52Z") job := testData.InvalidationJobs[0] - job.StartTime = &tc.Time{ - Time: dt.AddDate(0, 0, 1), - Valid: true, - } + job.StartTime = dt.AddDate(0, 0, 1) testData.InvalidationJobs[0] = job - request := tc.InvalidationJobInput{ - DeliveryService: job.DeliveryService, - Regex: job.Regex, - StartTime: job.StartTime, + request := tc.InvalidationJobCreateV40{ + DeliveryService: job.DeliveryService, + Regex: job.Regex, + InvalidationType: job.InvalidationType, } resp, reqInf, err := TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err == nil { @@ -938,13 +895,14 @@ func CreateJobsMissingTtl(t *testing.T) { t.Errorf("Expected status code 400, got %v", reqInf.StatusCode) } - //Empty TTL - job.TTL = nil - request = tc.InvalidationJobInput{ - DeliveryService: job.DeliveryService, - Regex: job.Regex, - TTL: job.TTL, - StartTime: job.StartTime, + //Invalid TTL + job.TTLHours = 0 + request = tc.InvalidationJobCreateV40{ + DeliveryService: job.DeliveryService, + Regex: job.Regex, + TTLHours: job.TTLHours, + StartTime: job.StartTime, + InvalidationType: job.InvalidationType, } resp, _, err = TOSession.CreateInvalidationJob(request, client.RequestOptions{}) if err == nil { @@ -963,15 +921,13 @@ func UpdateTestJobsInvalidDS(t *testing.T) { t.Fatal("Found a Delivery Service in the testing data with null or undefined XMLID") } xmlID := *testData.DeliveryServices[0].XMLID - startTime := tc.Time{ - Time: time.Now().Add(time.Hour), - Valid: true, - } - firstJob := tc.InvalidationJobInput{ - DeliveryService: util.InterfacePtr(&xmlID), - Regex: util.StrPtr(`/\.*([A-Z]0?)`), - TTL: util.InterfacePtr(16), - StartTime: &startTime, + startTime := time.Now().Add(time.Hour) + firstJob := tc.InvalidationJobCreateV40{ + DeliveryService: xmlID, + Regex: `/\.*([A-Z]0?)`, + TTLHours: 16, + StartTime: startTime, + InvalidationType: tc.REFRESH, } resp, _, err := TOSession.CreateInvalidationJob(firstJob, client.RequestOptions{}) @@ -985,13 +941,13 @@ func UpdateTestJobsInvalidDS(t *testing.T) { t.Fatalf("unable to get invalidation jobs: %v - alerts: %+v", err, jobs.Alerts) } - var realJob tc.InvalidationJob + var realJob tc.InvalidationJobV40 for i, job := range jobs.Response { if job.StartTime == nil || job.DeliveryService == nil || job.CreatedBy == nil { t.Error("Traffic Ops returned a representation of a content invalidation Job that had null or undefined Start Time and/or Delivery Service and/or Created By") continue } - diff := firstJob.StartTime.Time.Sub(job.StartTime.Time) + diff := firstJob.StartTime.Sub(*job.StartTime) if *job.DeliveryService == xmlID && *job.CreatedBy == "admin" && diff < time.Second { realJob = jobs.Response[i] break @@ -1003,10 +959,7 @@ func UpdateTestJobsInvalidDS(t *testing.T) { //update existing jobs with new ds id originalJob := realJob - newTime := tc.Time{ - Time: startTime.Time.Add(time.Hour * 2), - Valid: true, - } + newTime := startTime.Add(time.Hour * 2) originalJob.StartTime = &newTime originalJob.DeliveryService = testData.DeliveryServices[1].XMLID alerts, reqInf, err := TOSession.UpdateInvalidationJob(originalJob, client.RequestOptions{}) @@ -1089,11 +1042,11 @@ func UpdateTestJobsInvalidDS(t *testing.T) { t.Errorf("Expected status code 409, got %v", reqInf.StatusCode) } - //update existing jobs with blank parameters - blankParametersJob := realJob - parameters := "" - blankParametersJob.Parameters = ¶meters - alerts, reqInf, err = TOSession.UpdateInvalidationJob(blankParametersJob, client.RequestOptions{}) + //update existing jobs with blank invalidation types + blankInvalidationTypeJob := realJob + invalidationType := "" + blankInvalidationTypeJob.InvalidationType = &invalidationType + alerts, reqInf, err = TOSession.UpdateInvalidationJob(blankInvalidationTypeJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected parameters: cannot be blank. alerts: %+v", alerts.Alerts) } @@ -1105,10 +1058,8 @@ func UpdateTestJobsInvalidDS(t *testing.T) { startDateFutureJob := realJob dt := time.Now() dt.Format("2019-10-12T07:20:50.52Z") - startDateFutureJob.StartTime = &tc.Time{ - Time: dt.AddDate(0, 0, 3), - Valid: true, - } + addThreeDays := dt.AddDate(0, 0, 3) + startDateFutureJob.StartTime = &addThreeDays alerts, reqInf, err = TOSession.UpdateInvalidationJob(startDateFutureJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected startTime: must be within two days from now. alerts: %+v", alerts.Alerts) @@ -1121,10 +1072,8 @@ func UpdateTestJobsInvalidDS(t *testing.T) { pastStartDateJob := realJob dt = time.Now() dt.Format("2019-06-18 21:28:31") - pastStartDateJob.StartTime = &tc.Time{ - Time: dt.AddDate(0, 0, -3), - Valid: true, - } + minusThreeDays := dt.AddDate(0, 0, -3) + pastStartDateJob.StartTime = &minusThreeDays alerts, reqInf, err = TOSession.UpdateInvalidationJob(pastStartDateJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected startTime: cannot be in the past. alerts: %+v", alerts.Alerts) @@ -1137,10 +1086,8 @@ func UpdateTestJobsInvalidDS(t *testing.T) { pastStartDateJob = realJob dt = time.Now() dt.Format("2019-10-12T07:20:50.52Z") - pastStartDateJob.StartTime = &tc.Time{ - Time: dt.AddDate(0, 0, -1), - Valid: true, - } + minusOneDay := dt.AddDate(0, 0, -1) + pastStartDateJob.StartTime = &minusOneDay alerts, reqInf, err = TOSession.UpdateInvalidationJob(pastStartDateJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected startTime: cannot be in the past. alerts: %+v", alerts.Alerts) @@ -1151,10 +1098,8 @@ func UpdateTestJobsInvalidDS(t *testing.T) { //update jobs with UNIX Format past start date pastStartDateJob = realJob - pastStartDateJob.StartTime = &tc.Time{ - Time: time.Unix(1, 0), - Valid: true, - } + unixTimeFormat := time.Unix(1, 0) + pastStartDateJob.StartTime = &unixTimeFormat alerts, reqInf, err = TOSession.UpdateInvalidationJob(pastStartDateJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected startTime: cannot be in the past. alerts: %+v", alerts.Alerts) @@ -1167,10 +1112,8 @@ func UpdateTestJobsInvalidDS(t *testing.T) { pastStartDateJob = realJob dt = time.Now() dt.Format("2020-03-11 14:12:20-06") - pastStartDateJob.StartTime = &tc.Time{ - Time: dt.AddDate(0, 0, -1), - Valid: true, - } + oneLessDay := dt.AddDate(0, 0, -1) + pastStartDateJob.StartTime = &oneLessDay alerts, reqInf, err = TOSession.UpdateInvalidationJob(pastStartDateJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected startTime: cannot be in the past. alerts: %+v", alerts.Alerts) @@ -1183,10 +1126,8 @@ func UpdateTestJobsInvalidDS(t *testing.T) { startDateFutureJob = realJob dt = time.Now() dt.Format("2019-10-12T07:20:50.52Z") - startDateFutureJob.StartTime = &tc.Time{ - Time: dt.AddDate(0, 0, 1), - Valid: true, - } + oneMoreDay := dt.AddDate(0, 0, 1) + startDateFutureJob.StartTime = &oneMoreDay alerts, reqInf, err = TOSession.UpdateInvalidationJob(startDateFutureJob, client.RequestOptions{}) if err != nil { t.Fatalf("Expected Content invalidation job updated. alerts: %+v", alerts.Alerts) @@ -1199,10 +1140,8 @@ func UpdateTestJobsInvalidDS(t *testing.T) { startDateFutureJob = realJob dt = time.Now() dt.Format(".000") - startDateFutureJob.StartTime = &tc.Time{ - Time: dt.AddDate(0, 0, 1), - Valid: true, - } + andAnotherDay := dt.AddDate(0, 0, 1) + startDateFutureJob.StartTime = &andAnotherDay alerts, reqInf, err = TOSession.UpdateInvalidationJob(startDateFutureJob, client.RequestOptions{}) if err != nil { t.Fatalf("Expected Content invalidation job updated. alerts: %+v", alerts.Alerts) @@ -1215,10 +1154,8 @@ func UpdateTestJobsInvalidDS(t *testing.T) { startDateFutureJob = realJob dt = time.Now() dt.Format("2020-03-11 14:12:20-06") - startDateFutureJob.StartTime = &tc.Time{ - Time: dt.AddDate(0, 0, 1), - Valid: true, - } + addEvenMore := dt.AddDate(0, 0, 1) + startDateFutureJob.StartTime = &addEvenMore alerts, reqInf, err = TOSession.UpdateInvalidationJob(startDateFutureJob, client.RequestOptions{}) if err != nil { t.Fatalf("Expected Content invalidation job updated. alerts: %+v", alerts.Alerts) diff --git a/traffic_ops/testing/api/v4/tc-fixtures.json b/traffic_ops/testing/api/v4/tc-fixtures.json index aefe7a71d1..da13b832c6 100644 --- a/traffic_ops/testing/api/v4/tc-fixtures.json +++ b/traffic_ops/testing/api/v4/tc-fixtures.json @@ -5678,44 +5678,23 @@ { "deliveryService": "ds1", "regex": "/.*", - "startTime": 4117118271000, - "ttl": "121m" + "startTime": "2100-06-19T13:57:51Z", + "ttlHours": 72, + "invalidationType" : "REFRESH" }, { "deliveryService": "ds1", "regex": "/foo", - "startTime": 4117118271000, - "ttl": 2160 + "startTime": "2100-06-19T13:57:51Z", + "ttlHours": 36, + "invalidationType" : "REFRESH" }, { "deliveryService": "ds2", "regex": "\\/some-path?.+\\.jpg", - "startTime": "2100-06-19T13:57:51-06:00", - "ttl": 2.1 - }, - { - "deliveryService": "ds1", - "regex": "/oldest", - "startTime": 1000, - "ttl": 2160 - }, - { - "deliveryService": "ds1", - "regex": "/older", - "startTime": 1001, - "ttl": 2160 - }, - { - "deliveryService": "ds1", - "regex": "/old", - "startTime": 1002, - "ttl": 2160 - }, - { - "deliveryService": "ds-forked-topology", - "regex": "/this-ds-is-in-cdn2", - "startTime": 4117118271000, - "ttl": 2160 + "startTime": "2100-06-19T13:57:51Z", + "ttlHours": 144, + "invalidationType" : "REFRESH" } ], "statsSummaries": [ diff --git a/traffic_ops/testing/api/v4/todb_test.go b/traffic_ops/testing/api/v4/todb_test.go index 7e4ccdd27d..83f34d41a4 100644 --- a/traffic_ops/testing/api/v4/todb_test.go +++ b/traffic_ops/testing/api/v4/todb_test.go @@ -80,18 +80,6 @@ func SetupTestData(*sql.DB) error { os.Exit(1) } - err = SetupJobAgents(db) - if err != nil { - fmt.Printf("\nError setting up job agents %s - %s, %v\n", Config.TrafficOps.URL, Config.TrafficOps.Users.Admin, err) - os.Exit(1) - } - - err = SetupJobStatuses(db) - if err != nil { - fmt.Printf("\nError setting up job agents %s - %s, %v\n", Config.TrafficOps.URL, Config.TrafficOps.Users.Admin, err) - os.Exit(1) - } - err = SetupTypes(db) if err != nil { fmt.Printf("\nError setting up types %s - %s, %v\n", Config.TrafficOps.URL, Config.TrafficOps.Users.Admin, err) @@ -361,39 +349,13 @@ INSERT INTO deliveryservice_tmuser (deliveryservice, tm_user_id, last_updated) V return nil } -// SetupJobStatuses ... -func SetupJobStatuses(db *sql.DB) error { - - sqlStmt := ` -INSERT INTO job_status (id, name, description, last_updated) VALUES (1, 'PENDING', 'Job is queued, but has not been picked up by any agents yet', '2018-01-19 21:19:32.444857'); -` - err := execSQL(db, sqlStmt) - if err != nil { - return fmt.Errorf("exec failed %v", err) - } - return nil -} - -// SetupJobAgents ... -func SetupJobAgents(db *sql.DB) error { - - sqlStmt := ` -INSERT INTO job_agent (id, name, description, active, last_updated) VALUES (1, 'agent1', 'Test Agent1', 0, '2018-01-19 21:19:32.448076'); -` - err := execSQL(db, sqlStmt) - if err != nil { - return fmt.Errorf("exec failed %v", err) - } - return nil -} - // SetupJobs ... func SetupJobs(db *sql.DB) error { sqlStmt := ` -INSERT INTO job (id, agent, object_type, object_name, keyword, parameters, asset_url, asset_type, status, start_time, entered_time, job_user, last_updated, job_deliveryservice) VALUES (100, 1, null, null, 'PURGE', 'TTL:48h', 'http://cdn2.edge/job1/.*', 'file', 1, '2018-01-19 21:01:14.000000', '2018-01-19 21:01:14.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.468643', 100); -INSERT INTO job (id, agent, object_type, object_name, keyword, parameters, asset_url, asset_type, status, start_time, entered_time, job_user, last_updated, job_deliveryservice) VALUES (200, 1, null, null, 'PURGE', 'TTL:48h', 'http://cdn2.edge/job2/.*', 'file', 1, '2018-01-19 21:09:34.000000', '2018-01-19 21:09:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.450915', 200); -INSERT INTO job (id, agent, object_type, object_name, keyword, parameters, asset_url, asset_type, status, start_time, entered_time, job_user, last_updated, job_deliveryservice) VALUES (300, 1, null, null, 'PURGE', 'TTL:48h', 'http://cdn2.edge/job3/.*', 'file', 1, '2018-01-19 21:14:34.000000', '2018-01-19 21:14:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.460870', 100); +INSERT INTO job (id, ttl_hr, asset_url, start_time, entered_time, job_user, last_updated, job_deliveryservice, invalidation_type) VALUES (100, 24, 'http://cdn2.edge/job1/.*', '2018-01-19 21:01:14.000000', '2018-01-19 21:01:14.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.468643', 100, 'REFRESH'); +INSERT INTO job (id, ttl_hr, asset_url, start_time, entered_time, job_user, last_updated, job_deliveryservice, invalidation_type) VALUES (200, 36, 'http://cdn2.edge/job2/.*', '2018-01-19 21:09:34.000000', '2018-01-19 21:09:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.450915', 200, 'REFETCH'); +INSERT INTO job (id, ttl_hr, asset_url, start_time, entered_time, job_user, last_updated, job_deliveryservice, invalidation_type) VALUES (300, 72, 'http://cdn2.edge/job3/.*', '2018-01-19 21:14:34.000000', '2018-01-19 21:14:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.460870', 100, 'REFRESH'); ` err := execSQL(db, sqlStmt) if err != nil { @@ -442,8 +404,6 @@ func Teardown(db *sql.DB) error { DELETE FROM to_extension; DELETE FROM staticdnsentry; DELETE FROM job; - DELETE FROM job_agent; - DELETE FROM job_status; DELETE FROM log; DELETE FROM asn; DELETE FROM deliveryservice_tmuser; diff --git a/traffic_ops/testing/api/v4/traffic_control_test.go b/traffic_ops/testing/api/v4/traffic_control_test.go index 2de4786ae8..20052c9879 100644 --- a/traffic_ops/testing/api/v4/traffic_control_test.go +++ b/traffic_ops/testing/api/v4/traffic_control_test.go @@ -56,5 +56,5 @@ type TrafficControl struct { SteeringTargets []tc.SteeringTargetNullable `json:"steeringTargets"` Serverchecks []tc.ServercheckRequestNullable `json:"serverchecks"` Users []tc.UserV4 `json:"users"` - InvalidationJobs []tc.InvalidationJobInput `json:"invalidationJobs"` + InvalidationJobs []tc.InvalidationJobCreateV40 `json:"invalidationJobs"` } diff --git a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go index 1529d6743e..0062925972 100644 --- a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go +++ b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go @@ -388,7 +388,7 @@ func (job *InvalidationJobV40) Read(h http.Header, useIMS bool) ([]interface{}, err := rows.Scan(&job.ID, &job.AssetURL, &job.CreatedBy, - &job.DeliveryServiceXMLID, + &job.DeliveryService, &job.TTLHours, &job.InvalidationType, &job.StartTime) @@ -584,7 +584,7 @@ func CreateV40(w http.ResponseWriter, r *http.Request) { &result.ID, &result.AssetURL, &result.CreatedBy, - &result.DeliveryServiceXMLID, + &result.DeliveryService, &result.TTLHours, &result.InvalidationType, &result.StartTime) @@ -642,7 +642,7 @@ func CreateV40(w http.ResponseWriter, r *http.Request) { api.Created, duplicate, *result.ID, - *result.DeliveryServiceXMLID, + *result.DeliveryService, *result.AssetURL, *result.TTLHours, *result.InvalidationType, @@ -822,7 +822,7 @@ func UpdateV40(w http.ResponseWriter, r *http.Request) { &job.CreatedBy, &uid, &dsid, - &job.DeliveryServiceXMLID, + &job.DeliveryService, &job.AssetURL, &job.TTLHours, &job.StartTime, @@ -893,7 +893,7 @@ func UpdateV40(w http.ResponseWriter, r *http.Request) { return } - if *job.DeliveryServiceXMLID != *input.DeliveryServiceXMLID { + if *job.DeliveryService != *input.DeliveryService { userErr = errors.New("Cannot change 'deliveryService' of existing invalidation job!") errCode = http.StatusConflict api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) @@ -931,7 +931,7 @@ func UpdateV40(w http.ResponseWriter, r *http.Request) { *job.ID) err = row.Scan(&job.AssetURL, &job.CreatedBy, - &job.DeliveryServiceXMLID, + &job.DeliveryService, &job.ID, &job.TTLHours, &job.StartTime, @@ -943,7 +943,7 @@ func UpdateV40(w http.ResponseWriter, r *http.Request) { return } - if err = setRevalFlags(*job.DeliveryServiceXMLID, inf.Tx.Tx); err != nil { + if err = setRevalFlags(*job.DeliveryService, inf.Tx.Tx); err != nil { api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, fmt.Errorf("Setting reval flags: %v", err)) return } @@ -982,7 +982,7 @@ func UpdateV40(w http.ResponseWriter, r *http.Request) { changeLogMsg := fmt.Sprintf("%s content invalidation job - ID: %d DSXMLID: %s ASSET_URL: '%s' TTLHRs: %d INVALIDATION: %s", api.Updated, *input.ID, - *input.DeliveryServiceXMLID, + *input.DeliveryService, *input.AssetURL, *input.TTLHours, *input.InvalidationType, @@ -1226,7 +1226,7 @@ func DeleteV40(w http.ResponseWriter, r *http.Request) { &result.ID, &result.AssetURL, &result.CreatedBy, - &result.DeliveryServiceXMLID, + &result.DeliveryService, &result.TTLHours, &result.InvalidationType, &result.StartTime) @@ -1263,7 +1263,7 @@ func DeleteV40(w http.ResponseWriter, r *http.Request) { changeLogMsg := fmt.Sprintf("%s content invalidation job - ID: %d DSXMLID: %s ASSET_URL: '%s' TTLHRs: %d INVALIDATION: %s", api.Deleted, *result.ID, - *result.DeliveryServiceXMLID, + *result.DeliveryService, *result.AssetURL, *result.TTLHours, *result.InvalidationType, diff --git a/traffic_ops/v4-client/job.go b/traffic_ops/v4-client/job.go index 54f612617c..a2ba5bd66a 100644 --- a/traffic_ops/v4-client/job.go +++ b/traffic_ops/v4-client/job.go @@ -28,7 +28,7 @@ import ( const apiJobs = "/jobs" // CreateInvalidationJob creates the passed Content Invalidation Job. -func (to *Session) CreateInvalidationJob(job tc.InvalidationJobInput, opts RequestOptions) (tc.Alerts, toclientlib.ReqInf, error) { +func (to *Session) CreateInvalidationJob(job tc.InvalidationJobCreateV40, opts RequestOptions) (tc.Alerts, toclientlib.ReqInf, error) { var alerts tc.Alerts reqInf, err := to.post(apiJobs, opts, job, &alerts) return alerts, reqInf, err @@ -48,7 +48,7 @@ func (to *Session) DeleteInvalidationJob(jobID uint64, opts RequestOptions) (tc. // UpdateInvalidationJob updates the passed Content Invalidation Job (it is // expected to have an ID). -func (to *Session) UpdateInvalidationJob(job tc.InvalidationJob, opts RequestOptions) (tc.Alerts, toclientlib.ReqInf, error) { +func (to *Session) UpdateInvalidationJob(job tc.InvalidationJobV40, opts RequestOptions) (tc.Alerts, toclientlib.ReqInf, error) { var alerts tc.Alerts if job.ID == nil { return alerts, toclientlib.ReqInf{}, errors.New("job has a nil ID") @@ -63,8 +63,8 @@ func (to *Session) UpdateInvalidationJob(job tc.InvalidationJob, opts RequestOpt // GetInvalidationJobs returns a list of Content Invalidation Jobs visible to // your Tenant. -func (to *Session) GetInvalidationJobs(opts RequestOptions) (tc.InvalidationJobsResponse, toclientlib.ReqInf, error) { - var data tc.InvalidationJobsResponse +func (to *Session) GetInvalidationJobs(opts RequestOptions) (tc.InvalidationJobsResponseV40, toclientlib.ReqInf, error) { + var data tc.InvalidationJobsResponseV40 reqInf, err := to.get(apiJobs, opts, &data) return data, reqInf, err } From 2512152b123032088aada7b7871da704a3b24088 Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Mon, 16 Aug 2021 16:49:53 -0600 Subject: [PATCH 11/35] Commit current state for help with api testing --- lib/go-tc/invalidationjobs.go | 35 +++++++++++++++++++++++++ traffic_ops/testing/api/v4/jobs_test.go | 7 +++++ 2 files changed, 42 insertions(+) diff --git a/lib/go-tc/invalidationjobs.go b/lib/go-tc/invalidationjobs.go index 39c5425c17..b26fa5b248 100644 --- a/lib/go-tc/invalidationjobs.go +++ b/lib/go-tc/invalidationjobs.go @@ -551,6 +551,41 @@ type InvalidationJobV40 struct { StartTime *time.Time `json:"startTime"` } +func (job InvalidationJobV40) String() string { + var ID, AssetURL, CreatedBy, DeliveryService, TTLHours, InvalidationType, StartTime string + if job.ID != nil { + ID = strconv.FormatUint(*job.ID, 10) + } else { + ID = "-1" + } + if job.AssetURL != nil { + AssetURL = *job.AssetURL + } + if job.CreatedBy != nil { + CreatedBy = *job.CreatedBy + } + if job.DeliveryService != nil { + DeliveryService = *job.DeliveryService + } + if job.TTLHours != nil { + TTLHours = strconv.FormatUint(uint64(*job.TTLHours), 10) + } + if job.InvalidationType != nil { + InvalidationType = *job.InvalidationType + } + if job.StartTime != nil { + StartTime = job.StartTime.String() + } + return fmt.Sprintf("{\nID:\t\t\t\t\t%s,\nAssetURL:\t\t\t%s,\nCreatedBy:\t\t\t%s,\nDeliveryService:\t%s,\nTTLHours:\t\t\t%s,\nInvalidationType:\t%s,\nStartTime:\t\t\t%s,\n}", + ID, + AssetURL, + CreatedBy, + DeliveryService, + TTLHours, + InvalidationType, + StartTime) +} + // Validate checks that the InvalidationJob is valid, by ensuring all of its fields are well-defined. // // This returns an error describing any and all problematic fields encountered during validation. diff --git a/traffic_ops/testing/api/v4/jobs_test.go b/traffic_ops/testing/api/v4/jobs_test.go index 29ce604524..7768f9ca4e 100644 --- a/traffic_ops/testing/api/v4/jobs_test.go +++ b/traffic_ops/testing/api/v4/jobs_test.go @@ -22,6 +22,7 @@ import ( "testing" "time" + "github.com/apache/trafficcontrol/lib/go-log" "github.com/apache/trafficcontrol/lib/go-tc" client "github.com/apache/trafficcontrol/traffic_ops/v4-client" ) @@ -352,7 +353,13 @@ func GetTestInvalidationJobs(t *testing.T) { } func GetTestJobsByValidData(t *testing.T) { + log.Infof("GetTestJobsByValidData") + toJobs, _, err := TOSession.GetInvalidationJobs(client.RequestOptions{}) + log.Infof("toJobs len: %d", len(toJobs.Response)) + for _, job := range toJobs.Response { + log.Infof("Job: %v\n", job) + } if err != nil { t.Fatalf("error getting jobs %v - alerts: %+v", err, toJobs.Alerts) } From faa56cd9e6178b38f297eb4c680b5124b13113a6 Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Mon, 16 Aug 2021 21:11:15 -0600 Subject: [PATCH 12/35] Fix v2 and v3 to work with modified refetch jobs DB schema --- traffic_ops/testing/api/v2/todb_test.go | 46 ++----------------------- traffic_ops/testing/api/v3/todb_test.go | 46 ++----------------------- 2 files changed, 6 insertions(+), 86 deletions(-) diff --git a/traffic_ops/testing/api/v2/todb_test.go b/traffic_ops/testing/api/v2/todb_test.go index 370187d886..b5aa360549 100644 --- a/traffic_ops/testing/api/v2/todb_test.go +++ b/traffic_ops/testing/api/v2/todb_test.go @@ -86,18 +86,6 @@ func SetupTestData(*sql.DB) error { os.Exit(1) } - err = SetupJobAgents(db) - if err != nil { - fmt.Printf("\nError setting up job agents %s - %s, %v\n", Config.TrafficOps.URL, Config.TrafficOps.Users.Admin, err) - os.Exit(1) - } - - err = SetupJobStatuses(db) - if err != nil { - fmt.Printf("\nError setting up job agents %s - %s, %v\n", Config.TrafficOps.URL, Config.TrafficOps.Users.Admin, err) - os.Exit(1) - } - err = SetupTypes(db) if err != nil { fmt.Printf("\nError setting up types %s - %s, %v\n", Config.TrafficOps.URL, Config.TrafficOps.Users.Admin, err) @@ -229,39 +217,13 @@ INSERT INTO deliveryservice_tmuser (deliveryservice, tm_user_id, last_updated) V return nil } -// SetupJobStatuses ... -func SetupJobStatuses(db *sql.DB) error { - - sqlStmt := ` -INSERT INTO job_status (id, name, description, last_updated) VALUES (1, 'PENDING', 'Job is queued, but has not been picked up by any agents yet', '2018-01-19 21:19:32.444857'); -` - err := execSQL(db, sqlStmt) - if err != nil { - return fmt.Errorf("exec failed %v", err) - } - return nil -} - -// SetupJobAgents ... -func SetupJobAgents(db *sql.DB) error { - - sqlStmt := ` -INSERT INTO job_agent (id, name, description, active, last_updated) VALUES (1, 'agent1', 'Test Agent1', 0, '2018-01-19 21:19:32.448076'); -` - err := execSQL(db, sqlStmt) - if err != nil { - return fmt.Errorf("exec failed %v", err) - } - return nil -} - // SetupJobs ... func SetupJobs(db *sql.DB) error { sqlStmt := ` -INSERT INTO job (id, agent, object_type, object_name, keyword, parameters, asset_url, asset_type, status, start_time, entered_time, job_user, last_updated, job_deliveryservice) VALUES (100, 1, null, null, 'PURGE', 'TTL:48h', 'http://cdn2.edge/job1/.*', 'file', 1, '2018-01-19 21:01:14.000000', '2018-01-19 21:01:14.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.468643', 100); -INSERT INTO job (id, agent, object_type, object_name, keyword, parameters, asset_url, asset_type, status, start_time, entered_time, job_user, last_updated, job_deliveryservice) VALUES (200, 1, null, null, 'PURGE', 'TTL:48h', 'http://cdn2.edge/job2/.*', 'file', 1, '2018-01-19 21:09:34.000000', '2018-01-19 21:09:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.450915', 200); -INSERT INTO job (id, agent, object_type, object_name, keyword, parameters, asset_url, asset_type, status, start_time, entered_time, job_user, last_updated, job_deliveryservice) VALUES (300, 1, null, null, 'PURGE', 'TTL:48h', 'http://cdn2.edge/job3/.*', 'file', 1, '2018-01-19 21:14:34.000000', '2018-01-19 21:14:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.460870', 100); +INSERT INTO job (id, ttl_hr, asset_url, start_time, entered_time, job_user, last_updated, job_deliveryservice, invalidation_type) VALUES (100, 24, 'http://cdn2.edge/job1/.*', '2018-01-19 21:01:14.000000', '2018-01-19 21:01:14.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.468643', 100, 'REFRESH'); +INSERT INTO job (id, ttl_hr, asset_url, start_time, entered_time, job_user, last_updated, job_deliveryservice, invalidation_type) VALUES (200, 36, 'http://cdn2.edge/job2/.*', '2018-01-19 21:09:34.000000', '2018-01-19 21:09:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.450915', 200, 'REFETCH'); +INSERT INTO job (id, ttl_hr, asset_url, start_time, entered_time, job_user, last_updated, job_deliveryservice, invalidation_type) VALUES (300, 72, 'http://cdn2.edge/job3/.*', '2018-01-19 21:14:34.000000', '2018-01-19 21:14:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.460870', 100, 'REFRESH'); ` err := execSQL(db, sqlStmt) if err != nil { @@ -310,8 +272,6 @@ func Teardown(db *sql.DB) error { DELETE FROM to_extension; DELETE FROM staticdnsentry; DELETE FROM job; - DELETE FROM job_agent; - DELETE FROM job_status; DELETE FROM log; DELETE FROM asn; DELETE FROM deliveryservice_tmuser; diff --git a/traffic_ops/testing/api/v3/todb_test.go b/traffic_ops/testing/api/v3/todb_test.go index c0ab6a66da..088717fdc8 100644 --- a/traffic_ops/testing/api/v3/todb_test.go +++ b/traffic_ops/testing/api/v3/todb_test.go @@ -86,18 +86,6 @@ func SetupTestData(*sql.DB) error { os.Exit(1) } - err = SetupJobAgents(db) - if err != nil { - fmt.Printf("\nError setting up job agents %s - %s, %v\n", Config.TrafficOps.URL, Config.TrafficOps.Users.Admin, err) - os.Exit(1) - } - - err = SetupJobStatuses(db) - if err != nil { - fmt.Printf("\nError setting up job agents %s - %s, %v\n", Config.TrafficOps.URL, Config.TrafficOps.Users.Admin, err) - os.Exit(1) - } - err = SetupTypes(db) if err != nil { fmt.Printf("\nError setting up types %s - %s, %v\n", Config.TrafficOps.URL, Config.TrafficOps.Users.Admin, err) @@ -229,39 +217,13 @@ INSERT INTO deliveryservice_tmuser (deliveryservice, tm_user_id, last_updated) V return nil } -// SetupJobStatuses ... -func SetupJobStatuses(db *sql.DB) error { - - sqlStmt := ` -INSERT INTO job_status (id, name, description, last_updated) VALUES (1, 'PENDING', 'Job is queued, but has not been picked up by any agents yet', '2018-01-19 21:19:32.444857'); -` - err := execSQL(db, sqlStmt) - if err != nil { - return fmt.Errorf("exec failed %v", err) - } - return nil -} - -// SetupJobAgents ... -func SetupJobAgents(db *sql.DB) error { - - sqlStmt := ` -INSERT INTO job_agent (id, name, description, active, last_updated) VALUES (1, 'agent1', 'Test Agent1', 0, '2018-01-19 21:19:32.448076'); -` - err := execSQL(db, sqlStmt) - if err != nil { - return fmt.Errorf("exec failed %v", err) - } - return nil -} - // SetupJobs ... func SetupJobs(db *sql.DB) error { sqlStmt := ` -INSERT INTO job (id, agent, object_type, object_name, keyword, parameters, asset_url, asset_type, status, start_time, entered_time, job_user, last_updated, job_deliveryservice) VALUES (100, 1, null, null, 'PURGE', 'TTL:48h', 'http://cdn2.edge/job1/.*', 'file', 1, '2018-01-19 21:01:14.000000', '2018-01-19 21:01:14.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.468643', 100); -INSERT INTO job (id, agent, object_type, object_name, keyword, parameters, asset_url, asset_type, status, start_time, entered_time, job_user, last_updated, job_deliveryservice) VALUES (200, 1, null, null, 'PURGE', 'TTL:48h', 'http://cdn2.edge/job2/.*', 'file', 1, '2018-01-19 21:09:34.000000', '2018-01-19 21:09:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.450915', 200); -INSERT INTO job (id, agent, object_type, object_name, keyword, parameters, asset_url, asset_type, status, start_time, entered_time, job_user, last_updated, job_deliveryservice) VALUES (300, 1, null, null, 'PURGE', 'TTL:48h', 'http://cdn2.edge/job3/.*', 'file', 1, '2018-01-19 21:14:34.000000', '2018-01-19 21:14:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.460870', 100); +INSERT INTO job (id, ttl_hr, asset_url, start_time, entered_time, job_user, last_updated, job_deliveryservice, invalidation_type) VALUES (100, 24, 'http://cdn2.edge/job1/.*', '2018-01-19 21:01:14.000000', '2018-01-19 21:01:14.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.468643', 100, 'REFRESH'); +INSERT INTO job (id, ttl_hr, asset_url, start_time, entered_time, job_user, last_updated, job_deliveryservice, invalidation_type) VALUES (200, 36, 'http://cdn2.edge/job2/.*', '2018-01-19 21:09:34.000000', '2018-01-19 21:09:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.450915', 200, 'REFETCH'); +INSERT INTO job (id, ttl_hr, asset_url, start_time, entered_time, job_user, last_updated, job_deliveryservice, invalidation_type) VALUES (300, 72, 'http://cdn2.edge/job3/.*', '2018-01-19 21:14:34.000000', '2018-01-19 21:14:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.460870', 100, 'REFRESH'); ` err := execSQL(db, sqlStmt) if err != nil { @@ -310,8 +272,6 @@ func Teardown(db *sql.DB) error { DELETE FROM to_extension; DELETE FROM staticdnsentry; DELETE FROM job; - DELETE FROM job_agent; - DELETE FROM job_status; DELETE FROM log; DELETE FROM asn; DELETE FROM deliveryservice_tmuser; From 9f7b0892747f78583bd1dd854b67cd1144d42cb6 Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Tue, 17 Aug 2021 08:38:27 -0600 Subject: [PATCH 13/35] Remove old DB table references from ORT/T3C api tests --- cache-config/testing/ort-tests/tcdata/todb.go | 46 ++----------------- 1 file changed, 3 insertions(+), 43 deletions(-) diff --git a/cache-config/testing/ort-tests/tcdata/todb.go b/cache-config/testing/ort-tests/tcdata/todb.go index 91e91faf41..2de9861739 100644 --- a/cache-config/testing/ort-tests/tcdata/todb.go +++ b/cache-config/testing/ort-tests/tcdata/todb.go @@ -87,18 +87,6 @@ func (r *TCData) SetupTestData(*sql.DB) error { os.Exit(1) } - err = SetupJobAgents(db) - if err != nil { - fmt.Printf("\nError setting up job agents %s - %s, %v\n", r.Config.TrafficOps.URL, r.Config.TrafficOps.Users.Admin, err) - os.Exit(1) - } - - err = SetupJobStatuses(db) - if err != nil { - fmt.Printf("\nError setting up job agents %s - %s, %v\n", r.Config.TrafficOps.URL, r.Config.TrafficOps.Users.Admin, err) - os.Exit(1) - } - err = SetupTypes(db) if err != nil { fmt.Printf("\nError setting up types %s - %s, %v\n", r.Config.TrafficOps.URL, r.Config.TrafficOps.Users.Admin, err) @@ -230,39 +218,13 @@ INSERT INTO deliveryservice_tmuser (deliveryservice, tm_user_id, last_updated) V return nil } -// SetupJobStatuses ... -func SetupJobStatuses(db *sql.DB) error { - - sqlStmt := ` -INSERT INTO job_status (id, name, description, last_updated) VALUES (1, 'PENDING', 'Job is queued, but has not been picked up by any agents yet', '2018-01-19 21:19:32.444857'); -` - err := execSQL(db, sqlStmt) - if err != nil { - return fmt.Errorf("exec failed %v", err) - } - return nil -} - -// SetupJobAgents ... -func SetupJobAgents(db *sql.DB) error { - - sqlStmt := ` -INSERT INTO job_agent (id, name, description, active, last_updated) VALUES (1, 'agent1', 'Test Agent1', 0, '2018-01-19 21:19:32.448076'); -` - err := execSQL(db, sqlStmt) - if err != nil { - return fmt.Errorf("exec failed %v", err) - } - return nil -} - // SetupJobs ... func SetupJobs(db *sql.DB) error { sqlStmt := ` -INSERT INTO job (id, agent, object_type, object_name, keyword, parameters, asset_url, asset_type, status, start_time, entered_time, job_user, last_updated, job_deliveryservice) VALUES (100, 1, null, null, 'PURGE', 'TTL:48h', 'http://cdn2.edge/job1/.*', 'file', 1, '2018-01-19 21:01:14.000000', '2018-01-19 21:01:14.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.468643', 100); -INSERT INTO job (id, agent, object_type, object_name, keyword, parameters, asset_url, asset_type, status, start_time, entered_time, job_user, last_updated, job_deliveryservice) VALUES (200, 1, null, null, 'PURGE', 'TTL:48h', 'http://cdn2.edge/job2/.*', 'file', 1, '2018-01-19 21:09:34.000000', '2018-01-19 21:09:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.450915', 200); -INSERT INTO job (id, agent, object_type, object_name, keyword, parameters, asset_url, asset_type, status, start_time, entered_time, job_user, last_updated, job_deliveryservice) VALUES (300, 1, null, null, 'PURGE', 'TTL:48h', 'http://cdn2.edge/job3/.*', 'file', 1, '2018-01-19 21:14:34.000000', '2018-01-19 21:14:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.460870', 100); +INSERT INTO job (id, ttl_hr, asset_url, start_time, entered_time, job_user, last_updated, job_deliveryservice, invalidation_type) VALUES (100, 24, 'http://cdn2.edge/job1/.*', '2018-01-19 21:01:14.000000', '2018-01-19 21:01:14.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.468643', 100, 'REFRESH'); +INSERT INTO job (id, ttl_hr, asset_url, start_time, entered_time, job_user, last_updated, job_deliveryservice, invalidation_type) VALUES (200, 36, 'http://cdn2.edge/job2/.*', '2018-01-19 21:09:34.000000', '2018-01-19 21:09:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.450915', 200, 'REFETCH'); +INSERT INTO job (id, ttl_hr, asset_url, start_time, entered_time, job_user, last_updated, job_deliveryservice, invalidation_type) VALUES (300, 72, 'http://cdn2.edge/job3/.*', '2018-01-19 21:14:34.000000', '2018-01-19 21:14:34.000000', (SELECT id FROM tm_user where username = 'admin'), '2018-01-19 21:19:32.460870', 100, 'REFRESH'); ` err := execSQL(db, sqlStmt) if err != nil { @@ -311,8 +273,6 @@ func (r *TCData) Teardown(db *sql.DB) error { DELETE FROM to_extension; DELETE FROM staticdnsentry; DELETE FROM job; - DELETE FROM job_agent; - DELETE FROM job_status; DELETE FROM log; DELETE FROM asn; DELETE FROM deliveryservice_tmuser; From 94b71aabe0a59742663ee4b166955410eeb02cc0 Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Wed, 18 Aug 2021 11:25:31 -0600 Subject: [PATCH 14/35] Fix id comparison for TO client test. Change TP to use stable (3.1) api for invalidation requests --- traffic_ops/testing/api/v4/jobs_test.go | 2 +- traffic_portal/app/src/common/api/JobService.js | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/traffic_ops/testing/api/v4/jobs_test.go b/traffic_ops/testing/api/v4/jobs_test.go index 7768f9ca4e..6e617fdc70 100644 --- a/traffic_ops/testing/api/v4/jobs_test.go +++ b/traffic_ops/testing/api/v4/jobs_test.go @@ -399,7 +399,7 @@ func GetTestJobsByValidData(t *testing.T) { } //Get Jobs by ID - if id != nil && *id >= 1 { + if *id >= 1 { opts := client.NewRequestOptions() opts.QueryParameters.Set("id", strconv.FormatUint(uint64(*id), 10)) toJobs, _, _ = TOSession.GetInvalidationJobs(opts) diff --git a/traffic_portal/app/src/common/api/JobService.js b/traffic_portal/app/src/common/api/JobService.js index 39155f9552..4db2d6161d 100644 --- a/traffic_portal/app/src/common/api/JobService.js +++ b/traffic_portal/app/src/common/api/JobService.js @@ -20,7 +20,7 @@ var JobService = function($http, ENV) { this.getJobs = function(queryParams) { - return $http.get(ENV.api.unstable + 'jobs', {params: queryParams}).then( + return $http.get(ENV.api.stable + 'jobs', {params: queryParams}).then( function(result) { return result.data.response; }, @@ -31,7 +31,7 @@ var JobService = function($http, ENV) { }; this.createJob = function(job) { - return $http.post(ENV.api.unstable + 'jobs', job).then( + return $http.post(ENV.api.stable + 'jobs', job).then( function (result) { return result; }, @@ -42,7 +42,7 @@ var JobService = function($http, ENV) { }; this.deleteJob = function(id) { - return $http.delete(ENV.api.unstable + 'jobs', {params: {id: id}}).then( + return $http.delete(ENV.api.stable + 'jobs', {params: {id: id}}).then( function(result) { return result; }, From afc69381adc8c3566514b0acf223430c12e4e0ef Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Wed, 18 Aug 2021 14:26:37 -0600 Subject: [PATCH 15/35] Remove references to 'job_agent' and 'job_status' from seeds.sql --- traffic_ops/app/db/seeds.sql | 8 -------- 1 file changed, 8 deletions(-) diff --git a/traffic_ops/app/db/seeds.sql b/traffic_ops/app/db/seeds.sql index 67cd77de45..49ef3546e7 100644 --- a/traffic_ops/app/db/seeds.sql +++ b/traffic_ops/app/db/seeds.sql @@ -19,12 +19,6 @@ -- cdns insert into cdn (name, dnssec_enabled, domain_name) values ('ALL', false, '-') ON CONFLICT (name) DO NOTHING; --- job agents -insert into job_agent (name, description, active) values ('dummy', 'Description of Purge Agent', 1) ON CONFLICT (name) DO NOTHING; - --- job statuses -insert into job_status (name, description) values ('PENDING', 'Job is queued, but has not been picked up by any agents yet') ON CONFLICT (name) DO NOTHING; - -- parameters -- Moved into postinstall global parameters insert into profile (name, description, type, cdn) values ('GLOBAL', 'Global Traffic Ops profile, DO NOT DELETE', 'UNK_PROFILE', (SELECT id FROM cdn WHERE name='ALL')) ON CONFLICT (name) DO NOTHING; @@ -917,8 +911,6 @@ insert into last_deleted (table_name) VALUES ('federation_resolver') ON CONFLICT insert into last_deleted (table_name) VALUES ('federation_tmuser') ON CONFLICT (table_name) DO NOTHING; insert into last_deleted (table_name) VALUES ('hwinfo') ON CONFLICT (table_name) DO NOTHING; insert into last_deleted (table_name) VALUES ('job') ON CONFLICT (table_name) DO NOTHING; -insert into last_deleted (table_name) VALUES ('job_agent') ON CONFLICT (table_name) DO NOTHING; -insert into last_deleted (table_name) VALUES ('job_status') ON CONFLICT (table_name) DO NOTHING; insert into last_deleted (table_name) VALUES ('log') ON CONFLICT (table_name) DO NOTHING; insert into last_deleted (table_name) VALUES ('origin') ON CONFLICT (table_name) DO NOTHING; insert into last_deleted (table_name) VALUES ('parameter') ON CONFLICT (table_name) DO NOTHING; From e3e338e538a37d7ee7ba025e511af30d05f1daeb Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Wed, 25 Aug 2021 12:40:35 -0600 Subject: [PATCH 16/35] Update T3C to account for new InvalidationJob type --- cache-config/t3cutil/getdatacfg.go | 2 +- cache-config/t3cutil/toreq/clientfuncs.go | 10 +- lib/go-atscfg/atscfg.go | 32 + lib/go-atscfg/regexrevalidatedotconfig.go | 61 +- .../regexrevalidatedotconfig_test.go | 90 +- lib/go-util/ptr.go | 6 + traffic_ops/results.txt | 1204 +++++++++++++++++ 7 files changed, 1330 insertions(+), 75 deletions(-) create mode 100644 traffic_ops/results.txt diff --git a/cache-config/t3cutil/getdatacfg.go b/cache-config/t3cutil/getdatacfg.go index 84e6fdfc79..7c23615802 100644 --- a/cache-config/t3cutil/getdatacfg.go +++ b/cache-config/t3cutil/getdatacfg.go @@ -69,7 +69,7 @@ type ConfigData struct { Server *atscfg.Server `json:"server,omitempty"` // Jobs must be all Jobs on the server's CDN. May include jobs on other CDNs. - Jobs []tc.InvalidationJob `json:"jobs,omitempty"` + Jobs []tc.InvalidationJobV40 `json:"jobs,omitempty"` // CDN must be the CDN of the server. CDN *tc.CDN `json:"cdn,omitempty"` diff --git a/cache-config/t3cutil/toreq/clientfuncs.go b/cache-config/t3cutil/toreq/clientfuncs.go index a28476ab0b..408c8601c5 100644 --- a/cache-config/t3cutil/toreq/clientfuncs.go +++ b/cache-config/t3cutil/toreq/clientfuncs.go @@ -515,17 +515,21 @@ func (cl *TOClient) GetDeliveryServiceRegexes(reqHdr http.Header) ([]tc.Delivery return regexes, reqInf, nil } +<<<<<<< HEAD func (cl *TOClient) GetJobs(reqHdr http.Header, cdnName string) ([]tc.InvalidationJob, toclientlib.ReqInf, error) { +======= +func (cl *TOClient) GetJobs(reqHdr http.Header) ([]tc.InvalidationJobV40, toclientlib.ReqInf, error) { +>>>>>>> Update T3C to account for new InvalidationJob type if cl.c == nil { oldJobs, inf, err := cl.old.GetJobs() - jobs, err := atscfg.JobsToInvalidationJobs(oldJobs) + jobs, err := atscfg.JobsToInvalidationJobsV40(oldJobs) if err != nil { return nil, inf, errors.New("converting old []tc.Job to []tc.InvalidationJob: " + err.Error()) } return jobs, inf, err } - jobs := []tc.InvalidationJob{} + jobs := []tc.InvalidationJobV40{} reqInf := toclientlib.ReqInf{} err := torequtil.GetRetry(cl.NumRetries, "jobs_cdn_"+cdnName, &jobs, func(obj interface{}) error { opts := *ReqOpts(reqHdr) @@ -535,7 +539,7 @@ func (cl *TOClient) GetJobs(reqHdr http.Header, cdnName string) ([]tc.Invalidati if err != nil { return errors.New("getting jobs from Traffic Ops '" + torequtil.MaybeIPStr(reqInf.RemoteAddr) + "': " + err.Error()) } - jobs := obj.(*[]tc.InvalidationJob) + jobs := obj.(*[]tc.InvalidationJobV40) *jobs = toJobs.Response reqInf = toReqInf return nil diff --git a/lib/go-atscfg/atscfg.go b/lib/go-atscfg/atscfg.go index 7c76df6326..e17febfe60 100644 --- a/lib/go-atscfg/atscfg.go +++ b/lib/go-atscfg/atscfg.go @@ -696,3 +696,35 @@ func JobToInvalidationJob(jb tc.Job) (tc.InvalidationJob, error) { StartTime: &startTime, }, nil } + +func JobsToInvalidationJobsV40(oldJobs []tc.Job) ([]tc.InvalidationJobV40, error) { + jobs := make([]tc.InvalidationJobV40, len(oldJobs), len(oldJobs)) + err := error(nil) + for i, oldJob := range oldJobs { + jobs[i], err = JobToInvalidationJobV40(oldJob) + if err != nil { + return nil, errors.New("converting old tc.Job to tc.InvalidationJob: " + err.Error()) + } + } + return jobs, nil +} + +func JobToInvalidationJobV40(jb tc.Job) (tc.InvalidationJobV40, error) { + startTime := tc.Time{} + if err := json.Unmarshal([]byte(`"`+jb.StartTime+`"`), &startTime); err != nil { + return tc.InvalidationJobV40{}, errors.New("unmarshalling time: " + err.Error()) + } + ttl, err := strconv.Atoi(strings.TrimSuffix(strings.TrimPrefix(jb.Parameters, "TTL:"), "h")) + if err != nil { + return tc.InvalidationJobV40{}, errors.New("unmarshalling ttl: " + err.Error()) + } + return tc.InvalidationJobV40{ + AssetURL: util.StrPtr(jb.AssetURL), + CreatedBy: util.StrPtr(jb.CreatedBy), + DeliveryService: util.StrPtr(jb.DeliveryService), + ID: util.Uint64Ptr(uint64(jb.ID)), + TTLHours: util.UIntPtr(uint(ttl)), + InvalidationType: util.StrPtr(tc.REFRESH), + StartTime: &startTime.Time, + }, nil +} diff --git a/lib/go-atscfg/regexrevalidatedotconfig.go b/lib/go-atscfg/regexrevalidatedotconfig.go index f20d78bdeb..2b10035063 100644 --- a/lib/go-atscfg/regexrevalidatedotconfig.go +++ b/lib/go-atscfg/regexrevalidatedotconfig.go @@ -57,7 +57,7 @@ func MakeRegexRevalidateDotConfig( server *Server, deliveryServices []DeliveryService, globalParams []tc.Parameter, - jobs []tc.InvalidationJob, + jobs []tc.InvalidationJobV40, opt *RegexRevalidateDotConfigOpts, ) (Cfg, error) { if opt == nil { @@ -80,7 +80,7 @@ func MakeRegexRevalidateDotConfig( dsNames[*ds.XMLID] = struct{}{} } - dsJobs := []tc.InvalidationJob{} + dsJobs := []tc.InvalidationJobV40{} for _, job := range jobs { if job.DeliveryService == nil { warnings = append(warnings, "got job from Traffic Ops with a nil DeliveryService! Skipping!") @@ -150,7 +150,7 @@ func (jb jobsSort) Less(i, j int) bool { // - are "purge" jobs // - have a start_time+ttl > now. That is, jobs that haven't expired yet. // Returns the filtered jobs, and any warnings. -func filterJobs(tcJobs []tc.InvalidationJob, maxReval time.Duration, minTTL time.Duration) ([]revalJob, []string) { +func filterJobs(tcJobs []tc.InvalidationJobV40, maxReval time.Duration, minTTL time.Duration) ([]revalJob, []string) { warnings := []string{} jobMap := map[string]revalJob{} @@ -159,26 +159,8 @@ func filterJobs(tcJobs []tc.InvalidationJob, maxReval time.Duration, minTTL time if tcJob.DeliveryService == nil || *tcJob.DeliveryService == "" { continue } - if tcJob.Parameters == nil { - continue - } - if !strings.HasPrefix(*tcJob.Parameters, `TTL:`) { - continue - } - if !strings.HasSuffix(*tcJob.Parameters, `h`) { - continue - } - ttlHoursStr := *tcJob.Parameters - ttlHoursStr = strings.TrimPrefix(ttlHoursStr, `TTL:`) - ttlHoursStr = strings.TrimSuffix(ttlHoursStr, `h`) - ttlHours, err := strconv.Atoi(ttlHoursStr) - if err != nil { - warnings = append(warnings, fmt.Sprintf("job %+v has unexpected parameters ttl format, config generation skipping!\n", tcJob)) - continue - } - - ttl := time.Duration(ttlHours) * time.Hour + ttl := time.Duration(*tcJob.TTLHours) * time.Hour if ttl > maxReval { ttl = maxReval } else if ttl < minTTL { @@ -197,25 +179,13 @@ func filterJobs(tcJobs []tc.InvalidationJob, maxReval time.Duration, minTTL time if tcJob.StartTime.Add(ttl).Before(time.Now()) { continue } - if tcJob.Keyword == nil || *tcJob.Keyword != JobKeywordPurge { - continue - } - if tcJob.AssetURL == nil { continue } // process the __REFETCH__ keyword - assetURL := *tcJob.AssetURL - var jobType string - - if strings.HasSuffix(assetURL, RefetchSuffix) { - assetURL = strings.TrimSuffix(assetURL, RefetchSuffix) - jobType = "MISS" - } else if strings.HasSuffix(assetURL, RefreshSuffix) { // also default - assetURL = strings.TrimSuffix(assetURL, RefreshSuffix) - jobType = "STALE" - } + // There is no InvalidationType string before TO v4.0, so pass empty string + jobType, assetURL := ProcessRefetch(*tcJob.InvalidationType, *tcJob.AssetURL) purgeEnd := tcJob.StartTime.Add(ttl) @@ -232,3 +202,22 @@ func filterJobs(tcJobs []tc.InvalidationJob, maxReval time.Duration, minTTL time return newJobs, warnings } + +const MISS = "MISS" +const STALE = "STALE" + +// ProcessRefetch determines the type of Invalidation, returns the corresponding jobtype +// and "cleans" the regex URL for the asset to be invalidated. REFETCH trumps REFRESH, +// whether in the AssetURL or as InvalidationType +func ProcessRefetch(invalidationType, assetURL string) (string, string) { + + if (len(invalidationType) > 0 && invalidationType == tc.REFETCH) || strings.HasSuffix(assetURL, RefetchSuffix) { + assetURL = strings.TrimSuffix(assetURL, RefetchSuffix) + return MISS, assetURL + } + + // Default value. Either the InvalidationType == REFRESH + // or the suffix is ##REFRESH## or neither + assetURL = strings.TrimSuffix(assetURL, RefreshSuffix) + return STALE, assetURL +} diff --git a/lib/go-atscfg/regexrevalidatedotconfig_test.go b/lib/go-atscfg/regexrevalidatedotconfig_test.go index add0439ab6..3fa2194470 100644 --- a/lib/go-atscfg/regexrevalidatedotconfig_test.go +++ b/lib/go-atscfg/regexrevalidatedotconfig_test.go @@ -41,46 +41,66 @@ func TestMakeRegexRevalidateDotConfig(t *testing.T) { dses := []DeliveryService{*ds} params := makeParamsFromMapArr("GLOBAL", RegexRevalidateFileName, map[string][]string{ - RegexRevalidateMaxRevalDurationDaysParamName: []string{"42"}, - "unrelated": []string{"unrelated0", "unrelated1"}, + RegexRevalidateMaxRevalDurationDaysParamName: {"42"}, + "unrelated": {"unrelated0", "unrelated1"}, }) - jobs := []tc.InvalidationJob{ - tc.InvalidationJob{ - AssetURL: util.StrPtr("assetURL0"), - StartTime: &tc.Time{Time: time.Now().Add(42*24*time.Hour + time.Hour), Valid: true}, - DeliveryService: util.StrPtr("myds"), - CreatedBy: util.StrPtr("me"), - ID: util.UInt64Ptr(42), - Parameters: util.StrPtr("TTL:14h"), - Keyword: util.StrPtr(JobKeywordPurge), + jobs := []tc.InvalidationJobV40{ + { + AssetURL: util.StrPtr("assetURL0"), + StartTime: util.TimePtr(time.Now().Add(42*24*time.Hour + time.Hour)), + DeliveryService: util.StrPtr("myds"), + CreatedBy: util.StrPtr("me"), + ID: util.UInt64Ptr(42), + TTLHours: util.UIntPtr(14), + InvalidationType: util.StrPtr(tc.REFRESH), }, - tc.InvalidationJob{ - AssetURL: util.StrPtr("expiredassetURL0"), - StartTime: &tc.Time{Time: time.Now().Add(-24 * time.Hour), Valid: true}, - DeliveryService: util.StrPtr("expiredmyds"), - CreatedBy: util.StrPtr("expiredme"), - ID: util.UInt64Ptr(42), - Parameters: util.StrPtr("TTL:14h"), - Keyword: util.StrPtr(JobKeywordPurge), + { + AssetURL: util.StrPtr("expiredassetURL0"), + StartTime: util.TimePtr(time.Now().Add(-24 * time.Hour)), + DeliveryService: util.StrPtr("expiredmyds"), + CreatedBy: util.StrPtr("expiredme"), + ID: util.UInt64Ptr(42), + TTLHours: util.UIntPtr(14), + InvalidationType: util.StrPtr(tc.REFRESH), }, - tc.InvalidationJob{ - AssetURL: util.StrPtr("refetchasset##REFETCH##"), - StartTime: &tc.Time{Time: time.Now().Add(24 * time.Hour), Valid: true}, - DeliveryService: util.StrPtr("myds"), - CreatedBy: util.StrPtr("want_refetch"), - ID: util.UInt64Ptr(42), - Parameters: util.StrPtr("TTL:24h"), - Keyword: util.StrPtr(JobKeywordPurge), + { + AssetURL: util.StrPtr("refetchasset##REFETCH##"), + StartTime: util.TimePtr(time.Now().Add(24 * time.Hour)), + DeliveryService: util.StrPtr("myds"), + CreatedBy: util.StrPtr("want_refetch"), + ID: util.UInt64Ptr(42), + TTLHours: util.UIntPtr(24), + InvalidationType: util.StrPtr(tc.REFETCH), }, - tc.InvalidationJob{ - AssetURL: util.StrPtr("refreshasset##REFRESH##"), - StartTime: &tc.Time{Time: time.Now().Add(24 * time.Hour), Valid: true}, - DeliveryService: util.StrPtr("myds"), - CreatedBy: util.StrPtr("want_refresh"), - ID: util.UInt64Ptr(42), - Parameters: util.StrPtr("TTL:24h"), - Keyword: util.StrPtr(JobKeywordPurge), + { + AssetURL: util.StrPtr("refetchtype"), + StartTime: util.TimePtr(time.Now().Add(24 * time.Hour)), + DeliveryService: util.StrPtr("myds"), + CreatedBy: util.StrPtr("want_refetch"), + ID: util.UInt64Ptr(42), + TTLHours: util.UIntPtr(24), + InvalidationType: util.StrPtr(tc.REFETCH), + }, + { + // Mixed assetURL and invalidation type. REFETCH should trump REFRESH + // for backwards compatibility + AssetURL: util.StrPtr("shouldbeREFETCH##REFETCH##"), + StartTime: util.TimePtr(time.Now().Add(24 * time.Hour)), + DeliveryService: util.StrPtr("myds"), + CreatedBy: util.StrPtr("want_refetch"), + ID: util.UInt64Ptr(42), + TTLHours: util.UIntPtr(24), + InvalidationType: util.StrPtr(tc.REFRESH), + }, + { + AssetURL: util.StrPtr("refreshasset##REFRESH##"), + StartTime: util.TimePtr(time.Now().Add(24 * time.Hour)), + DeliveryService: util.StrPtr("myds"), + CreatedBy: util.StrPtr("want_refresh"), + ID: util.UInt64Ptr(42), + TTLHours: util.UIntPtr(24), + InvalidationType: util.StrPtr(tc.REFRESH), }, } diff --git a/lib/go-util/ptr.go b/lib/go-util/ptr.go index 5ffc3d5f1e..94e2b79183 100644 --- a/lib/go-util/ptr.go +++ b/lib/go-util/ptr.go @@ -1,5 +1,7 @@ package util +import "time" + /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -54,3 +56,7 @@ func FloatPtr(f float64) *float64 { func InterfacePtr(i interface{}) *interface{} { return &i } + +func TimePtr(t time.Time) *time.Time { + return &t +} diff --git a/traffic_ops/results.txt b/traffic_ops/results.txt new file mode 100644 index 0000000000..7f0f05fd42 --- /dev/null +++ b/traffic_ops/results.txt @@ -0,0 +1,1204 @@ +? github.com/apache/trafficcontrol/traffic_ops/app/bin/checks/DnssecRefresh [no test files] +? github.com/apache/trafficcontrol/traffic_ops/app/bin/checks/DnssecRefresh/config [no test files] +? github.com/apache/trafficcontrol/traffic_ops/app/db [no test files] +? github.com/apache/trafficcontrol/traffic_ops/app/db/reencrypt [no test files] +=== RUN TestRiakBackend +--- PASS: TestRiakBackend (0.00s) +=== RUN TestPGBackend +--- PASS: TestPGBackend (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/app/db/traffic_vault_migrate (cached) +=== RUN TestValidateParameter +[ERROR] Parameter trafficserver does not match value +[ERROR] Actual Value: 6.2.2 Expected Value: 622 +--- PASS: TestValidateParameter (0.00s) +=== RUN TestDeleteParameter +Deleting parameter {"proxy.cluster.a", "records.config", "INT 1"} +Deleting parameter {"proxy.cluster.b", "records.config", "STRING abc"} +--- PASS: TestDeleteParameter (0.00s) +=== RUN TestModifyConfigFile +Updating parameter {"LogFormat.Format", "logs_xml.config", " >"} to {"LogFormat.Format", "logging.config", " >"} +Updating parameter {"LogFormat.Name", "logs_xml.config", "custom_ats"} to {"LogFormat.Name", "logging.config", "custom_ats"} +--- PASS: TestModifyConfigFile (0.00s) +=== RUN TestModifyValueForce +Updating parameter {"proxy.config.hostdb.timeout", "records.config", "INT 1440"} to {"proxy.config.hostdb.timeout", "records.config", "INT 86400"} +--- PASS: TestModifyValueForce (0.00s) +=== RUN TestModifyValueSkip +[ACTION REQUIRED] Found modified value. Skip modifying {"proxy.config.hostdb.timeout", "records.config", "INT 5000"}. Please update manually +--- PASS: TestModifyValueSkip (0.00s) +=== RUN TestModifyNameValue +Updating parameter {"proxy.config.log.xml_config_file", "records.config", "logs_xml.config"} to {"proxy.config.log.config.filename", "records.config", "logging.config"} +--- PASS: TestModifyNameValue (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/install/bin/convert_profile (cached) +? github.com/apache/trafficcontrol/traffic_ops/testing/api/config [no test files] +? github.com/apache/trafficcontrol/traffic_ops/testing/api/utils [no test files] +Error Loading Config: failed to read CDN configuration: open traffic-ops-test.conf: no such file or directory +FAIL github.com/apache/trafficcontrol/traffic_ops/testing/api/v2 0.203s +Error Loading Config: failed to read CDN configuration: open traffic-ops-test.conf: no such file or directory +FAIL github.com/apache/trafficcontrol/traffic_ops/testing/api/v3 0.289s +INFO: traffic_ops_test.go:71: 2021-08-19T21:50:36.492742Z: Using Config values: + TO Config File: traffic-ops-test.conf + TO Fixtures: tc-fixtures.json + TO URL: https://localhost:8443 + TO Session Timeout In Secs: 60 + DB Server: localhost + DB User: traffic_ops + DB Name: to_test + DB Ssl: false + UseIMS: true + +Error creating session to https://localhost:8443 - traffic_ops, logging in: Login error Post "https://localhost:8443/api/4.0/user/login": dial tcp [::1]:8443: connect: connection refused, alerts string: {Alerts:[]} +FAIL github.com/apache/trafficcontrol/traffic_ops/testing/api/v4 0.431s +? github.com/apache/trafficcontrol/traffic_ops/toclientlib [no test files] +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang [no test files] +=== RUN TestSplitRPMVersion +--- PASS: TestSplitRPMVersion (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/about (cached) +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/acme [no test files] +=== RUN TestCamelCase +--- PASS: TestCamelCase (0.00s) +=== RUN TestRespWrittenAfterErrFails +--- PASS: TestRespWrittenAfterErrFails (0.00s) +=== RUN TestWriteAlertsObjEmpty +--- PASS: TestWriteAlertsObjEmpty (0.00s) +=== RUN TestWriteAlertsObj +--- PASS: TestWriteAlertsObj (0.00s) +=== RUN TestWriteResp +--- PASS: TestWriteResp (0.00s) +=== RUN TestWriteRespRaw +--- PASS: TestWriteRespRaw (0.00s) +=== RUN TestWriteRespVals +--- PASS: TestWriteRespVals (0.00s) +=== RUN TestRespWriter +--- PASS: TestRespWriter (0.00s) +=== RUN TestRespWriterVals +--- PASS: TestRespWriterVals (0.00s) +=== RUN TestWriteRespAlert +--- PASS: TestWriteRespAlert (0.00s) +=== RUN TestWriteRespAlertObj +--- PASS: TestWriteRespAlertObj (0.00s) +=== RUN TestParseRestrictFKConstraint +=== RUN TestParseRestrictFKConstraint/FK_Constraint_Error + api_test.go:245: Starting test scenario: FK Constraint Error +=== RUN TestParseRestrictFKConstraint/FK_Constraint_Error_with_underscores_in_table_name + api_test.go:245: Starting test scenario: FK Constraint Error with underscores in table name +=== RUN TestParseRestrictFKConstraint/Non_FK_Constraint_Error + api_test.go:245: Starting test scenario: Non FK Constraint Error +--- PASS: TestParseRestrictFKConstraint (0.00s) + --- PASS: TestParseRestrictFKConstraint/FK_Constraint_Error (0.00s) + --- PASS: TestParseRestrictFKConstraint/FK_Constraint_Error_with_underscores_in_table_name (0.00s) + --- PASS: TestParseRestrictFKConstraint/Non_FK_Constraint_Error (0.00s) +=== RUN TestInsertAsyncStatus +--- PASS: TestInsertAsyncStatus (0.00s) +=== RUN TestUpdateAsyncStatus +--- PASS: TestUpdateAsyncStatus (0.00s) +=== RUN TestUpdateAsyncStatusFinished +--- PASS: TestUpdateAsyncStatusFinished (0.00s) +=== RUN TestCreateChangeLog +--- PASS: TestCreateChangeLog (0.00s) +=== RUN TestCreateHandler +--- PASS: TestCreateHandler (0.00s) +=== RUN TestReadHandler +--- PASS: TestReadHandler (0.00s) +=== RUN TestReadHandlerIMS +--- PASS: TestReadHandlerIMS (0.00s) +=== RUN TestUpdateHandler +--- PASS: TestUpdateHandler (0.00s) +=== RUN TestDeleteHandler +--- PASS: TestDeleteHandler (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/api (cached) +=== RUN TestGetAPICapabilities +--- PASS: TestGetAPICapabilities (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/apicapability (cached) +=== RUN TestInterfaces +--- PASS: TestInterfaces (0.00s) +=== RUN TestIsUpdateable +--- PASS: TestIsUpdateable (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/apitenant (cached) +=== RUN TestGetASNs +--- PASS: TestGetASNs (0.00s) +=== RUN TestInterfaces +--- PASS: TestInterfaces (0.00s) +=== RUN TestValidate +--- PASS: TestValidate (0.00s) +=== RUN TestCheckNumberForUpdate +--- PASS: TestCheckNumberForUpdate (0.00s) +=== RUN TestASNExistsForUpdateFailure +--- PASS: TestASNExistsForUpdateFailure (0.00s) +=== RUN TestASNExistsForUpdateSuccess +--- PASS: TestASNExistsForUpdateSuccess (0.00s) +=== RUN TestASNExists +--- PASS: TestASNExists (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/asn (cached) +=== RUN TestDeriveGoodPassword +--- PASS: TestDeriveGoodPassword (0.08s) +=== RUN TestDeriveBadPassword +--- PASS: TestDeriveBadPassword (0.08s) +=== RUN TestScryptPasswordIsRequired +--- PASS: TestScryptPasswordIsRequired (0.00s) +=== RUN TestUsernamePassword +--- PASS: TestUsernamePassword (0.01s) +=== RUN TestCommonPassword +--- PASS: TestCommonPassword (0.01s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/auth (cached) +=== RUN TestReadCacheGroups +--- PASS: TestReadCacheGroups (0.00s) +=== RUN TestFuncs +--- PASS: TestFuncs (0.00s) +=== RUN TestInterfaces +--- PASS: TestInterfaces (0.00s) +=== RUN TestValidate +--- PASS: TestValidate (0.00s) +=== RUN TestBadTypeParamCacheGroups +--- PASS: TestBadTypeParamCacheGroups (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/cachegroup (cached) +=== RUN TestReadCacheGroupParameters +=== RUN TestReadCacheGroupParameters/Success:_Read_Cache_Group_Parameters_-_Parameters + parameters_test.go:160: Starting test scenario: Success: Read Cache Group Parameters - Parameters +=== RUN TestReadCacheGroupParameters/Success:_Read_Cache_Group_Parameters_with_parameter_id_-_Parameters + parameters_test.go:160: Starting test scenario: Success: Read Cache Group Parameters with parameter id - Parameters +=== RUN TestReadCacheGroupParameters/Success:_Read_Cache_Group_Parameters_no_data_-_Parameters + parameters_test.go:160: Starting test scenario: Success: Read Cache Group Parameters no data - Parameters +=== RUN TestReadCacheGroupParameters/Failure:_Storage_Error_reading_Cache_Group_Parameters_-_Parameters + parameters_test.go:160: Starting test scenario: Failure: Storage Error reading Cache Group Parameters - Parameters +=== RUN TestReadCacheGroupParameters/Failure:_User_Error_invalid_params_-_Parameters + parameters_test.go:160: Starting test scenario: Failure: User Error invalid params - Parameters +=== RUN TestReadCacheGroupParameters/Failure:_System_Error_getting_cache_group_-_Parameters + parameters_test.go:160: Starting test scenario: Failure: System Error getting cache group - Parameters +=== RUN TestReadCacheGroupParameters/Failure:_Cache_group_does_not_exist_-_Parameters + parameters_test.go:160: Starting test scenario: Failure: Cache group does not exist - Parameters +--- PASS: TestReadCacheGroupParameters (0.00s) + --- PASS: TestReadCacheGroupParameters/Success:_Read_Cache_Group_Parameters_-_Parameters (0.00s) + --- PASS: TestReadCacheGroupParameters/Success:_Read_Cache_Group_Parameters_with_parameter_id_-_Parameters (0.00s) + --- PASS: TestReadCacheGroupParameters/Success:_Read_Cache_Group_Parameters_no_data_-_Parameters (0.00s) + --- PASS: TestReadCacheGroupParameters/Failure:_Storage_Error_reading_Cache_Group_Parameters_-_Parameters (0.00s) + --- PASS: TestReadCacheGroupParameters/Failure:_User_Error_invalid_params_-_Parameters (0.00s) + --- PASS: TestReadCacheGroupParameters/Failure:_System_Error_getting_cache_group_-_Parameters (0.00s) + --- PASS: TestReadCacheGroupParameters/Failure:_Cache_group_does_not_exist_-_Parameters (0.00s) +=== RUN TestInterfaces +--- PASS: TestInterfaces (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/cachegroupparameter (cached) +=== RUN TestAddStatsInvalidString +--- PASS: TestAddStatsInvalidString (0.00s) +=== RUN TestAddStatsValidString +--- PASS: TestAddStatsValidString (0.00s) +=== RUN TestAddStatsEmptyJsonObject +--- PASS: TestAddStatsEmptyJsonObject (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/cachesstats (cached) +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/capabilities [no test files] +=== RUN TestGetStatsFromServiceInterface +--- PASS: TestGetStatsFromServiceInterface (0.00s) +=== RUN TestReadCDNs +--- PASS: TestReadCDNs (0.00s) +=== RUN TestFuncs +--- PASS: TestFuncs (0.00s) +=== RUN TestInterfaces +--- PASS: TestInterfaces (0.00s) +=== RUN TestValidate +--- PASS: TestValidate (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/cdn (cached) +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/cdn_lock [no test files] +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/cdnfederation [no test files] +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/cdnnotification [no test files] +=== RUN TestLogLocation +--- PASS: TestLogLocation (0.00s) +=== RUN TestLoadConfig +--- PASS: TestLoadConfig (0.00s) +=== RUN TestValidateRoutingBlacklist +--- PASS: TestValidateRoutingBlacklist (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/config (cached) +=== RUN TestReadCoordinates +--- PASS: TestReadCoordinates (0.00s) +=== RUN TestFuncs +--- PASS: TestFuncs (0.00s) +=== RUN TestInterfaces +--- PASS: TestInterfaces (0.00s) +=== RUN TestValidate +--- PASS: TestValidate (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/coordinate (cached) +=== RUN TestGetConfigParams +--- PASS: TestGetConfigParams (0.00s) +=== RUN TestMakeCRConfigConfig +--- PASS: TestMakeCRConfigConfig (0.00s) +=== RUN TestGetTMURLHost +--- PASS: TestGetTMURLHost (0.00s) +=== RUN TestMakeDSes +--- PASS: TestMakeDSes (0.01s) +=== RUN TestGetServerProfileParams +--- PASS: TestGetServerProfileParams (0.00s) +=== RUN TestGetDSRegexesDomains +--- PASS: TestGetDSRegexesDomains (0.00s) +=== RUN TestGetStaticDNSEntries +--- PASS: TestGetStaticDNSEntries (0.00s) +=== RUN TestMakeLocations +--- PASS: TestMakeLocations (0.00s) +=== RUN TestGetServerParams +--- PASS: TestGetServerParams (0.00s) +=== RUN TestGetAllServers +--- PASS: TestGetAllServers (0.00s) +=== RUN TestGetAllServersNonService +--- PASS: TestGetAllServersNonService (0.00s) +=== RUN TestGetServerDSNames +--- PASS: TestGetServerDSNames (0.00s) +=== RUN TestGetServerDSes +--- PASS: TestGetServerDSes (0.00s) +=== RUN TestGetCDNInfo +--- PASS: TestGetCDNInfo (0.00s) +=== RUN TestGetCDNNameFromID +--- PASS: TestGetCDNNameFromID (0.00s) +=== RUN TestGetSnapshot +--- PASS: TestGetSnapshot (0.00s) +=== RUN TestSnapshot +--- PASS: TestSnapshot (0.00s) +=== RUN TestMakeStats +--- PASS: TestMakeStats (0.00s) +=== RUN TestMakeTops +--- PASS: TestMakeTops (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/crconfig (cached) +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/crstats [no test files] +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/dbdump [no test files] +=== RUN TestBuildQuery +--- PASS: TestBuildQuery (0.00s) +=== RUN TestGetCacheGroupByName +=== RUN TestGetCacheGroupByName/Success:_Cache_Group_exists + db_helpers_test.go:116: Starting test scenario: Success: Cache Group exists +=== RUN TestGetCacheGroupByName/Failure:_Cache_Group_does_not_exist + db_helpers_test.go:116: Starting test scenario: Failure: Cache Group does not exist +=== RUN TestGetCacheGroupByName/Failure:_Storage_error_getting_Cache_Group + db_helpers_test.go:116: Starting test scenario: Failure: Storage error getting Cache Group +--- PASS: TestGetCacheGroupByName (0.00s) + --- PASS: TestGetCacheGroupByName/Success:_Cache_Group_exists (0.00s) + --- PASS: TestGetCacheGroupByName/Failure:_Cache_Group_does_not_exist (0.00s) + --- PASS: TestGetCacheGroupByName/Failure:_Storage_error_getting_Cache_Group (0.00s) +=== RUN TestGetServerInterfaces +--- PASS: TestGetServerInterfaces (0.00s) +=== RUN TestGetDSIDAndCDNFromName +=== RUN TestGetDSIDAndCDNFromName/Success:_DS_ID_and_CDN_Name_found + db_helpers_test.go:311: Starting test scenario: Success: DS ID and CDN Name found +=== RUN TestGetDSIDAndCDNFromName/Failure:_DS_ID_or_CDN_Name_not_found + db_helpers_test.go:311: Starting test scenario: Failure: DS ID or CDN Name not found +=== RUN TestGetDSIDAndCDNFromName/Failure:_Storage_error_getting_DS_ID_or_CDN_Name + db_helpers_test.go:311: Starting test scenario: Failure: Storage error getting DS ID or CDN Name +--- PASS: TestGetDSIDAndCDNFromName (0.00s) + --- PASS: TestGetDSIDAndCDNFromName/Success:_DS_ID_and_CDN_Name_found (0.00s) + --- PASS: TestGetDSIDAndCDNFromName/Failure:_DS_ID_or_CDN_Name_not_found (0.00s) + --- PASS: TestGetDSIDAndCDNFromName/Failure:_Storage_error_getting_DS_ID_or_CDN_Name (0.00s) +=== RUN TestGetCDNIDFromName +=== RUN TestGetCDNIDFromName/Success:_CDN_ID_found + db_helpers_test.go:372: Starting test scenario: Success: CDN ID found +=== RUN TestGetCDNIDFromName/Failure:_CDN_ID_not_found + db_helpers_test.go:372: Starting test scenario: Failure: CDN ID not found +=== RUN TestGetCDNIDFromName/Failure:_Storage_error_getting_CDN_ID + db_helpers_test.go:372: Starting test scenario: Failure: Storage error getting CDN ID +--- PASS: TestGetCDNIDFromName (0.00s) + --- PASS: TestGetCDNIDFromName/Success:_CDN_ID_found (0.00s) + --- PASS: TestGetCDNIDFromName/Failure:_CDN_ID_not_found (0.00s) + --- PASS: TestGetCDNIDFromName/Failure:_Storage_error_getting_CDN_ID (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/dbhelpers (cached) +=== RUN TestDeliveryServicesRequiredCapabilityInterfaces +--- PASS: TestDeliveryServicesRequiredCapabilityInterfaces (0.00s) +=== RUN TestCreateDeliveryServicesRequiredCapability +--- PASS: TestCreateDeliveryServicesRequiredCapability (0.00s) +=== RUN TestUnauthorizedCreateDeliveryServicesRequiredCapability +--- PASS: TestUnauthorizedCreateDeliveryServicesRequiredCapability (0.00s) +=== RUN TestReadDeliveryServicesRequiredCapability +--- PASS: TestReadDeliveryServicesRequiredCapability (0.00s) +=== RUN TestDeleteDeliveryServicesRequiredCapability +--- PASS: TestDeleteDeliveryServicesRequiredCapability (0.00s) +=== RUN TestUnauthorizedDeleteDeliveryServicesRequiredCapability +--- PASS: TestUnauthorizedDeleteDeliveryServicesRequiredCapability (0.00s) +=== RUN TestCreateDeliveryServicesRequiredCapabilityInvalidDSType +--- PASS: TestCreateDeliveryServicesRequiredCapabilityInvalidDSType (0.00s) +=== RUN TestGetDetails +--- PASS: TestGetDetails (0.00s) +=== RUN TestGetOldDetailsError +--- PASS: TestGetOldDetailsError (0.00s) +=== RUN TestGetDeliveryServicesMatchLists +--- PASS: TestGetDeliveryServicesMatchLists (0.00s) +=== RUN TestGetDSTLSVersions +--- PASS: TestGetDSTLSVersions (0.00s) +=== RUN TestMakeExampleURLs +--- PASS: TestMakeExampleURLs (0.00s) +=== RUN TestReadGetDeliveryServices +--- PASS: TestReadGetDeliveryServices (0.00s) +=== RUN TestGetEligibleServers +--- PASS: TestGetEligibleServers (0.00s) +=== RUN TestDecodePrivateKeyPKCS8RSA2048 +--- PASS: TestDecodePrivateKeyPKCS8RSA2048 (0.00s) +=== RUN TestDecodePrivateKeyPKCS1RSA2048 +--- PASS: TestDecodePrivateKeyPKCS1RSA2048 (0.00s) +=== RUN TestDecodePrivateKeyECDSANISTPrime256V1 +--- PASS: TestDecodePrivateKeyECDSANISTPrime256V1 (0.00s) +=== RUN TestDecodePrivateKeyECDSANISTPrime256V1WithoutParams +--- PASS: TestDecodePrivateKeyECDSANISTPrime256V1WithoutParams (0.00s) +=== RUN TestDecodePrivateKeyECDSANISTSecP384R1 +--- PASS: TestDecodePrivateKeyECDSANISTSecP384R1 (0.00s) +=== RUN TestDecodePrivateKeyECDSANISTSecP384R1WithoutParams +--- PASS: TestDecodePrivateKeyECDSANISTSecP384R1WithoutParams (0.00s) +=== RUN TestDecodeRSAPrivateKeyBadData + keys_test.go:2202: expected error message: private key validation error: no RSA private key PEM blocks found +--- PASS: TestDecodeRSAPrivateKeyBadData (0.00s) +=== RUN TestDecodeECDSAPrivateKeyBadData + keys_test.go:2213: expected error message: private key validation error: no EC private key PEM blocks found +--- PASS: TestDecodeECDSAPrivateKeyBadData (0.00s) +=== RUN TestDecodePrivateKeyRSAEncrypted + keys_test.go:2224: expected error message: private key validation error: encrypted private key not supported - header: 4,ENCRYPTED +--- PASS: TestDecodePrivateKeyRSAEncrypted (0.00s) +=== RUN TestDecodePrivateKeyECDSAEncrypted + keys_test.go:2235: expected error message: private key validation error: encrypted private key not supported - header: 4,ENCRYPTED +--- PASS: TestDecodePrivateKeyECDSAEncrypted (0.00s) +=== RUN TestVerifyAndEncodeCertificateBadData +--- PASS: TestVerifyAndEncodeCertificateBadData (0.00s) +=== RUN TestVerifyAndEncodeCertificateSelfSignedCertKeyPairDSA + keys_test.go:2254: expected error message: cert/key validation error: DSA public key algorithm unsupported +--- PASS: TestVerifyAndEncodeCertificateSelfSignedCertKeyPairDSA (0.00s) +=== RUN TestVerifyAndEncodeCertificateSelfSignedX509v1 +--- PASS: TestVerifyAndEncodeCertificateSelfSignedX509v1 (0.11s) +=== RUN TestVerifyAndEncodeCertificateSelfSignedNoSkiAkiCertKeyPair +--- PASS: TestVerifyAndEncodeCertificateSelfSignedNoSkiAkiCertKeyPair (0.00s) +=== RUN TestVerifyAndEncodeCertificateSelfSignedCertKeyPair +--- PASS: TestVerifyAndEncodeCertificateSelfSignedCertKeyPair (0.00s) +=== RUN TestVerifyAndEncodeCertificateSelfSignedCertKeyPairMisMatchedPrivateKey + keys_test.go:2344: expected error message: cert/key (rsa) mismatch error: RSA public N modulus value mismatch +--- PASS: TestVerifyAndEncodeCertificateSelfSignedCertKeyPairMisMatchedPrivateKey (0.00s) +=== RUN TestVerifyAndEncodeCertificateSelfSignedNoServerAuthExtKeyUsage + keys_test.go:2356: expected error message: certificate (x509v3) validation error: server certificate missing 'serverAuth' extended key usage +--- PASS: TestVerifyAndEncodeCertificateSelfSignedNoServerAuthExtKeyUsage (0.00s) +=== RUN TestVerifyAndEncodeCertificateSelfSignedRSANoKeyEnciphermentKeyUsage + keys_test.go:2369: expected error message: cert/key (rsa) validation: no keyEncipherment keyUsage extension present in x509v3 server certificate +--- PASS: TestVerifyAndEncodeCertificateSelfSignedRSANoKeyEnciphermentKeyUsage (0.00s) +=== RUN TestVerifyAndEncodeCertificateCASignedCertKeyPair +--- PASS: TestVerifyAndEncodeCertificateCASignedCertKeyPair (0.00s) +=== RUN TestVerifyAndEncodeCertificateCASignedCertKeyPairWithRootCA +--- PASS: TestVerifyAndEncodeCertificateCASignedCertKeyPairWithRootCA (0.00s) +=== RUN TestVerifyAndEncodeCertificateCASignedNoSkiAkiCertKeyPair +--- PASS: TestVerifyAndEncodeCertificateCASignedNoSkiAkiCertKeyPair (0.00s) +=== RUN TestVerifyAndEncodeCertificateCASignedNoSkiAkiCertKeyPairWithRootCA +--- PASS: TestVerifyAndEncodeCertificateCASignedNoSkiAkiCertKeyPairWithRootCA (0.00s) +=== RUN TestVerifyAndEncodeCertificateECDSASelfSignedCertificateKeyPairWithoutDigitalSignatureKeyUsage + keys_test.go:2489: expected error message: cert/key (ecdsa) validation error: no digitalSignature keyUsage extension present in x509v3 server certificate +--- PASS: TestVerifyAndEncodeCertificateECDSASelfSignedCertificateKeyPairWithoutDigitalSignatureKeyUsage (0.00s) +=== RUN TestVerifyAndEncodeCertificateECDSASelfSignedCertificateKeyPair +--- PASS: TestVerifyAndEncodeCertificateECDSASelfSignedCertificateKeyPair (0.00s) +=== RUN TestVerifyAndEncodeCertificateECDSASelfSignedCertificateKeyPairECDisabled + keys_test.go:2509: expected error message: cert/key validation error: ECDSA public key algorithm unsupported for non-DNS delivery service type +--- PASS: TestVerifyAndEncodeCertificateECDSASelfSignedCertificateKeyPairECDisabled (0.00s) +=== RUN TestVerifyAndEncodeCertificateECDSASelfSignedCertificateKeyPairWithoutParams +--- PASS: TestVerifyAndEncodeCertificateECDSASelfSignedCertificateKeyPairWithoutParams (0.00s) +=== RUN TestVerifyAndEncodeCertificateECDSASelfSignedCertificateKeyPairMisMatchedPrivateKey + keys_test.go:2529: expected error message: cert/key (ecdsa) mismatch error: ECDSA public X value mismatch +--- PASS: TestVerifyAndEncodeCertificateECDSASelfSignedCertificateKeyPairMisMatchedPrivateKey (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/deliveryservice (cached) +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/deliveryservice/consistenthash [no test files] +=== RUN TestGetAssignee +--- PASS: TestGetAssignee (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/deliveryservice/request (cached) +=== RUN TestFuncs +--- PASS: TestFuncs (0.00s) +=== RUN TestInterfaces +--- PASS: TestInterfaces (0.00s) +=== RUN TestValidate +--- PASS: TestValidate (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/deliveryservice/request/comment (cached) +=== RUN TestValidateDSSAssignments +--- PASS: TestValidateDSSAssignments (0.00s) +=== RUN TestReadServers +--- PASS: TestReadServers (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/deliveryservice/servers (cached) +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/deliveryservicerequests [no test files] +=== RUN TestValidateDSRegexOrderExisting +--- PASS: TestValidateDSRegexOrderExisting (0.00s) +=== RUN TestValidateDSRegex +--- PASS: TestValidateDSRegex (0.00s) +=== RUN TestUpdateImmutableRegex +--- PASS: TestUpdateImmutableRegex (0.00s) +=== RUN TestGetCurrentDetails +--- PASS: TestGetCurrentDetails (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/deliveryservicesregexes (cached) +=== RUN TestGetDivisions +--- PASS: TestGetDivisions (0.00s) +=== RUN TestInterfaces +--- PASS: TestInterfaces (0.00s) +=== RUN TestValidation +--- PASS: TestValidation (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/division (cached) +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/federation_resolvers [no test files] +=== RUN TestCheckFedDSDeletion +=== RUN TestCheckFedDSDeletion/Success:_Deleted_Federation_Delivery_Service + ds_test.go:68: Starting test scenario: Success: Deleted Federation Delivery Service +=== RUN TestCheckFedDSDeletion/Failure:_Remove_last_Federation_Delivery_Service + ds_test.go:68: Starting test scenario: Failure: Remove last Federation Delivery Service +=== RUN TestCheckFedDSDeletion/Failure:_Federation_not_found + ds_test.go:68: Starting test scenario: Failure: Federation not found +=== RUN TestCheckFedDSDeletion/Failure:_Delivery_Service_not_found + ds_test.go:68: Starting test scenario: Failure: Delivery Service not found +--- PASS: TestCheckFedDSDeletion (0.00s) + --- PASS: TestCheckFedDSDeletion/Success:_Deleted_Federation_Delivery_Service (0.00s) + --- PASS: TestCheckFedDSDeletion/Failure:_Remove_last_Federation_Delivery_Service (0.00s) + --- PASS: TestCheckFedDSDeletion/Failure:_Federation_not_found (0.00s) + --- PASS: TestCheckFedDSDeletion/Failure:_Delivery_Service_not_found (0.00s) +=== RUN TestAddFederationResolverMappingsForCurrentUser +=== RUN TestAddFederationResolverMappingsForCurrentUser/add_Federation_Resolver_Mappings_for_the_current_user +=== RUN TestAddFederationResolverMappingsForCurrentUser/add_Federation_Resolver_Mappings_for_the_current_user_when_no_federations_exist/are_assigned_to_them + federations_test.go:156: Got expected user error: No federation(s) found for user test on delivery service 'test'. +=== RUN TestAddFederationResolverMappingsForCurrentUser/add_Federation_Resolver_Mappings_for_a_DS_unauthorized_to_the_current_user's_tenant + federations_test.go:210: Got expected user error: 'test' - no such Delivery Service + federations_test.go:215: Got expected system error: User 'test' requested unauthorized federation resolver mapping modification on the 'test' Delivery Service +--- PASS: TestAddFederationResolverMappingsForCurrentUser (0.00s) + --- PASS: TestAddFederationResolverMappingsForCurrentUser/add_Federation_Resolver_Mappings_for_the_current_user (0.00s) + --- PASS: TestAddFederationResolverMappingsForCurrentUser/add_Federation_Resolver_Mappings_for_the_current_user_when_no_federations_exist/are_assigned_to_them (0.00s) + --- PASS: TestAddFederationResolverMappingsForCurrentUser/add_Federation_Resolver_Mappings_for_a_DS_unauthorized_to_the_current_user's_tenant (0.00s) +=== RUN TestGetMappingsFromRequestBody +--- PASS: TestGetMappingsFromRequestBody (0.00s) +=== RUN TestRemoveFederationResolverMappingsForCurrentUser +--- PASS: TestRemoveFederationResolverMappingsForCurrentUser (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/federations (cached) +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/invalidationjobs [no test files] +=== RUN TestCrypt +=== RUN TestCrypt/#00 + crypt_test.go:68: crypt("password", "salt") = "$1$salt$qJH7.N4xYta3aEG/dfqo/0" +=== RUN TestCrypt/#01 + crypt_test.go:68: crypt("Traffic Ops", "pepper") = "$1$pepper$AHauHHBeRPuBP0LCO0oBW0" +=== RUN TestCrypt/#02 + crypt_test.go:68: crypt("T0p S3cr3T", "N4xYta3a") = "$1$N4xYta3a$E4g3CFzttHfxgEvY4PmrI/" +=== RUN TestCrypt/#03 + crypt_test.go:68: crypt("a", "b") = "$1$b$J4vSIPg.1IiJxJ.JOHsOS1" +=== RUN TestCrypt/#04 + crypt_test.go:68: crypt("", "salt") = "$1$salt$UsdFqFVB.FsuinRDK5eE.." +=== RUN TestCrypt/#05 + crypt_test.go:68: crypt("pw", "") = "$1$$F0Fc2lbYpzr3KKdKkM0Wj." +=== RUN TestCrypt/#06 + crypt_test.go:68: crypt("", "") = "$1$$qRPK7m23GJusamGpoGLby/" +--- PASS: TestCrypt (0.00s) + --- PASS: TestCrypt/#00 (0.00s) + --- PASS: TestCrypt/#01 (0.00s) + --- PASS: TestCrypt/#02 (0.00s) + --- PASS: TestCrypt/#03 (0.00s) + --- PASS: TestCrypt/#04 (0.00s) + --- PASS: TestCrypt/#05 (0.00s) + --- PASS: TestCrypt/#06 (0.00s) +=== RUN TestRndSalt +=== RUN TestRndSalt/0 +=== RUN TestRndSalt/1 +=== RUN TestRndSalt/2 +=== RUN TestRndSalt/3 +=== RUN TestRndSalt/4 +=== RUN TestRndSalt/5 +=== RUN TestRndSalt/6 +=== RUN TestRndSalt/7 +=== RUN TestRndSalt/8 +=== RUN TestRndSalt/9 +--- PASS: TestRndSalt (0.00s) + --- PASS: TestRndSalt/0 (0.00s) + --- PASS: TestRndSalt/1 (0.00s) + --- PASS: TestRndSalt/2 (0.00s) + --- PASS: TestRndSalt/3 (0.00s) + --- PASS: TestRndSalt/4 (0.00s) + --- PASS: TestRndSalt/5 (0.00s) + --- PASS: TestRndSalt/6 (0.00s) + --- PASS: TestRndSalt/7 (0.00s) + --- PASS: TestRndSalt/8 (0.00s) + --- PASS: TestRndSalt/9 (0.00s) +=== RUN TestHelperMockCmd +--- PASS: TestHelperMockCmd (0.00s) +=== RUN TestISOS +=== RUN TestISOS/cmd_success + iso_test.go:83: response status = 200 + iso_test.go:90: header "Content-Type" = "attachment; filename=\"db.infra.ciab.test-centos72.iso\"" + iso_test.go:97: header "Content-Type" = "application/octet-stream" + iso_test.go:106: got command: "mkisofs -joliet-long -input-charset utf-8 -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 -boot-info-table -R -J -v -T /var/folders/r5/snb0k76d0n742rp0mtwzmx_80000gp/T/TestISOS2460179975/centos72" +=== RUN TestISOS/cmd_failure + iso_test.go:144: header "Content-Type" = "application/json" + iso_test.go:153: response: tc.Alerts{Alerts:[]tc.Alert{tc.Alert{Text:"Internal Server Error", Level:"error"}}} +--- PASS: TestISOS (2.87s) + --- PASS: TestISOS/cmd_success (0.81s) + --- PASS: TestISOS/cmd_failure (2.06s) +=== RUN TestISORequest_validate +=== RUN TestISORequest_validate/valid_with_dhcp_false + iso_test.go:487: isoRequest.validate() = +=== RUN TestISORequest_validate/valid_with_CIDR_prefix_on_IPv6_address + iso_test.go:487: isoRequest.validate() = +=== RUN TestISORequest_validate/valid_with_valid_IPv6_address + iso_test.go:487: isoRequest.validate() = +=== RUN TestISORequest_validate/invalid_with_valid_IPv4_address_as_an_IPv6 + iso_test.go:487: isoRequest.validate() = ip6Address must be a valid IPv6 address (with optional CIDR prefix) +=== RUN TestISORequest_validate/invalid_with_invalid_IPv6 + iso_test.go:487: isoRequest.validate() = ip6Address must be a valid IPv6 address (with optional CIDR prefix) +=== RUN TestISORequest_validate/valid_with_dhcp_true + iso_test.go:487: isoRequest.validate() = +=== RUN TestISORequest_validate/invalid_with_dhcp_false + iso_test.go:487: isoRequest.validate() = ipAddress is required if DHCP is no, ipNetmask is required if DHCP is no, ipGateway is required if DHCP is no +=== RUN TestISORequest_validate/valid_with_mgmt_addr + iso_test.go:487: isoRequest.validate() = +=== RUN TestISORequest_validate/invalid_with_mgmt_addr + iso_test.go:487: isoRequest.validate() = mgmtInterface is required when mgmtIpAddress is provided +=== RUN TestISORequest_validate/invalid_with_zero_values + iso_test.go:487: isoRequest.validate() = osversionDir is required, hostName is required, domainName is required, rootPass is required, dhcp is required, interfaceMtu is required, disk is required +--- PASS: TestISORequest_validate (0.00s) + --- PASS: TestISORequest_validate/valid_with_dhcp_false (0.00s) + --- PASS: TestISORequest_validate/valid_with_CIDR_prefix_on_IPv6_address (0.00s) + --- PASS: TestISORequest_validate/valid_with_valid_IPv6_address (0.00s) + --- PASS: TestISORequest_validate/invalid_with_valid_IPv4_address_as_an_IPv6 (0.00s) + --- PASS: TestISORequest_validate/invalid_with_invalid_IPv6 (0.00s) + --- PASS: TestISORequest_validate/valid_with_dhcp_true (0.00s) + --- PASS: TestISORequest_validate/invalid_with_dhcp_false (0.00s) + --- PASS: TestISORequest_validate/valid_with_mgmt_addr (0.00s) + --- PASS: TestISORequest_validate/invalid_with_mgmt_addr (0.00s) + --- PASS: TestISORequest_validate/invalid_with_zero_values (0.00s) +=== RUN TestBoolStr_UnmarshalText +=== RUN TestBoolStr_UnmarshalText/no + iso_test.go:530: got {isSet:true v:false} +=== RUN TestBoolStr_UnmarshalText/No + iso_test.go:530: got {isSet:true v:false} +=== RUN TestBoolStr_UnmarshalText/YES + iso_test.go:530: got {isSet:true v:true} +=== RUN TestBoolStr_UnmarshalText/other + iso_test.go:530: got {isSet:false v:false} +=== RUN TestBoolStr_UnmarshalText/#00 + iso_test.go:530: got {isSet:false v:false} +--- PASS: TestBoolStr_UnmarshalText (0.00s) + --- PASS: TestBoolStr_UnmarshalText/no (0.00s) + --- PASS: TestBoolStr_UnmarshalText/No (0.00s) + --- PASS: TestBoolStr_UnmarshalText/YES (0.00s) + --- PASS: TestBoolStr_UnmarshalText/other (0.00s) + --- PASS: TestBoolStr_UnmarshalText/#00 (0.00s) +=== RUN TestISORequest_validateOSDir +=== RUN TestISORequest_validateOSDir/VALID-OS-DIR + iso_test.go:617: validateOSDir() got true +=== RUN TestISORequest_validateOSDir/ANOTHER-VALID-OS-DIR + iso_test.go:617: validateOSDir() got true +=== RUN TestISORequest_validateOSDir/INVALID-OS-DIR + iso_test.go:617: validateOSDir() got false +--- PASS: TestISORequest_validateOSDir (0.00s) + --- PASS: TestISORequest_validateOSDir/VALID-OS-DIR (0.00s) + --- PASS: TestISORequest_validateOSDir/ANOTHER-VALID-OS-DIR (0.00s) + --- PASS: TestISORequest_validateOSDir/INVALID-OS-DIR (0.00s) +=== RUN TestStreamISOCmd_stdout + mkisofs_test.go:67: got: "mkisofs -joliet-long -input-charset utf-8 -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 -boot-info-table -R -J -v -T /tmp/nothing/here" +--- PASS: TestStreamISOCmd_stdout (1.36s) +=== RUN TestStreamISOCmd_stdout_err + mkisofs_test.go:96: got (expected) error: "exit status 1: mkisofs -joliet-long -input-charset utf-8 -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 -boot-info-table -R -J -v -T /tmp/nothing/here" +--- PASS: TestStreamISOCmd_stdout_err (0.87s) +=== RUN TestStreamISOCmd_file + mkisofs_test.go:125: isoDest = "/var/folders/r5/snb0k76d0n742rp0mtwzmx_80000gp/T/genISO2108450873/tmp.iso" + mkisofs_test.go:143: got: "/var/folders/r5/snb0k76d0n742rp0mtwzmx_80000gp/T/TestStreamISOCmd_file2029127300/generate /var/folders/r5/snb0k76d0n742rp0mtwzmx_80000gp/T/genISO2108450873/tmp.iso" + mkisofs_test.go:153: stat of "/var/folders/r5/snb0k76d0n742rp0mtwzmx_80000gp/T/genISO2108450873" (expected): stat /var/folders/r5/snb0k76d0n742rp0mtwzmx_80000gp/T/genISO2108450873: no such file or directory +--- PASS: TestStreamISOCmd_file (0.03s) +=== RUN TestStreamISOCmd_file_err + mkisofs_test.go:182: isoDest = "/var/folders/r5/snb0k76d0n742rp0mtwzmx_80000gp/T/genISO732973818/tmp.iso" + mkisofs_test.go:197: got (expected) error: "exit status 1: /var/folders/r5/snb0k76d0n742rp0mtwzmx_80000gp/T/TestStreamISOCmd_file_err3515397006/generate /var/folders/r5/snb0k76d0n742rp0mtwzmx_80000gp/T/genISO732973818/tmp.iso" +--- PASS: TestStreamISOCmd_file_err (0.02s) +=== RUN TestKickstarterDir +=== RUN TestKickstarterDir/default + mkisofs_test.go:267: kickstarterDir(tx, "templeOS") = "/var/www/files/templeOS" +=== RUN TestKickstarterDir/param-override + mkisofs_test.go:267: kickstarterDir(tx, "anotherOS/dir") = "/var/override/dir/anotherOS/dir" +--- PASS: TestKickstarterDir (0.00s) + --- PASS: TestKickstarterDir/default (0.00s) + --- PASS: TestKickstarterDir/param-override (0.00s) +=== RUN TestWriteKSCfgs +=== RUN TestWriteKSCfgs/empty + mkisofs_test.go:321: writeKSCfgs("/var/folders/r5/snb0k76d0n742rp0mtwzmx_80000gp/T/TestWriteKSCfgs4024128887", isoRequest): + mkisofs_test.go:348: state.out: + Dir== /var/folders/r5/snb0k76d0n742rp0mtwzmx_80000gp/T/TestWriteKSCfgs4024128887 + + mkisofs_test.go:348: network.cfg: + IPADDR="" + NETMASK="" + GATEWAY="" + DEVICE="" + MTU="0" + NAMESERVER="192.168.1.1" + HOSTNAME="" + NETWORKING_IPV6="yes" + IPV6ADDR="" + IPV6_DEFAULTGW="" + DHCP="no" + mkisofs_test.go:348: mgmt_network.cfg: + IPADDR="" + NETMASK="" + GATEWAY="" + DEVICE="" + mkisofs_test.go:348: password.cfg: + rootpw --iscrypted $1$62fSX9Hf$oqtWslyzKZzqQIemUMaij0 + mkisofs_test.go:348: disk.cfg: + boot_drives="" +=== RUN TestWriteKSCfgs/complete + mkisofs_test.go:321: writeKSCfgs("/var/folders/r5/snb0k76d0n742rp0mtwzmx_80000gp/T/TestWriteKSCfgs1059205786", isoRequest): + mkisofs_test.go:348: state.out: + Dir== /var/folders/r5/snb0k76d0n742rp0mtwzmx_80000gp/T/TestWriteKSCfgs1059205786 + /usr/local/bin mkisofs arg1 -opt1 -opt2 + mkisofs_test.go:348: network.cfg: + IPADDR="192.168.1.2" + NETMASK="255.255.255.0" + GATEWAY="192.168.1.255" + BOND_DEVICE="bond0" + MTU="1500" + NAMESERVER="192.168.1.1" + HOSTNAME="test.server.cdn.example" + NETWORKING_IPV6="yes" + IPV6ADDR="beef::1" + IPV6_DEFAULTGW="::1" + BONDING_OPTS="miimon=100 mode=4 lacp_rate=fast xmit_hash_policy=layer3+4" + DHCP="yes" + mkisofs_test.go:348: mgmt_network.cfg: + IPADDR="10.10.0.1" + NETMASK="10.10.255.255" + GATEWAY="10.10.0.255" + DEVICE="eth0" + mkisofs_test.go:348: password.cfg: + rootpw --iscrypted $1$PdibCMHc$.8C8L6Ig7jSgGeHebVP5y0 + mkisofs_test.go:348: disk.cfg: + boot_drives="sda1" +--- PASS: TestWriteKSCfgs (0.01s) + --- PASS: TestWriteKSCfgs/empty (0.00s) + --- PASS: TestWriteKSCfgs/complete (0.00s) +=== RUN TestWriteNetworkCfg +=== RUN TestWriteNetworkCfg/empty + mkisofs_test.go:489: writeNetworkCfg(): + IPADDR="" + NETMASK="" + GATEWAY="" + DEVICE="" + MTU="0" + NAMESERVER="" + HOSTNAME="" + NETWORKING_IPV6="yes" + IPV6ADDR="" + IPV6_DEFAULTGW="" + DHCP="no" +=== RUN TestWriteNetworkCfg/no_domain + mkisofs_test.go:489: writeNetworkCfg(): + IPADDR="192.168.1.2" + NETMASK="255.255.255.0" + GATEWAY="192.168.1.255" + DEVICE="eth0" + MTU="1500" + NAMESERVER="8.8.8.8,1.1.1.1" + HOSTNAME="test.server" + NETWORKING_IPV6="yes" + IPV6ADDR="beef::1" + IPV6_DEFAULTGW="::1" + DHCP="yes" +=== RUN TestWriteNetworkCfg/non-bonded + mkisofs_test.go:489: writeNetworkCfg(): + IPADDR="192.168.1.2" + NETMASK="255.255.255.0" + GATEWAY="192.168.1.255" + DEVICE="eth0" + MTU="1500" + NAMESERVER="8.8.8.8,1.1.1.1" + HOSTNAME="test.server.example.com" + NETWORKING_IPV6="yes" + IPV6ADDR="beef::1" + IPV6_DEFAULTGW="::1" + DHCP="yes" +=== RUN TestWriteNetworkCfg/bonded + mkisofs_test.go:489: writeNetworkCfg(): + IPADDR="192.168.1.2" + NETMASK="255.255.255.0" + GATEWAY="192.168.1.255" + BOND_DEVICE="bond01" + MTU="1500" + NAMESERVER="8.8.8.8,1.1.1.1" + HOSTNAME="test.server" + NETWORKING_IPV6="yes" + IPV6ADDR="beef::1" + IPV6_DEFAULTGW="::1" + BONDING_OPTS="miimon=100 mode=4 lacp_rate=fast xmit_hash_policy=layer3+4" + DHCP="yes" +--- PASS: TestWriteNetworkCfg (0.00s) + --- PASS: TestWriteNetworkCfg/empty (0.00s) + --- PASS: TestWriteNetworkCfg/no_domain (0.00s) + --- PASS: TestWriteNetworkCfg/non-bonded (0.00s) + --- PASS: TestWriteNetworkCfg/bonded (0.00s) +=== RUN TestWriteMgmtNetworkCfg +=== RUN TestWriteMgmtNetworkCfg/empty + mkisofs_test.go:560: writeMgmtNetworkCfg(): + IPADDR="" + NETMASK="" + GATEWAY="" + DEVICE="" +=== RUN TestWriteMgmtNetworkCfg/IPv4 + mkisofs_test.go:560: writeMgmtNetworkCfg(): + IPADDR="192.168.2.3" + NETMASK="255.255.255.255" + GATEWAY="192.168.1.255" + DEVICE="eth0" +=== RUN TestWriteMgmtNetworkCfg/IPv6 + mkisofs_test.go:560: writeMgmtNetworkCfg(): + IPV6ADDR="beef::1" + NETMASK="255.255.255.255" + GATEWAY="192.168.1.255" + DEVICE="eth0" +--- PASS: TestWriteMgmtNetworkCfg (0.00s) + --- PASS: TestWriteMgmtNetworkCfg/empty (0.00s) + --- PASS: TestWriteMgmtNetworkCfg/IPv4 (0.00s) + --- PASS: TestWriteMgmtNetworkCfg/IPv6 (0.00s) +=== RUN TestWriteDiskCfg +=== RUN TestWriteDiskCfg/empty + mkisofs_test.go:603: writeDiskCfg(): + boot_drives="" +=== RUN TestWriteDiskCfg/non-empty + mkisofs_test.go:603: writeDiskCfg(): + boot_drives="sda1" +--- PASS: TestWriteDiskCfg (0.00s) + --- PASS: TestWriteDiskCfg/empty (0.00s) + --- PASS: TestWriteDiskCfg/non-empty (0.00s) +=== RUN TestWritePasswordCfg +=== RUN TestWritePasswordCfg/empty + mkisofs_test.go:644: writePasswordCfg(): + "rootpw --iscrypted $1$salt$UsdFqFVB.FsuinRDK5eE..\n" +=== RUN TestWritePasswordCfg/non-empty + mkisofs_test.go:644: writePasswordCfg(): + "rootpw --iscrypted $1$salt$17HeaymOIi.65dl76MkK01\n" +--- PASS: TestWritePasswordCfg (0.00s) + --- PASS: TestWritePasswordCfg/empty (0.00s) + --- PASS: TestWritePasswordCfg/non-empty (0.00s) +=== RUN TestWritePasswordCfg_rndSalt +=== RUN TestWritePasswordCfg_rndSalt/empty + mkisofs_test.go:696: writePasswordCfg(): + "rootpw --iscrypted $1$09R4vGuA$0shIp9zZOCLhGHv9KEt3a1\n" +=== RUN TestWritePasswordCfg_rndSalt/non-empty + mkisofs_test.go:696: writePasswordCfg(): + "rootpw --iscrypted $1$ZZ4mQA5e$0/TxriFqQkychTbb0oZux/\n" +=== RUN TestWritePasswordCfg_rndSalt/long + mkisofs_test.go:696: writePasswordCfg(): + "rootpw --iscrypted $1$BjbBMl5B$qswW3lFyMdSmhyPfzVYCI.\n" +--- PASS: TestWritePasswordCfg_rndSalt (0.00s) + --- PASS: TestWritePasswordCfg_rndSalt/empty (0.00s) + --- PASS: TestWritePasswordCfg_rndSalt/non-empty (0.00s) + --- PASS: TestWritePasswordCfg_rndSalt/long (0.00s) +=== RUN TestGetOSVersions +=== RUN TestGetOSVersions/valid-file + osversions_test.go:94: getOSVersions(): tc.OSVersionsResponse{"TempleOS":"temple503"} +=== RUN TestGetOSVersions/invalid-file + osversions_test.go:137: getOSVersions() err (expected) = open /var/folders/r5/snb0k76d0n742rp0mtwzmx_80000gp/T/TestGetOSVersions1146670759/osversions.json: no such file or directory +--- PASS: TestGetOSVersions (0.00s) + --- PASS: TestGetOSVersions/valid-file (0.00s) + --- PASS: TestGetOSVersions/invalid-file (0.00s) +=== RUN TestOsversionsCfgPath +=== RUN TestOsversionsCfgPath/default + osversions_test.go:205: osversionCfgPath(): "/var/www/files/osversions.json" +=== RUN TestOsversionsCfgPath/override + osversions_test.go:205: osversionCfgPath(): "/this/is/not/the/default/osversions.json" +=== RUN TestOsversionsCfgPath/override-cwd + osversions_test.go:205: osversionCfgPath(): "osversions.json" +--- PASS: TestOsversionsCfgPath (0.00s) + --- PASS: TestOsversionsCfgPath/default (0.00s) + --- PASS: TestOsversionsCfgPath/override (0.00s) + --- PASS: TestOsversionsCfgPath/override-cwd (0.00s) +=== RUN TestParseResolve +=== RUN TestParseResolve/empty + resolve_test.go:92: parseResolve(): [] +=== RUN TestParseResolve/comments + resolve_test.go:92: parseResolve(): [] +=== RUN TestParseResolve/invalid_IPs + resolve_test.go:92: parseResolve(): [] +=== RUN TestParseResolve/single + resolve_test.go:92: parseResolve(): [127.0.0.1] +=== RUN TestParseResolve/multi + resolve_test.go:92: parseResolve(): [127.0.0.1 192.168.1.10 beef::1 ::0] +--- PASS: TestParseResolve (0.00s) + --- PASS: TestParseResolve/empty (0.00s) + --- PASS: TestParseResolve/comments (0.00s) + --- PASS: TestParseResolve/invalid_IPs (0.00s) + --- PASS: TestParseResolve/single (0.00s) + --- PASS: TestParseResolve/multi (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/iso (cached) +=== RUN TestLoginWithEmptyCredentials +--- PASS: TestLoginWithEmptyCredentials (0.00s) +=== RUN TestVerifyUrlOnWhiteList +--- PASS: TestVerifyUrlOnWhiteList (0.00s) +=== RUN TestTemplateRender + login_test.go:117: From: no-reply@test.quest + To: em@i.l + Content-Type: text/html + Subject: TO API Unit Tests Password Reset Request + + + + + TO API Unit Tests Password Reset Request + + + + +
+

Someone has requested to change your password for the TO API Unit Tests. If you requested this change, please click the link below and change your password. Otherwise, you can disregard this email.

+

Click to Reset Your Password

+
+
+

Thank you,
+ The TO API Unit Tests Team

+
+ + +--- PASS: TestTemplateRender (0.00s) +=== RUN TestLogout +--- PASS: TestLogout (0.00s) +=== RUN TestRegistrationTemplateRender + register_test.go:57: From: no-reply@test.quest + To: em@i.l + Content-Type: text/html + Subject: test New User Registration + + + + + test New User Registration + + + + +
+

A new account has been created for you on the test Portal. In the + test Portal, you'll find a dashboard that provides access to all of your + Delivery Services.

+

Click here to finish your registration

+
+
+

Thank you,
+ The test Team

+
+ + +--- PASS: TestRegistrationTemplateRender (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/login (cached) +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/logs [no test files] +=== RUN TestGetMonitoringServers +--- PASS: TestGetMonitoringServers (0.00s) +=== RUN TestGetProfileWithParams +--- PASS: TestGetProfileWithParams (0.00s) +=== RUN TestGetCachegroups +--- PASS: TestGetCachegroups (0.00s) +=== RUN TestGetProfiles +--- PASS: TestGetProfiles (0.00s) +=== RUN TestGetDeliveryServices +--- PASS: TestGetDeliveryServices (0.00s) +=== RUN TestGetConfig +--- PASS: TestGetConfig (0.00s) +=== RUN TestGetMonitoringJSON +--- PASS: TestGetMonitoringJSON (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/monitoring (cached) +=== RUN TestReadOrigins +--- PASS: TestReadOrigins (0.00s) +=== RUN TestFuncs +--- PASS: TestFuncs (0.00s) +=== RUN TestInterfaces +--- PASS: TestInterfaces (0.00s) +=== RUN TestValidate +--- PASS: TestValidate (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/origin (cached) +=== RUN TestGetParameters +--- PASS: TestGetParameters (0.00s) +=== RUN TestInterfaces +--- PASS: TestInterfaces (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/parameter (cached) +=== RUN TestGetPhysLocations +--- PASS: TestGetPhysLocations (0.00s) +=== RUN TestInterfaces +--- PASS: TestInterfaces (0.00s) +=== RUN TestValidate +--- PASS: TestValidate (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/physlocation (cached) +=== RUN TestPingHandler +--- PASS: TestPingHandler (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/ping (cached) +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/plugin [no test files] +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/plugins [no test files] +=== RUN TestCopyProfileInvalidExistingProfile +=== RUN TestCopyProfileInvalidExistingProfile/multiple_profiles_with_existing_name_returned +=== RUN TestCopyProfileInvalidExistingProfile/existing_profile_does_not_exist +--- PASS: TestCopyProfileInvalidExistingProfile (0.00s) + --- PASS: TestCopyProfileInvalidExistingProfile/multiple_profiles_with_existing_name_returned (0.00s) + --- PASS: TestCopyProfileInvalidExistingProfile/existing_profile_does_not_exist (0.00s) +=== RUN TestCopyNewProfileExists +--- PASS: TestCopyNewProfileExists (0.00s) +=== RUN TestCopyProfile +--- PASS: TestCopyProfile (0.00s) +=== RUN TestGetExportProfileResponse +=== RUN TestGetExportProfileResponse/Success:_Read_export_profile_successful + profile_export_test.go:74: Starting test scenario: Success: Read export profile successful +=== RUN TestGetExportProfileResponse/Success:_Read_export_profile_with_no_parameters_successful + profile_export_test.go:74: Starting test scenario: Success: Read export profile with no parameters successful +=== RUN TestGetExportProfileResponse/Failure:_Storage_error_reading_profile + profile_export_test.go:74: Starting test scenario: Failure: Storage error reading profile +--- PASS: TestGetExportProfileResponse (0.00s) + --- PASS: TestGetExportProfileResponse/Success:_Read_export_profile_successful (0.00s) + --- PASS: TestGetExportProfileResponse/Success:_Read_export_profile_with_no_parameters_successful (0.00s) + --- PASS: TestGetExportProfileResponse/Failure:_Storage_error_reading_profile (0.00s) +=== RUN TestGetImportProfile +=== RUN TestGetImportProfile/Success:_Import_profile_successful + profile_import_test.go:86: Starting test scenario: Success: Import profile successful +=== RUN TestGetImportProfile/Failure:_Import_profile_didn't_insert_row + profile_import_test.go:86: Starting test scenario: Failure: Import profile didn't insert row +=== RUN TestGetImportProfile/Failure:_Import_profile_storage_error + profile_import_test.go:86: Starting test scenario: Failure: Import profile storage error +--- PASS: TestGetImportProfile (0.00s) + --- PASS: TestGetImportProfile/Success:_Import_profile_successful (0.00s) + --- PASS: TestGetImportProfile/Failure:_Import_profile_didn't_insert_row (0.00s) + --- PASS: TestGetImportProfile/Failure:_Import_profile_storage_error (0.00s) +=== RUN TestGetImportProfileParameters +=== RUN TestGetImportProfileParameters/Success:_All_import_parameters_new + profile_import_test.go:292: Starting test scenario: Success: All import parameters new +=== RUN TestGetImportProfileParameters/Success:_All_parameters_exisiting + profile_import_test.go:292: Starting test scenario: Success: All parameters exisiting +=== RUN TestGetImportProfileParameters/Success:_Mix_of_existing/new_parameters + profile_import_test.go:292: Starting test scenario: Success: Mix of existing/new parameters +=== RUN TestGetImportProfileParameters/Success:_Dup_of_existing + profile_import_test.go:292: Starting test scenario: Success: Dup of existing +=== RUN TestGetImportProfileParameters/Fail:_Storage_error_selecting_param + profile_import_test.go:292: Starting test scenario: Fail: Storage error selecting param +=== RUN TestGetImportProfileParameters/Fail:_Storage_error_inserting_param + profile_import_test.go:292: Starting test scenario: Fail: Storage error inserting param +--- PASS: TestGetImportProfileParameters (0.00s) + --- PASS: TestGetImportProfileParameters/Success:_All_import_parameters_new (0.00s) + --- PASS: TestGetImportProfileParameters/Success:_All_parameters_exisiting (0.00s) + --- PASS: TestGetImportProfileParameters/Success:_Mix_of_existing/new_parameters (0.00s) + --- PASS: TestGetImportProfileParameters/Success:_Dup_of_existing (0.00s) + --- PASS: TestGetImportProfileParameters/Fail:_Storage_error_selecting_param (0.00s) + --- PASS: TestGetImportProfileParameters/Fail:_Storage_error_inserting_param (0.00s) +=== RUN TestGetProfiles +--- PASS: TestGetProfiles (0.00s) +=== RUN TestInterfaces +--- PASS: TestInterfaces (0.00s) +=== RUN TestValidate +--- PASS: TestValidate (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/profile (cached) +=== RUN TestGetProfileParameters +--- PASS: TestGetProfileParameters (0.00s) +=== RUN TestInterfaces +--- PASS: TestInterfaces (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/profileparameter (cached) +=== RUN TestReadRegions +--- PASS: TestReadRegions (0.00s) +=== RUN TestInterfaces +--- PASS: TestInterfaces (0.00s) +=== RUN TestValidation + regions_test.go:137: Got expected error validating region with no division: region 'division' is required +--- PASS: TestValidation (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/region (cached) +=== RUN TestFuncs +--- PASS: TestFuncs (0.00s) +=== RUN TestInterfaces +--- PASS: TestInterfaces (0.00s) +=== RUN TestValidate +--- PASS: TestValidate (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/role (cached) +=== RUN TestCompileRoutes + routing_test.go:160: testing path GET api/2.0/cdns + routing_test.go:160: testing path POST api/1.4/users/login + routing_test.go:160: testing path POST api/3.0/cdns + routing_test.go:160: testing path POST api/3.0/users + routing_test.go:160: testing path PUT api/3.0/deliveryservices/3 + routing_test.go:160: testing path DELETE api/3.0/servers/777 + routing_test.go:160: testing path GET api/3.0/cdns/1 + routing_test.go:160: testing path GET /api/1.1/about + routing_test.go:160: testing path GET api/3.0/notatypeweknowabout + routing_test.go:160: testing path GET api/99999.99999/cdns + routing_test.go:160: testing path GET blahblah/api/3.0/cdns + routing_test.go:160: testing path GET internal/api/2.0/federations.json + routing_test.go:160: testing path GET api/3.0/servers + routing_test.go:160: testing path GET api/2.0/servers +--- PASS: TestCompileRoutes (0.01s) +=== RUN TestRoutes +--- PASS: TestRoutes (0.00s) +=== RUN TestCreateRouteMap +--- PASS: TestCreateRouteMap (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/routing (cached) +=== RUN TestWrapHeaders +--- PASS: TestWrapHeaders (0.00s) +=== RUN TestWrapPanicRecover +--- PASS: TestWrapPanicRecover (0.00s) +=== RUN TestGzip +--- PASS: TestGzip (0.00s) +=== RUN TestWrapAuth +--- PASS: TestWrapAuth (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/routing/middleware (cached) +=== RUN TestGetDetailServers +--- PASS: TestGetDetailServers (0.00s) +=== RUN TestAssignDsesToServer +--- PASS: TestAssignDsesToServer (0.00s) +=== RUN TestInterfaces +--- PASS: TestInterfaces (0.00s) +=== RUN TestUpdateServer +--- PASS: TestUpdateServer (0.00s) +=== RUN TestGetServersByCachegroup +--- PASS: TestGetServersByCachegroup (0.00s) +=== RUN TestGetMidServers +--- PASS: TestGetMidServers (0.00s) +=== RUN TestV3Validations + servers_test.go:578: Got expected error validating server with no interfaces: a server must have at least one interface + servers_test.go:592: Got expected error validating server with nil interfaces: a server must have at least one interface + servers_test.go:609: Got expected error validating server with an MTU < 1280: 'interface 'eth0' mtu' must be at least 1280 + servers_test.go:625: Got expected error validating server with no IP addresses: 'interface 'eth0' ipAddresses' cannot be blank, a server must have at least one service address + servers_test.go:640: Got expected error validating server with nil IP addresses: 'interface 'eth0' ipAddresses' cannot be blank, a server must have at least one service address + servers_test.go:661: Got expected error validating server with no service addresses: a server must have at least one service address + servers_test.go:675: Got expected error validating server with too many interfaces with service addresses: interfaces: address '127.0.0.1/32' of interface 'eth0' is marked as a service address, but an IPv4 service address appears earlier in the list + servers_test.go:695: Got expected error validating server with no service addresses: interfaces: address '1.2.3.4/1' of interface 'eth0' is marked as a service address, but an IPv4 service address appears earlier in the list +--- PASS: TestV3Validations (0.00s) +=== RUN TestGetServerUpdateStatus +--- PASS: TestGetServerUpdateStatus (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/server (cached) +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/servercapability [no test files] +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/servercheck [no test files] +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/servercheck/extensions [no test files] +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/servicecategory [no test files] +=== RUN TestFuncs +--- PASS: TestFuncs (0.00s) +=== RUN TestInterfaces +--- PASS: TestInterfaces (0.00s) +=== RUN TestValidate +--- PASS: TestValidate (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/staticdnsentry (cached) +=== RUN TestReadStatuses +--- PASS: TestReadStatuses (0.00s) +=== RUN TestInterfaces +--- PASS: TestInterfaces (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/status (cached) +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/steering [no test files] +=== RUN TestInvalidSteeringTargetType +--- PASS: TestInvalidSteeringTargetType (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/steeringtargets (cached) +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/swaggerdocs/v13 [no test files] +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/swaggerdocs/v13/swaggerspec-server [no test files] +=== RUN TestGetSystemInfo +--- PASS: TestGetSystemInfo (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/systeminfo (cached) +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/tenant [no test files] +=== RUN TestErrorCode +--- PASS: TestErrorCode (0.00s) +=== RUN TestErrorContextPanicMode +--- PASS: TestErrorContextPanicMode (0.00s) +=== RUN TestErrorContextMapping +--- PASS: TestErrorContextMapping (0.00s) +=== RUN TestErrorContextMisc +--- PASS: TestErrorContextMisc (0.00s) +=== RUN TestErrorContext +--- PASS: TestErrorContext (0.00s) +=== RUN TestColsFromStructByTag +--- PASS: TestColsFromStructByTag (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/test (cached) +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/tocookie [no test files] +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/topology [no test files] +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/topology/topology_validation [no test files] +=== RUN TestTSConfigFromRequest +--- PASS: TestTSConfigFromRequest (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/trafficstats (cached) +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/trafficvault [no test files] +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/trafficvault/backends [no test files] +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/trafficvault/backends/disabled [no test files] +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/trafficvault/backends/postgres [no test files] +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/trafficvault/backends/postgres/hashicorpvault [no test files] +=== RUN TestFetchObjectValues +--- PASS: TestFetchObjectValues (0.00s) +=== RUN TestSaveObject +--- PASS: TestSaveObject (0.00s) +=== RUN TestDeleteObject +--- PASS: TestDeleteObject (0.00s) +=== RUN TestSetTLSVersion +--- PASS: TestSetTLSVersion (0.00s) +=== RUN TestGetRiakCluster +--- PASS: TestGetRiakCluster (0.10s) +=== RUN TestUnmarshalGoodRiakConfig +--- PASS: TestUnmarshalGoodRiakConfig (0.00s) +=== RUN TestUnmarshalBadRiakConfig +--- PASS: TestUnmarshalBadRiakConfig (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/trafficvault/backends/riaksvc (cached) +=== RUN TestGetType +--- PASS: TestGetType (0.00s) +=== RUN TestInterfaces +--- PASS: TestInterfaces (0.00s) +=== RUN TestUpdateInvalidType +--- PASS: TestUpdateInvalidType (0.00s) +=== RUN TestDeleteInvalidType +--- PASS: TestDeleteInvalidType (0.00s) +=== RUN TestCreateInvalidType +--- PASS: TestCreateInvalidType (0.00s) +=== RUN TestValidate +--- PASS: TestValidate (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/types (cached) +=== RUN TestValidateURIKeyset +--- PASS: TestValidateURIKeyset (0.00s) +PASS +ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/urisigning (cached) +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/user [no test files] +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/util/ims [no test files] +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/util/monitorhlp [no test files] +? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/vault [no test files] +? github.com/apache/trafficcontrol/traffic_ops/v2-client [no test files] +? github.com/apache/trafficcontrol/traffic_ops/v3-client [no test files] +? github.com/apache/trafficcontrol/traffic_ops/v4-client [no test files] +FAIL From 1bdb1f73106f0f07ddfffed3f1b39fcd1c4e95af Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Wed, 25 Aug 2021 13:31:44 -0600 Subject: [PATCH 17/35] Fix config file tests in t3c generate --- .../t3c-generate/cfgfile/cfgfile_test.go | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/cache-config/t3c-generate/cfgfile/cfgfile_test.go b/cache-config/t3c-generate/cfgfile/cfgfile_test.go index 991501ac70..e82e2d09a2 100644 --- a/cache-config/t3c-generate/cfgfile/cfgfile_test.go +++ b/cache-config/t3c-generate/cfgfile/cfgfile_test.go @@ -162,6 +162,11 @@ func randUint64() *uint64 { return &i } +func randUint() *uint { + i := uint(rand.Int()) + return &i +} + func randFloat64() *float64 { f := rand.Float64() return &f @@ -351,16 +356,16 @@ func randParam() *tc.Parameter { } } -func randJob() *tc.InvalidationJob { - now := &tc.Time{Time: time.Now(), Valid: true} - return &tc.InvalidationJob{ - Parameters: randStr(), - Keyword: randStr(), - AssetURL: randStr(), - CreatedBy: randStr(), - StartTime: now, - ID: randUint64(), - DeliveryService: randStr(), +func randJob() *tc.InvalidationJobV40 { + now := time.Now() + return &tc.InvalidationJobV40{ + AssetURL: randStr(), + CreatedBy: randStr(), + StartTime: &now, + ID: randUint64(), + DeliveryService: randStr(), + TTLHours: randUint(), + InvalidationType: util.StrPtr(tc.REFRESH), } } @@ -486,7 +491,7 @@ func MakeFakeTOData() *t3cutil.ConfigData { }, DeliveryServiceServers: dss, Server: sv0, - Jobs: []tc.InvalidationJob{ + Jobs: []tc.InvalidationJobV40{ *randJob(), *randJob(), }, From 5493dd9c42b934eb340409934ef9caac1ad8e3ab Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Fri, 27 Aug 2021 08:51:42 -0600 Subject: [PATCH 18/35] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae687dc8b8..9f62ce8b21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -170,6 +170,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - CDN in a Box now uses Apache Traffic Server 8.1. - Customer names in payloads sent to the `/deliveryservices/request` Traffic Ops API endpoint can no longer contain characters besides alphanumerics, @, !, #, $, %, ^, &, *, (, ), [, ], '.', ' ', and '-'. This fixes a vulnerability that allowed email content injection. - Go version 1.17 is used to compile Traffic Ops, T3C, Traffic Monitor, Traffic Stats, and Grove. +- Changed Invalidation Jobs in Traffic Ops to account for the ability to do both REFRESH and REFETCH requests for resources. ### Deprecated - The Riak Traffic Vault backend is now deprecated and its support may be removed in a future release. It is highly recommended to use the new PostgreSQL backend instead. From ebd6ee2c1cbcb8854ed6b2b67f3189beec3e3962 Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Fri, 27 Aug 2021 10:19:56 -0600 Subject: [PATCH 19/35] Fix go.mod go.sum from poor rebasing --- go.sum | 12 ------------ vendor/github.com/jmoiron/sqlx/go.mod | 7 ------- vendor/github.com/jmoiron/sqlx/go.sum | 6 ------ vendor/github.com/lib/pq/go.mod | 3 --- vendor/modules.txt | 8 -------- 5 files changed, 36 deletions(-) delete mode 100644 vendor/github.com/jmoiron/sqlx/go.mod delete mode 100644 vendor/github.com/jmoiron/sqlx/go.sum delete mode 100644 vendor/github.com/lib/pq/go.mod diff --git a/go.sum b/go.sum index 8dd8dacbb0..630f3735db 100644 --- a/go.sum +++ b/go.sum @@ -205,10 +205,6 @@ github.com/hydrogen18/stoppableListener v0.0.0-20151210151943-dadc9ccc400c/go.mo github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/influxdata/influxdb v1.1.1-0.20170104212736-6a94d200c826 h1:F/ZGpGV51zWV9cpTGdCikuptiAwG1NK747a80m0yK+s= github.com/influxdata/influxdb v1.1.1-0.20170104212736-6a94d200c826/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> 7d0002a6e... Add go sum file to commit github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= @@ -234,11 +230,6 @@ github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQ github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -<<<<<<< HEAD -======= ->>>>>>> 7ab736e82... Change mod file to match master, run go vendor -======= ->>>>>>> 7d0002a6e... Add go sum file to commit github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/json-iterator/go v1.1.6-0.20181024152841-05d041de1043 h1:XWlFVbIXnWhOnWnMMUyVPElTFYoLeuO0TAarw7Jvdco= @@ -281,9 +272,6 @@ github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= -github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.5 h1:1IdxlwTNazvbKJQSxoJ5/9ECbEeaTTyeU7sEAZ5KKTQ= github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI= github.com/miekg/dns v1.0.6-0.20180406150955-01d59357d468 h1:ABwS8bIszuZkM3o4nP3Hw3rz9TGkMW9lV4Ifp1hMEaI= diff --git a/vendor/github.com/jmoiron/sqlx/go.mod b/vendor/github.com/jmoiron/sqlx/go.mod deleted file mode 100644 index 66c67561cc..0000000000 --- a/vendor/github.com/jmoiron/sqlx/go.mod +++ /dev/null @@ -1,7 +0,0 @@ -module github.com/jmoiron/sqlx - -require ( - github.com/go-sql-driver/mysql v1.4.0 - github.com/lib/pq v1.0.0 - github.com/mattn/go-sqlite3 v1.9.0 -) diff --git a/vendor/github.com/jmoiron/sqlx/go.sum b/vendor/github.com/jmoiron/sqlx/go.sum deleted file mode 100644 index a3239ada75..0000000000 --- a/vendor/github.com/jmoiron/sqlx/go.sum +++ /dev/null @@ -1,6 +0,0 @@ -github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= -github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4= -github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= diff --git a/vendor/github.com/lib/pq/go.mod b/vendor/github.com/lib/pq/go.mod deleted file mode 100644 index b5a5639ab6..0000000000 --- a/vendor/github.com/lib/pq/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module github.com/lib/pq - -go 1.13 diff --git a/vendor/modules.txt b/vendor/modules.txt index d80f361857..8361571ac6 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -130,20 +130,12 @@ github.com/lestrrat/go-jwx/internal/emap github.com/lestrrat/go-jwx/jwa github.com/lestrrat/go-jwx/jwk # github.com/lib/pq v1.8.0 -<<<<<<< HEAD ## explicit; go 1.13 -======= -## explicit ->>>>>>> 7ab736e82... Change mod file to match master, run go vendor github.com/lib/pq github.com/lib/pq/oid github.com/lib/pq/scram # github.com/mattn/go-sqlite3 v1.14.5 -<<<<<<< HEAD ## explicit; go 1.10 -======= -## explicit ->>>>>>> 7ab736e82... Change mod file to match master, run go vendor # github.com/miekg/dns v1.0.6-0.20180406150955-01d59357d468 ## explicit github.com/miekg/dns From 3a49c803b905d31180a1bed4d4f85b05da6bb8d5 Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Fri, 27 Aug 2021 10:27:01 -0600 Subject: [PATCH 20/35] More accurate changelog message --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f62ce8b21..0c5dbace08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -170,7 +170,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - CDN in a Box now uses Apache Traffic Server 8.1. - Customer names in payloads sent to the `/deliveryservices/request` Traffic Ops API endpoint can no longer contain characters besides alphanumerics, @, !, #, $, %, ^, &, *, (, ), [, ], '.', ' ', and '-'. This fixes a vulnerability that allowed email content injection. - Go version 1.17 is used to compile Traffic Ops, T3C, Traffic Monitor, Traffic Stats, and Grove. -- Changed Invalidation Jobs in Traffic Ops to account for the ability to do both REFRESH and REFETCH requests for resources. +- Changed Invalidation Jobs throughout (TO, TP, T3C, etc.) to account for the ability to do both REFRESH and REFETCH requests for resources. ### Deprecated - The Riak Traffic Vault backend is now deprecated and its support may be removed in a future release. It is highly recommended to use the new PostgreSQL backend instead. From c82cb1a617bfa2c6ab1c70d177aa76b2cf138936 Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Fri, 27 Aug 2021 13:20:17 -0600 Subject: [PATCH 21/35] Simplify DB migration up script UPDATE clause --- .../app/db/migrations/2021081321025500_refetch.up.sql | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/traffic_ops/app/db/migrations/2021081321025500_refetch.up.sql b/traffic_ops/app/db/migrations/2021081321025500_refetch.up.sql index c25c796d64..bab872e31b 100644 --- a/traffic_ops/app/db/migrations/2021081321025500_refetch.up.sql +++ b/traffic_ops/app/db/migrations/2021081321025500_refetch.up.sql @@ -49,13 +49,9 @@ ADD COLUMN IF NOT EXISTS invalidation_type text NOT NULL DEFAULT 'REFRESH'; * (adding ##REFETCH## to the end of the url) then assign the * correct invalidation_type. */ -UPDATE public.job +UPDATE public.job SET invalidation_type = 'REFETCH' -WHERE id IN ( - SELECT id - FROM public.job - WHERE asset_url LIKE '%##REFETCH##' -); +WHERE asset_url LIKE '%##REFETCH##%'; ALTER TABLE public.job RENAME COLUMN parameters TO ttl_hr; From 9218741aef4cc4bd66a5474f7a3b17dec6c98572 Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Fri, 27 Aug 2021 13:21:15 -0600 Subject: [PATCH 22/35] Delete accidentally committed test file --- traffic_ops/results.txt | 1204 --------------------------------------- 1 file changed, 1204 deletions(-) delete mode 100644 traffic_ops/results.txt diff --git a/traffic_ops/results.txt b/traffic_ops/results.txt deleted file mode 100644 index 7f0f05fd42..0000000000 --- a/traffic_ops/results.txt +++ /dev/null @@ -1,1204 +0,0 @@ -? github.com/apache/trafficcontrol/traffic_ops/app/bin/checks/DnssecRefresh [no test files] -? github.com/apache/trafficcontrol/traffic_ops/app/bin/checks/DnssecRefresh/config [no test files] -? github.com/apache/trafficcontrol/traffic_ops/app/db [no test files] -? github.com/apache/trafficcontrol/traffic_ops/app/db/reencrypt [no test files] -=== RUN TestRiakBackend ---- PASS: TestRiakBackend (0.00s) -=== RUN TestPGBackend ---- PASS: TestPGBackend (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/app/db/traffic_vault_migrate (cached) -=== RUN TestValidateParameter -[ERROR] Parameter trafficserver does not match value -[ERROR] Actual Value: 6.2.2 Expected Value: 622 ---- PASS: TestValidateParameter (0.00s) -=== RUN TestDeleteParameter -Deleting parameter {"proxy.cluster.a", "records.config", "INT 1"} -Deleting parameter {"proxy.cluster.b", "records.config", "STRING abc"} ---- PASS: TestDeleteParameter (0.00s) -=== RUN TestModifyConfigFile -Updating parameter {"LogFormat.Format", "logs_xml.config", " >"} to {"LogFormat.Format", "logging.config", " >"} -Updating parameter {"LogFormat.Name", "logs_xml.config", "custom_ats"} to {"LogFormat.Name", "logging.config", "custom_ats"} ---- PASS: TestModifyConfigFile (0.00s) -=== RUN TestModifyValueForce -Updating parameter {"proxy.config.hostdb.timeout", "records.config", "INT 1440"} to {"proxy.config.hostdb.timeout", "records.config", "INT 86400"} ---- PASS: TestModifyValueForce (0.00s) -=== RUN TestModifyValueSkip -[ACTION REQUIRED] Found modified value. Skip modifying {"proxy.config.hostdb.timeout", "records.config", "INT 5000"}. Please update manually ---- PASS: TestModifyValueSkip (0.00s) -=== RUN TestModifyNameValue -Updating parameter {"proxy.config.log.xml_config_file", "records.config", "logs_xml.config"} to {"proxy.config.log.config.filename", "records.config", "logging.config"} ---- PASS: TestModifyNameValue (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/install/bin/convert_profile (cached) -? github.com/apache/trafficcontrol/traffic_ops/testing/api/config [no test files] -? github.com/apache/trafficcontrol/traffic_ops/testing/api/utils [no test files] -Error Loading Config: failed to read CDN configuration: open traffic-ops-test.conf: no such file or directory -FAIL github.com/apache/trafficcontrol/traffic_ops/testing/api/v2 0.203s -Error Loading Config: failed to read CDN configuration: open traffic-ops-test.conf: no such file or directory -FAIL github.com/apache/trafficcontrol/traffic_ops/testing/api/v3 0.289s -INFO: traffic_ops_test.go:71: 2021-08-19T21:50:36.492742Z: Using Config values: - TO Config File: traffic-ops-test.conf - TO Fixtures: tc-fixtures.json - TO URL: https://localhost:8443 - TO Session Timeout In Secs: 60 - DB Server: localhost - DB User: traffic_ops - DB Name: to_test - DB Ssl: false - UseIMS: true - -Error creating session to https://localhost:8443 - traffic_ops, logging in: Login error Post "https://localhost:8443/api/4.0/user/login": dial tcp [::1]:8443: connect: connection refused, alerts string: {Alerts:[]} -FAIL github.com/apache/trafficcontrol/traffic_ops/testing/api/v4 0.431s -? github.com/apache/trafficcontrol/traffic_ops/toclientlib [no test files] -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang [no test files] -=== RUN TestSplitRPMVersion ---- PASS: TestSplitRPMVersion (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/about (cached) -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/acme [no test files] -=== RUN TestCamelCase ---- PASS: TestCamelCase (0.00s) -=== RUN TestRespWrittenAfterErrFails ---- PASS: TestRespWrittenAfterErrFails (0.00s) -=== RUN TestWriteAlertsObjEmpty ---- PASS: TestWriteAlertsObjEmpty (0.00s) -=== RUN TestWriteAlertsObj ---- PASS: TestWriteAlertsObj (0.00s) -=== RUN TestWriteResp ---- PASS: TestWriteResp (0.00s) -=== RUN TestWriteRespRaw ---- PASS: TestWriteRespRaw (0.00s) -=== RUN TestWriteRespVals ---- PASS: TestWriteRespVals (0.00s) -=== RUN TestRespWriter ---- PASS: TestRespWriter (0.00s) -=== RUN TestRespWriterVals ---- PASS: TestRespWriterVals (0.00s) -=== RUN TestWriteRespAlert ---- PASS: TestWriteRespAlert (0.00s) -=== RUN TestWriteRespAlertObj ---- PASS: TestWriteRespAlertObj (0.00s) -=== RUN TestParseRestrictFKConstraint -=== RUN TestParseRestrictFKConstraint/FK_Constraint_Error - api_test.go:245: Starting test scenario: FK Constraint Error -=== RUN TestParseRestrictFKConstraint/FK_Constraint_Error_with_underscores_in_table_name - api_test.go:245: Starting test scenario: FK Constraint Error with underscores in table name -=== RUN TestParseRestrictFKConstraint/Non_FK_Constraint_Error - api_test.go:245: Starting test scenario: Non FK Constraint Error ---- PASS: TestParseRestrictFKConstraint (0.00s) - --- PASS: TestParseRestrictFKConstraint/FK_Constraint_Error (0.00s) - --- PASS: TestParseRestrictFKConstraint/FK_Constraint_Error_with_underscores_in_table_name (0.00s) - --- PASS: TestParseRestrictFKConstraint/Non_FK_Constraint_Error (0.00s) -=== RUN TestInsertAsyncStatus ---- PASS: TestInsertAsyncStatus (0.00s) -=== RUN TestUpdateAsyncStatus ---- PASS: TestUpdateAsyncStatus (0.00s) -=== RUN TestUpdateAsyncStatusFinished ---- PASS: TestUpdateAsyncStatusFinished (0.00s) -=== RUN TestCreateChangeLog ---- PASS: TestCreateChangeLog (0.00s) -=== RUN TestCreateHandler ---- PASS: TestCreateHandler (0.00s) -=== RUN TestReadHandler ---- PASS: TestReadHandler (0.00s) -=== RUN TestReadHandlerIMS ---- PASS: TestReadHandlerIMS (0.00s) -=== RUN TestUpdateHandler ---- PASS: TestUpdateHandler (0.00s) -=== RUN TestDeleteHandler ---- PASS: TestDeleteHandler (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/api (cached) -=== RUN TestGetAPICapabilities ---- PASS: TestGetAPICapabilities (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/apicapability (cached) -=== RUN TestInterfaces ---- PASS: TestInterfaces (0.00s) -=== RUN TestIsUpdateable ---- PASS: TestIsUpdateable (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/apitenant (cached) -=== RUN TestGetASNs ---- PASS: TestGetASNs (0.00s) -=== RUN TestInterfaces ---- PASS: TestInterfaces (0.00s) -=== RUN TestValidate ---- PASS: TestValidate (0.00s) -=== RUN TestCheckNumberForUpdate ---- PASS: TestCheckNumberForUpdate (0.00s) -=== RUN TestASNExistsForUpdateFailure ---- PASS: TestASNExistsForUpdateFailure (0.00s) -=== RUN TestASNExistsForUpdateSuccess ---- PASS: TestASNExistsForUpdateSuccess (0.00s) -=== RUN TestASNExists ---- PASS: TestASNExists (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/asn (cached) -=== RUN TestDeriveGoodPassword ---- PASS: TestDeriveGoodPassword (0.08s) -=== RUN TestDeriveBadPassword ---- PASS: TestDeriveBadPassword (0.08s) -=== RUN TestScryptPasswordIsRequired ---- PASS: TestScryptPasswordIsRequired (0.00s) -=== RUN TestUsernamePassword ---- PASS: TestUsernamePassword (0.01s) -=== RUN TestCommonPassword ---- PASS: TestCommonPassword (0.01s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/auth (cached) -=== RUN TestReadCacheGroups ---- PASS: TestReadCacheGroups (0.00s) -=== RUN TestFuncs ---- PASS: TestFuncs (0.00s) -=== RUN TestInterfaces ---- PASS: TestInterfaces (0.00s) -=== RUN TestValidate ---- PASS: TestValidate (0.00s) -=== RUN TestBadTypeParamCacheGroups ---- PASS: TestBadTypeParamCacheGroups (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/cachegroup (cached) -=== RUN TestReadCacheGroupParameters -=== RUN TestReadCacheGroupParameters/Success:_Read_Cache_Group_Parameters_-_Parameters - parameters_test.go:160: Starting test scenario: Success: Read Cache Group Parameters - Parameters -=== RUN TestReadCacheGroupParameters/Success:_Read_Cache_Group_Parameters_with_parameter_id_-_Parameters - parameters_test.go:160: Starting test scenario: Success: Read Cache Group Parameters with parameter id - Parameters -=== RUN TestReadCacheGroupParameters/Success:_Read_Cache_Group_Parameters_no_data_-_Parameters - parameters_test.go:160: Starting test scenario: Success: Read Cache Group Parameters no data - Parameters -=== RUN TestReadCacheGroupParameters/Failure:_Storage_Error_reading_Cache_Group_Parameters_-_Parameters - parameters_test.go:160: Starting test scenario: Failure: Storage Error reading Cache Group Parameters - Parameters -=== RUN TestReadCacheGroupParameters/Failure:_User_Error_invalid_params_-_Parameters - parameters_test.go:160: Starting test scenario: Failure: User Error invalid params - Parameters -=== RUN TestReadCacheGroupParameters/Failure:_System_Error_getting_cache_group_-_Parameters - parameters_test.go:160: Starting test scenario: Failure: System Error getting cache group - Parameters -=== RUN TestReadCacheGroupParameters/Failure:_Cache_group_does_not_exist_-_Parameters - parameters_test.go:160: Starting test scenario: Failure: Cache group does not exist - Parameters ---- PASS: TestReadCacheGroupParameters (0.00s) - --- PASS: TestReadCacheGroupParameters/Success:_Read_Cache_Group_Parameters_-_Parameters (0.00s) - --- PASS: TestReadCacheGroupParameters/Success:_Read_Cache_Group_Parameters_with_parameter_id_-_Parameters (0.00s) - --- PASS: TestReadCacheGroupParameters/Success:_Read_Cache_Group_Parameters_no_data_-_Parameters (0.00s) - --- PASS: TestReadCacheGroupParameters/Failure:_Storage_Error_reading_Cache_Group_Parameters_-_Parameters (0.00s) - --- PASS: TestReadCacheGroupParameters/Failure:_User_Error_invalid_params_-_Parameters (0.00s) - --- PASS: TestReadCacheGroupParameters/Failure:_System_Error_getting_cache_group_-_Parameters (0.00s) - --- PASS: TestReadCacheGroupParameters/Failure:_Cache_group_does_not_exist_-_Parameters (0.00s) -=== RUN TestInterfaces ---- PASS: TestInterfaces (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/cachegroupparameter (cached) -=== RUN TestAddStatsInvalidString ---- PASS: TestAddStatsInvalidString (0.00s) -=== RUN TestAddStatsValidString ---- PASS: TestAddStatsValidString (0.00s) -=== RUN TestAddStatsEmptyJsonObject ---- PASS: TestAddStatsEmptyJsonObject (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/cachesstats (cached) -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/capabilities [no test files] -=== RUN TestGetStatsFromServiceInterface ---- PASS: TestGetStatsFromServiceInterface (0.00s) -=== RUN TestReadCDNs ---- PASS: TestReadCDNs (0.00s) -=== RUN TestFuncs ---- PASS: TestFuncs (0.00s) -=== RUN TestInterfaces ---- PASS: TestInterfaces (0.00s) -=== RUN TestValidate ---- PASS: TestValidate (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/cdn (cached) -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/cdn_lock [no test files] -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/cdnfederation [no test files] -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/cdnnotification [no test files] -=== RUN TestLogLocation ---- PASS: TestLogLocation (0.00s) -=== RUN TestLoadConfig ---- PASS: TestLoadConfig (0.00s) -=== RUN TestValidateRoutingBlacklist ---- PASS: TestValidateRoutingBlacklist (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/config (cached) -=== RUN TestReadCoordinates ---- PASS: TestReadCoordinates (0.00s) -=== RUN TestFuncs ---- PASS: TestFuncs (0.00s) -=== RUN TestInterfaces ---- PASS: TestInterfaces (0.00s) -=== RUN TestValidate ---- PASS: TestValidate (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/coordinate (cached) -=== RUN TestGetConfigParams ---- PASS: TestGetConfigParams (0.00s) -=== RUN TestMakeCRConfigConfig ---- PASS: TestMakeCRConfigConfig (0.00s) -=== RUN TestGetTMURLHost ---- PASS: TestGetTMURLHost (0.00s) -=== RUN TestMakeDSes ---- PASS: TestMakeDSes (0.01s) -=== RUN TestGetServerProfileParams ---- PASS: TestGetServerProfileParams (0.00s) -=== RUN TestGetDSRegexesDomains ---- PASS: TestGetDSRegexesDomains (0.00s) -=== RUN TestGetStaticDNSEntries ---- PASS: TestGetStaticDNSEntries (0.00s) -=== RUN TestMakeLocations ---- PASS: TestMakeLocations (0.00s) -=== RUN TestGetServerParams ---- PASS: TestGetServerParams (0.00s) -=== RUN TestGetAllServers ---- PASS: TestGetAllServers (0.00s) -=== RUN TestGetAllServersNonService ---- PASS: TestGetAllServersNonService (0.00s) -=== RUN TestGetServerDSNames ---- PASS: TestGetServerDSNames (0.00s) -=== RUN TestGetServerDSes ---- PASS: TestGetServerDSes (0.00s) -=== RUN TestGetCDNInfo ---- PASS: TestGetCDNInfo (0.00s) -=== RUN TestGetCDNNameFromID ---- PASS: TestGetCDNNameFromID (0.00s) -=== RUN TestGetSnapshot ---- PASS: TestGetSnapshot (0.00s) -=== RUN TestSnapshot ---- PASS: TestSnapshot (0.00s) -=== RUN TestMakeStats ---- PASS: TestMakeStats (0.00s) -=== RUN TestMakeTops ---- PASS: TestMakeTops (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/crconfig (cached) -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/crstats [no test files] -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/dbdump [no test files] -=== RUN TestBuildQuery ---- PASS: TestBuildQuery (0.00s) -=== RUN TestGetCacheGroupByName -=== RUN TestGetCacheGroupByName/Success:_Cache_Group_exists - db_helpers_test.go:116: Starting test scenario: Success: Cache Group exists -=== RUN TestGetCacheGroupByName/Failure:_Cache_Group_does_not_exist - db_helpers_test.go:116: Starting test scenario: Failure: Cache Group does not exist -=== RUN TestGetCacheGroupByName/Failure:_Storage_error_getting_Cache_Group - db_helpers_test.go:116: Starting test scenario: Failure: Storage error getting Cache Group ---- PASS: TestGetCacheGroupByName (0.00s) - --- PASS: TestGetCacheGroupByName/Success:_Cache_Group_exists (0.00s) - --- PASS: TestGetCacheGroupByName/Failure:_Cache_Group_does_not_exist (0.00s) - --- PASS: TestGetCacheGroupByName/Failure:_Storage_error_getting_Cache_Group (0.00s) -=== RUN TestGetServerInterfaces ---- PASS: TestGetServerInterfaces (0.00s) -=== RUN TestGetDSIDAndCDNFromName -=== RUN TestGetDSIDAndCDNFromName/Success:_DS_ID_and_CDN_Name_found - db_helpers_test.go:311: Starting test scenario: Success: DS ID and CDN Name found -=== RUN TestGetDSIDAndCDNFromName/Failure:_DS_ID_or_CDN_Name_not_found - db_helpers_test.go:311: Starting test scenario: Failure: DS ID or CDN Name not found -=== RUN TestGetDSIDAndCDNFromName/Failure:_Storage_error_getting_DS_ID_or_CDN_Name - db_helpers_test.go:311: Starting test scenario: Failure: Storage error getting DS ID or CDN Name ---- PASS: TestGetDSIDAndCDNFromName (0.00s) - --- PASS: TestGetDSIDAndCDNFromName/Success:_DS_ID_and_CDN_Name_found (0.00s) - --- PASS: TestGetDSIDAndCDNFromName/Failure:_DS_ID_or_CDN_Name_not_found (0.00s) - --- PASS: TestGetDSIDAndCDNFromName/Failure:_Storage_error_getting_DS_ID_or_CDN_Name (0.00s) -=== RUN TestGetCDNIDFromName -=== RUN TestGetCDNIDFromName/Success:_CDN_ID_found - db_helpers_test.go:372: Starting test scenario: Success: CDN ID found -=== RUN TestGetCDNIDFromName/Failure:_CDN_ID_not_found - db_helpers_test.go:372: Starting test scenario: Failure: CDN ID not found -=== RUN TestGetCDNIDFromName/Failure:_Storage_error_getting_CDN_ID - db_helpers_test.go:372: Starting test scenario: Failure: Storage error getting CDN ID ---- PASS: TestGetCDNIDFromName (0.00s) - --- PASS: TestGetCDNIDFromName/Success:_CDN_ID_found (0.00s) - --- PASS: TestGetCDNIDFromName/Failure:_CDN_ID_not_found (0.00s) - --- PASS: TestGetCDNIDFromName/Failure:_Storage_error_getting_CDN_ID (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/dbhelpers (cached) -=== RUN TestDeliveryServicesRequiredCapabilityInterfaces ---- PASS: TestDeliveryServicesRequiredCapabilityInterfaces (0.00s) -=== RUN TestCreateDeliveryServicesRequiredCapability ---- PASS: TestCreateDeliveryServicesRequiredCapability (0.00s) -=== RUN TestUnauthorizedCreateDeliveryServicesRequiredCapability ---- PASS: TestUnauthorizedCreateDeliveryServicesRequiredCapability (0.00s) -=== RUN TestReadDeliveryServicesRequiredCapability ---- PASS: TestReadDeliveryServicesRequiredCapability (0.00s) -=== RUN TestDeleteDeliveryServicesRequiredCapability ---- PASS: TestDeleteDeliveryServicesRequiredCapability (0.00s) -=== RUN TestUnauthorizedDeleteDeliveryServicesRequiredCapability ---- PASS: TestUnauthorizedDeleteDeliveryServicesRequiredCapability (0.00s) -=== RUN TestCreateDeliveryServicesRequiredCapabilityInvalidDSType ---- PASS: TestCreateDeliveryServicesRequiredCapabilityInvalidDSType (0.00s) -=== RUN TestGetDetails ---- PASS: TestGetDetails (0.00s) -=== RUN TestGetOldDetailsError ---- PASS: TestGetOldDetailsError (0.00s) -=== RUN TestGetDeliveryServicesMatchLists ---- PASS: TestGetDeliveryServicesMatchLists (0.00s) -=== RUN TestGetDSTLSVersions ---- PASS: TestGetDSTLSVersions (0.00s) -=== RUN TestMakeExampleURLs ---- PASS: TestMakeExampleURLs (0.00s) -=== RUN TestReadGetDeliveryServices ---- PASS: TestReadGetDeliveryServices (0.00s) -=== RUN TestGetEligibleServers ---- PASS: TestGetEligibleServers (0.00s) -=== RUN TestDecodePrivateKeyPKCS8RSA2048 ---- PASS: TestDecodePrivateKeyPKCS8RSA2048 (0.00s) -=== RUN TestDecodePrivateKeyPKCS1RSA2048 ---- PASS: TestDecodePrivateKeyPKCS1RSA2048 (0.00s) -=== RUN TestDecodePrivateKeyECDSANISTPrime256V1 ---- PASS: TestDecodePrivateKeyECDSANISTPrime256V1 (0.00s) -=== RUN TestDecodePrivateKeyECDSANISTPrime256V1WithoutParams ---- PASS: TestDecodePrivateKeyECDSANISTPrime256V1WithoutParams (0.00s) -=== RUN TestDecodePrivateKeyECDSANISTSecP384R1 ---- PASS: TestDecodePrivateKeyECDSANISTSecP384R1 (0.00s) -=== RUN TestDecodePrivateKeyECDSANISTSecP384R1WithoutParams ---- PASS: TestDecodePrivateKeyECDSANISTSecP384R1WithoutParams (0.00s) -=== RUN TestDecodeRSAPrivateKeyBadData - keys_test.go:2202: expected error message: private key validation error: no RSA private key PEM blocks found ---- PASS: TestDecodeRSAPrivateKeyBadData (0.00s) -=== RUN TestDecodeECDSAPrivateKeyBadData - keys_test.go:2213: expected error message: private key validation error: no EC private key PEM blocks found ---- PASS: TestDecodeECDSAPrivateKeyBadData (0.00s) -=== RUN TestDecodePrivateKeyRSAEncrypted - keys_test.go:2224: expected error message: private key validation error: encrypted private key not supported - header: 4,ENCRYPTED ---- PASS: TestDecodePrivateKeyRSAEncrypted (0.00s) -=== RUN TestDecodePrivateKeyECDSAEncrypted - keys_test.go:2235: expected error message: private key validation error: encrypted private key not supported - header: 4,ENCRYPTED ---- PASS: TestDecodePrivateKeyECDSAEncrypted (0.00s) -=== RUN TestVerifyAndEncodeCertificateBadData ---- PASS: TestVerifyAndEncodeCertificateBadData (0.00s) -=== RUN TestVerifyAndEncodeCertificateSelfSignedCertKeyPairDSA - keys_test.go:2254: expected error message: cert/key validation error: DSA public key algorithm unsupported ---- PASS: TestVerifyAndEncodeCertificateSelfSignedCertKeyPairDSA (0.00s) -=== RUN TestVerifyAndEncodeCertificateSelfSignedX509v1 ---- PASS: TestVerifyAndEncodeCertificateSelfSignedX509v1 (0.11s) -=== RUN TestVerifyAndEncodeCertificateSelfSignedNoSkiAkiCertKeyPair ---- PASS: TestVerifyAndEncodeCertificateSelfSignedNoSkiAkiCertKeyPair (0.00s) -=== RUN TestVerifyAndEncodeCertificateSelfSignedCertKeyPair ---- PASS: TestVerifyAndEncodeCertificateSelfSignedCertKeyPair (0.00s) -=== RUN TestVerifyAndEncodeCertificateSelfSignedCertKeyPairMisMatchedPrivateKey - keys_test.go:2344: expected error message: cert/key (rsa) mismatch error: RSA public N modulus value mismatch ---- PASS: TestVerifyAndEncodeCertificateSelfSignedCertKeyPairMisMatchedPrivateKey (0.00s) -=== RUN TestVerifyAndEncodeCertificateSelfSignedNoServerAuthExtKeyUsage - keys_test.go:2356: expected error message: certificate (x509v3) validation error: server certificate missing 'serverAuth' extended key usage ---- PASS: TestVerifyAndEncodeCertificateSelfSignedNoServerAuthExtKeyUsage (0.00s) -=== RUN TestVerifyAndEncodeCertificateSelfSignedRSANoKeyEnciphermentKeyUsage - keys_test.go:2369: expected error message: cert/key (rsa) validation: no keyEncipherment keyUsage extension present in x509v3 server certificate ---- PASS: TestVerifyAndEncodeCertificateSelfSignedRSANoKeyEnciphermentKeyUsage (0.00s) -=== RUN TestVerifyAndEncodeCertificateCASignedCertKeyPair ---- PASS: TestVerifyAndEncodeCertificateCASignedCertKeyPair (0.00s) -=== RUN TestVerifyAndEncodeCertificateCASignedCertKeyPairWithRootCA ---- PASS: TestVerifyAndEncodeCertificateCASignedCertKeyPairWithRootCA (0.00s) -=== RUN TestVerifyAndEncodeCertificateCASignedNoSkiAkiCertKeyPair ---- PASS: TestVerifyAndEncodeCertificateCASignedNoSkiAkiCertKeyPair (0.00s) -=== RUN TestVerifyAndEncodeCertificateCASignedNoSkiAkiCertKeyPairWithRootCA ---- PASS: TestVerifyAndEncodeCertificateCASignedNoSkiAkiCertKeyPairWithRootCA (0.00s) -=== RUN TestVerifyAndEncodeCertificateECDSASelfSignedCertificateKeyPairWithoutDigitalSignatureKeyUsage - keys_test.go:2489: expected error message: cert/key (ecdsa) validation error: no digitalSignature keyUsage extension present in x509v3 server certificate ---- PASS: TestVerifyAndEncodeCertificateECDSASelfSignedCertificateKeyPairWithoutDigitalSignatureKeyUsage (0.00s) -=== RUN TestVerifyAndEncodeCertificateECDSASelfSignedCertificateKeyPair ---- PASS: TestVerifyAndEncodeCertificateECDSASelfSignedCertificateKeyPair (0.00s) -=== RUN TestVerifyAndEncodeCertificateECDSASelfSignedCertificateKeyPairECDisabled - keys_test.go:2509: expected error message: cert/key validation error: ECDSA public key algorithm unsupported for non-DNS delivery service type ---- PASS: TestVerifyAndEncodeCertificateECDSASelfSignedCertificateKeyPairECDisabled (0.00s) -=== RUN TestVerifyAndEncodeCertificateECDSASelfSignedCertificateKeyPairWithoutParams ---- PASS: TestVerifyAndEncodeCertificateECDSASelfSignedCertificateKeyPairWithoutParams (0.00s) -=== RUN TestVerifyAndEncodeCertificateECDSASelfSignedCertificateKeyPairMisMatchedPrivateKey - keys_test.go:2529: expected error message: cert/key (ecdsa) mismatch error: ECDSA public X value mismatch ---- PASS: TestVerifyAndEncodeCertificateECDSASelfSignedCertificateKeyPairMisMatchedPrivateKey (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/deliveryservice (cached) -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/deliveryservice/consistenthash [no test files] -=== RUN TestGetAssignee ---- PASS: TestGetAssignee (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/deliveryservice/request (cached) -=== RUN TestFuncs ---- PASS: TestFuncs (0.00s) -=== RUN TestInterfaces ---- PASS: TestInterfaces (0.00s) -=== RUN TestValidate ---- PASS: TestValidate (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/deliveryservice/request/comment (cached) -=== RUN TestValidateDSSAssignments ---- PASS: TestValidateDSSAssignments (0.00s) -=== RUN TestReadServers ---- PASS: TestReadServers (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/deliveryservice/servers (cached) -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/deliveryservicerequests [no test files] -=== RUN TestValidateDSRegexOrderExisting ---- PASS: TestValidateDSRegexOrderExisting (0.00s) -=== RUN TestValidateDSRegex ---- PASS: TestValidateDSRegex (0.00s) -=== RUN TestUpdateImmutableRegex ---- PASS: TestUpdateImmutableRegex (0.00s) -=== RUN TestGetCurrentDetails ---- PASS: TestGetCurrentDetails (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/deliveryservicesregexes (cached) -=== RUN TestGetDivisions ---- PASS: TestGetDivisions (0.00s) -=== RUN TestInterfaces ---- PASS: TestInterfaces (0.00s) -=== RUN TestValidation ---- PASS: TestValidation (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/division (cached) -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/federation_resolvers [no test files] -=== RUN TestCheckFedDSDeletion -=== RUN TestCheckFedDSDeletion/Success:_Deleted_Federation_Delivery_Service - ds_test.go:68: Starting test scenario: Success: Deleted Federation Delivery Service -=== RUN TestCheckFedDSDeletion/Failure:_Remove_last_Federation_Delivery_Service - ds_test.go:68: Starting test scenario: Failure: Remove last Federation Delivery Service -=== RUN TestCheckFedDSDeletion/Failure:_Federation_not_found - ds_test.go:68: Starting test scenario: Failure: Federation not found -=== RUN TestCheckFedDSDeletion/Failure:_Delivery_Service_not_found - ds_test.go:68: Starting test scenario: Failure: Delivery Service not found ---- PASS: TestCheckFedDSDeletion (0.00s) - --- PASS: TestCheckFedDSDeletion/Success:_Deleted_Federation_Delivery_Service (0.00s) - --- PASS: TestCheckFedDSDeletion/Failure:_Remove_last_Federation_Delivery_Service (0.00s) - --- PASS: TestCheckFedDSDeletion/Failure:_Federation_not_found (0.00s) - --- PASS: TestCheckFedDSDeletion/Failure:_Delivery_Service_not_found (0.00s) -=== RUN TestAddFederationResolverMappingsForCurrentUser -=== RUN TestAddFederationResolverMappingsForCurrentUser/add_Federation_Resolver_Mappings_for_the_current_user -=== RUN TestAddFederationResolverMappingsForCurrentUser/add_Federation_Resolver_Mappings_for_the_current_user_when_no_federations_exist/are_assigned_to_them - federations_test.go:156: Got expected user error: No federation(s) found for user test on delivery service 'test'. -=== RUN TestAddFederationResolverMappingsForCurrentUser/add_Federation_Resolver_Mappings_for_a_DS_unauthorized_to_the_current_user's_tenant - federations_test.go:210: Got expected user error: 'test' - no such Delivery Service - federations_test.go:215: Got expected system error: User 'test' requested unauthorized federation resolver mapping modification on the 'test' Delivery Service ---- PASS: TestAddFederationResolverMappingsForCurrentUser (0.00s) - --- PASS: TestAddFederationResolverMappingsForCurrentUser/add_Federation_Resolver_Mappings_for_the_current_user (0.00s) - --- PASS: TestAddFederationResolverMappingsForCurrentUser/add_Federation_Resolver_Mappings_for_the_current_user_when_no_federations_exist/are_assigned_to_them (0.00s) - --- PASS: TestAddFederationResolverMappingsForCurrentUser/add_Federation_Resolver_Mappings_for_a_DS_unauthorized_to_the_current_user's_tenant (0.00s) -=== RUN TestGetMappingsFromRequestBody ---- PASS: TestGetMappingsFromRequestBody (0.00s) -=== RUN TestRemoveFederationResolverMappingsForCurrentUser ---- PASS: TestRemoveFederationResolverMappingsForCurrentUser (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/federations (cached) -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/invalidationjobs [no test files] -=== RUN TestCrypt -=== RUN TestCrypt/#00 - crypt_test.go:68: crypt("password", "salt") = "$1$salt$qJH7.N4xYta3aEG/dfqo/0" -=== RUN TestCrypt/#01 - crypt_test.go:68: crypt("Traffic Ops", "pepper") = "$1$pepper$AHauHHBeRPuBP0LCO0oBW0" -=== RUN TestCrypt/#02 - crypt_test.go:68: crypt("T0p S3cr3T", "N4xYta3a") = "$1$N4xYta3a$E4g3CFzttHfxgEvY4PmrI/" -=== RUN TestCrypt/#03 - crypt_test.go:68: crypt("a", "b") = "$1$b$J4vSIPg.1IiJxJ.JOHsOS1" -=== RUN TestCrypt/#04 - crypt_test.go:68: crypt("", "salt") = "$1$salt$UsdFqFVB.FsuinRDK5eE.." -=== RUN TestCrypt/#05 - crypt_test.go:68: crypt("pw", "") = "$1$$F0Fc2lbYpzr3KKdKkM0Wj." -=== RUN TestCrypt/#06 - crypt_test.go:68: crypt("", "") = "$1$$qRPK7m23GJusamGpoGLby/" ---- PASS: TestCrypt (0.00s) - --- PASS: TestCrypt/#00 (0.00s) - --- PASS: TestCrypt/#01 (0.00s) - --- PASS: TestCrypt/#02 (0.00s) - --- PASS: TestCrypt/#03 (0.00s) - --- PASS: TestCrypt/#04 (0.00s) - --- PASS: TestCrypt/#05 (0.00s) - --- PASS: TestCrypt/#06 (0.00s) -=== RUN TestRndSalt -=== RUN TestRndSalt/0 -=== RUN TestRndSalt/1 -=== RUN TestRndSalt/2 -=== RUN TestRndSalt/3 -=== RUN TestRndSalt/4 -=== RUN TestRndSalt/5 -=== RUN TestRndSalt/6 -=== RUN TestRndSalt/7 -=== RUN TestRndSalt/8 -=== RUN TestRndSalt/9 ---- PASS: TestRndSalt (0.00s) - --- PASS: TestRndSalt/0 (0.00s) - --- PASS: TestRndSalt/1 (0.00s) - --- PASS: TestRndSalt/2 (0.00s) - --- PASS: TestRndSalt/3 (0.00s) - --- PASS: TestRndSalt/4 (0.00s) - --- PASS: TestRndSalt/5 (0.00s) - --- PASS: TestRndSalt/6 (0.00s) - --- PASS: TestRndSalt/7 (0.00s) - --- PASS: TestRndSalt/8 (0.00s) - --- PASS: TestRndSalt/9 (0.00s) -=== RUN TestHelperMockCmd ---- PASS: TestHelperMockCmd (0.00s) -=== RUN TestISOS -=== RUN TestISOS/cmd_success - iso_test.go:83: response status = 200 - iso_test.go:90: header "Content-Type" = "attachment; filename=\"db.infra.ciab.test-centos72.iso\"" - iso_test.go:97: header "Content-Type" = "application/octet-stream" - iso_test.go:106: got command: "mkisofs -joliet-long -input-charset utf-8 -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 -boot-info-table -R -J -v -T /var/folders/r5/snb0k76d0n742rp0mtwzmx_80000gp/T/TestISOS2460179975/centos72" -=== RUN TestISOS/cmd_failure - iso_test.go:144: header "Content-Type" = "application/json" - iso_test.go:153: response: tc.Alerts{Alerts:[]tc.Alert{tc.Alert{Text:"Internal Server Error", Level:"error"}}} ---- PASS: TestISOS (2.87s) - --- PASS: TestISOS/cmd_success (0.81s) - --- PASS: TestISOS/cmd_failure (2.06s) -=== RUN TestISORequest_validate -=== RUN TestISORequest_validate/valid_with_dhcp_false - iso_test.go:487: isoRequest.validate() = -=== RUN TestISORequest_validate/valid_with_CIDR_prefix_on_IPv6_address - iso_test.go:487: isoRequest.validate() = -=== RUN TestISORequest_validate/valid_with_valid_IPv6_address - iso_test.go:487: isoRequest.validate() = -=== RUN TestISORequest_validate/invalid_with_valid_IPv4_address_as_an_IPv6 - iso_test.go:487: isoRequest.validate() = ip6Address must be a valid IPv6 address (with optional CIDR prefix) -=== RUN TestISORequest_validate/invalid_with_invalid_IPv6 - iso_test.go:487: isoRequest.validate() = ip6Address must be a valid IPv6 address (with optional CIDR prefix) -=== RUN TestISORequest_validate/valid_with_dhcp_true - iso_test.go:487: isoRequest.validate() = -=== RUN TestISORequest_validate/invalid_with_dhcp_false - iso_test.go:487: isoRequest.validate() = ipAddress is required if DHCP is no, ipNetmask is required if DHCP is no, ipGateway is required if DHCP is no -=== RUN TestISORequest_validate/valid_with_mgmt_addr - iso_test.go:487: isoRequest.validate() = -=== RUN TestISORequest_validate/invalid_with_mgmt_addr - iso_test.go:487: isoRequest.validate() = mgmtInterface is required when mgmtIpAddress is provided -=== RUN TestISORequest_validate/invalid_with_zero_values - iso_test.go:487: isoRequest.validate() = osversionDir is required, hostName is required, domainName is required, rootPass is required, dhcp is required, interfaceMtu is required, disk is required ---- PASS: TestISORequest_validate (0.00s) - --- PASS: TestISORequest_validate/valid_with_dhcp_false (0.00s) - --- PASS: TestISORequest_validate/valid_with_CIDR_prefix_on_IPv6_address (0.00s) - --- PASS: TestISORequest_validate/valid_with_valid_IPv6_address (0.00s) - --- PASS: TestISORequest_validate/invalid_with_valid_IPv4_address_as_an_IPv6 (0.00s) - --- PASS: TestISORequest_validate/invalid_with_invalid_IPv6 (0.00s) - --- PASS: TestISORequest_validate/valid_with_dhcp_true (0.00s) - --- PASS: TestISORequest_validate/invalid_with_dhcp_false (0.00s) - --- PASS: TestISORequest_validate/valid_with_mgmt_addr (0.00s) - --- PASS: TestISORequest_validate/invalid_with_mgmt_addr (0.00s) - --- PASS: TestISORequest_validate/invalid_with_zero_values (0.00s) -=== RUN TestBoolStr_UnmarshalText -=== RUN TestBoolStr_UnmarshalText/no - iso_test.go:530: got {isSet:true v:false} -=== RUN TestBoolStr_UnmarshalText/No - iso_test.go:530: got {isSet:true v:false} -=== RUN TestBoolStr_UnmarshalText/YES - iso_test.go:530: got {isSet:true v:true} -=== RUN TestBoolStr_UnmarshalText/other - iso_test.go:530: got {isSet:false v:false} -=== RUN TestBoolStr_UnmarshalText/#00 - iso_test.go:530: got {isSet:false v:false} ---- PASS: TestBoolStr_UnmarshalText (0.00s) - --- PASS: TestBoolStr_UnmarshalText/no (0.00s) - --- PASS: TestBoolStr_UnmarshalText/No (0.00s) - --- PASS: TestBoolStr_UnmarshalText/YES (0.00s) - --- PASS: TestBoolStr_UnmarshalText/other (0.00s) - --- PASS: TestBoolStr_UnmarshalText/#00 (0.00s) -=== RUN TestISORequest_validateOSDir -=== RUN TestISORequest_validateOSDir/VALID-OS-DIR - iso_test.go:617: validateOSDir() got true -=== RUN TestISORequest_validateOSDir/ANOTHER-VALID-OS-DIR - iso_test.go:617: validateOSDir() got true -=== RUN TestISORequest_validateOSDir/INVALID-OS-DIR - iso_test.go:617: validateOSDir() got false ---- PASS: TestISORequest_validateOSDir (0.00s) - --- PASS: TestISORequest_validateOSDir/VALID-OS-DIR (0.00s) - --- PASS: TestISORequest_validateOSDir/ANOTHER-VALID-OS-DIR (0.00s) - --- PASS: TestISORequest_validateOSDir/INVALID-OS-DIR (0.00s) -=== RUN TestStreamISOCmd_stdout - mkisofs_test.go:67: got: "mkisofs -joliet-long -input-charset utf-8 -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 -boot-info-table -R -J -v -T /tmp/nothing/here" ---- PASS: TestStreamISOCmd_stdout (1.36s) -=== RUN TestStreamISOCmd_stdout_err - mkisofs_test.go:96: got (expected) error: "exit status 1: mkisofs -joliet-long -input-charset utf-8 -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 -boot-info-table -R -J -v -T /tmp/nothing/here" ---- PASS: TestStreamISOCmd_stdout_err (0.87s) -=== RUN TestStreamISOCmd_file - mkisofs_test.go:125: isoDest = "/var/folders/r5/snb0k76d0n742rp0mtwzmx_80000gp/T/genISO2108450873/tmp.iso" - mkisofs_test.go:143: got: "/var/folders/r5/snb0k76d0n742rp0mtwzmx_80000gp/T/TestStreamISOCmd_file2029127300/generate /var/folders/r5/snb0k76d0n742rp0mtwzmx_80000gp/T/genISO2108450873/tmp.iso" - mkisofs_test.go:153: stat of "/var/folders/r5/snb0k76d0n742rp0mtwzmx_80000gp/T/genISO2108450873" (expected): stat /var/folders/r5/snb0k76d0n742rp0mtwzmx_80000gp/T/genISO2108450873: no such file or directory ---- PASS: TestStreamISOCmd_file (0.03s) -=== RUN TestStreamISOCmd_file_err - mkisofs_test.go:182: isoDest = "/var/folders/r5/snb0k76d0n742rp0mtwzmx_80000gp/T/genISO732973818/tmp.iso" - mkisofs_test.go:197: got (expected) error: "exit status 1: /var/folders/r5/snb0k76d0n742rp0mtwzmx_80000gp/T/TestStreamISOCmd_file_err3515397006/generate /var/folders/r5/snb0k76d0n742rp0mtwzmx_80000gp/T/genISO732973818/tmp.iso" ---- PASS: TestStreamISOCmd_file_err (0.02s) -=== RUN TestKickstarterDir -=== RUN TestKickstarterDir/default - mkisofs_test.go:267: kickstarterDir(tx, "templeOS") = "/var/www/files/templeOS" -=== RUN TestKickstarterDir/param-override - mkisofs_test.go:267: kickstarterDir(tx, "anotherOS/dir") = "/var/override/dir/anotherOS/dir" ---- PASS: TestKickstarterDir (0.00s) - --- PASS: TestKickstarterDir/default (0.00s) - --- PASS: TestKickstarterDir/param-override (0.00s) -=== RUN TestWriteKSCfgs -=== RUN TestWriteKSCfgs/empty - mkisofs_test.go:321: writeKSCfgs("/var/folders/r5/snb0k76d0n742rp0mtwzmx_80000gp/T/TestWriteKSCfgs4024128887", isoRequest): - mkisofs_test.go:348: state.out: - Dir== /var/folders/r5/snb0k76d0n742rp0mtwzmx_80000gp/T/TestWriteKSCfgs4024128887 - - mkisofs_test.go:348: network.cfg: - IPADDR="" - NETMASK="" - GATEWAY="" - DEVICE="" - MTU="0" - NAMESERVER="192.168.1.1" - HOSTNAME="" - NETWORKING_IPV6="yes" - IPV6ADDR="" - IPV6_DEFAULTGW="" - DHCP="no" - mkisofs_test.go:348: mgmt_network.cfg: - IPADDR="" - NETMASK="" - GATEWAY="" - DEVICE="" - mkisofs_test.go:348: password.cfg: - rootpw --iscrypted $1$62fSX9Hf$oqtWslyzKZzqQIemUMaij0 - mkisofs_test.go:348: disk.cfg: - boot_drives="" -=== RUN TestWriteKSCfgs/complete - mkisofs_test.go:321: writeKSCfgs("/var/folders/r5/snb0k76d0n742rp0mtwzmx_80000gp/T/TestWriteKSCfgs1059205786", isoRequest): - mkisofs_test.go:348: state.out: - Dir== /var/folders/r5/snb0k76d0n742rp0mtwzmx_80000gp/T/TestWriteKSCfgs1059205786 - /usr/local/bin mkisofs arg1 -opt1 -opt2 - mkisofs_test.go:348: network.cfg: - IPADDR="192.168.1.2" - NETMASK="255.255.255.0" - GATEWAY="192.168.1.255" - BOND_DEVICE="bond0" - MTU="1500" - NAMESERVER="192.168.1.1" - HOSTNAME="test.server.cdn.example" - NETWORKING_IPV6="yes" - IPV6ADDR="beef::1" - IPV6_DEFAULTGW="::1" - BONDING_OPTS="miimon=100 mode=4 lacp_rate=fast xmit_hash_policy=layer3+4" - DHCP="yes" - mkisofs_test.go:348: mgmt_network.cfg: - IPADDR="10.10.0.1" - NETMASK="10.10.255.255" - GATEWAY="10.10.0.255" - DEVICE="eth0" - mkisofs_test.go:348: password.cfg: - rootpw --iscrypted $1$PdibCMHc$.8C8L6Ig7jSgGeHebVP5y0 - mkisofs_test.go:348: disk.cfg: - boot_drives="sda1" ---- PASS: TestWriteKSCfgs (0.01s) - --- PASS: TestWriteKSCfgs/empty (0.00s) - --- PASS: TestWriteKSCfgs/complete (0.00s) -=== RUN TestWriteNetworkCfg -=== RUN TestWriteNetworkCfg/empty - mkisofs_test.go:489: writeNetworkCfg(): - IPADDR="" - NETMASK="" - GATEWAY="" - DEVICE="" - MTU="0" - NAMESERVER="" - HOSTNAME="" - NETWORKING_IPV6="yes" - IPV6ADDR="" - IPV6_DEFAULTGW="" - DHCP="no" -=== RUN TestWriteNetworkCfg/no_domain - mkisofs_test.go:489: writeNetworkCfg(): - IPADDR="192.168.1.2" - NETMASK="255.255.255.0" - GATEWAY="192.168.1.255" - DEVICE="eth0" - MTU="1500" - NAMESERVER="8.8.8.8,1.1.1.1" - HOSTNAME="test.server" - NETWORKING_IPV6="yes" - IPV6ADDR="beef::1" - IPV6_DEFAULTGW="::1" - DHCP="yes" -=== RUN TestWriteNetworkCfg/non-bonded - mkisofs_test.go:489: writeNetworkCfg(): - IPADDR="192.168.1.2" - NETMASK="255.255.255.0" - GATEWAY="192.168.1.255" - DEVICE="eth0" - MTU="1500" - NAMESERVER="8.8.8.8,1.1.1.1" - HOSTNAME="test.server.example.com" - NETWORKING_IPV6="yes" - IPV6ADDR="beef::1" - IPV6_DEFAULTGW="::1" - DHCP="yes" -=== RUN TestWriteNetworkCfg/bonded - mkisofs_test.go:489: writeNetworkCfg(): - IPADDR="192.168.1.2" - NETMASK="255.255.255.0" - GATEWAY="192.168.1.255" - BOND_DEVICE="bond01" - MTU="1500" - NAMESERVER="8.8.8.8,1.1.1.1" - HOSTNAME="test.server" - NETWORKING_IPV6="yes" - IPV6ADDR="beef::1" - IPV6_DEFAULTGW="::1" - BONDING_OPTS="miimon=100 mode=4 lacp_rate=fast xmit_hash_policy=layer3+4" - DHCP="yes" ---- PASS: TestWriteNetworkCfg (0.00s) - --- PASS: TestWriteNetworkCfg/empty (0.00s) - --- PASS: TestWriteNetworkCfg/no_domain (0.00s) - --- PASS: TestWriteNetworkCfg/non-bonded (0.00s) - --- PASS: TestWriteNetworkCfg/bonded (0.00s) -=== RUN TestWriteMgmtNetworkCfg -=== RUN TestWriteMgmtNetworkCfg/empty - mkisofs_test.go:560: writeMgmtNetworkCfg(): - IPADDR="" - NETMASK="" - GATEWAY="" - DEVICE="" -=== RUN TestWriteMgmtNetworkCfg/IPv4 - mkisofs_test.go:560: writeMgmtNetworkCfg(): - IPADDR="192.168.2.3" - NETMASK="255.255.255.255" - GATEWAY="192.168.1.255" - DEVICE="eth0" -=== RUN TestWriteMgmtNetworkCfg/IPv6 - mkisofs_test.go:560: writeMgmtNetworkCfg(): - IPV6ADDR="beef::1" - NETMASK="255.255.255.255" - GATEWAY="192.168.1.255" - DEVICE="eth0" ---- PASS: TestWriteMgmtNetworkCfg (0.00s) - --- PASS: TestWriteMgmtNetworkCfg/empty (0.00s) - --- PASS: TestWriteMgmtNetworkCfg/IPv4 (0.00s) - --- PASS: TestWriteMgmtNetworkCfg/IPv6 (0.00s) -=== RUN TestWriteDiskCfg -=== RUN TestWriteDiskCfg/empty - mkisofs_test.go:603: writeDiskCfg(): - boot_drives="" -=== RUN TestWriteDiskCfg/non-empty - mkisofs_test.go:603: writeDiskCfg(): - boot_drives="sda1" ---- PASS: TestWriteDiskCfg (0.00s) - --- PASS: TestWriteDiskCfg/empty (0.00s) - --- PASS: TestWriteDiskCfg/non-empty (0.00s) -=== RUN TestWritePasswordCfg -=== RUN TestWritePasswordCfg/empty - mkisofs_test.go:644: writePasswordCfg(): - "rootpw --iscrypted $1$salt$UsdFqFVB.FsuinRDK5eE..\n" -=== RUN TestWritePasswordCfg/non-empty - mkisofs_test.go:644: writePasswordCfg(): - "rootpw --iscrypted $1$salt$17HeaymOIi.65dl76MkK01\n" ---- PASS: TestWritePasswordCfg (0.00s) - --- PASS: TestWritePasswordCfg/empty (0.00s) - --- PASS: TestWritePasswordCfg/non-empty (0.00s) -=== RUN TestWritePasswordCfg_rndSalt -=== RUN TestWritePasswordCfg_rndSalt/empty - mkisofs_test.go:696: writePasswordCfg(): - "rootpw --iscrypted $1$09R4vGuA$0shIp9zZOCLhGHv9KEt3a1\n" -=== RUN TestWritePasswordCfg_rndSalt/non-empty - mkisofs_test.go:696: writePasswordCfg(): - "rootpw --iscrypted $1$ZZ4mQA5e$0/TxriFqQkychTbb0oZux/\n" -=== RUN TestWritePasswordCfg_rndSalt/long - mkisofs_test.go:696: writePasswordCfg(): - "rootpw --iscrypted $1$BjbBMl5B$qswW3lFyMdSmhyPfzVYCI.\n" ---- PASS: TestWritePasswordCfg_rndSalt (0.00s) - --- PASS: TestWritePasswordCfg_rndSalt/empty (0.00s) - --- PASS: TestWritePasswordCfg_rndSalt/non-empty (0.00s) - --- PASS: TestWritePasswordCfg_rndSalt/long (0.00s) -=== RUN TestGetOSVersions -=== RUN TestGetOSVersions/valid-file - osversions_test.go:94: getOSVersions(): tc.OSVersionsResponse{"TempleOS":"temple503"} -=== RUN TestGetOSVersions/invalid-file - osversions_test.go:137: getOSVersions() err (expected) = open /var/folders/r5/snb0k76d0n742rp0mtwzmx_80000gp/T/TestGetOSVersions1146670759/osversions.json: no such file or directory ---- PASS: TestGetOSVersions (0.00s) - --- PASS: TestGetOSVersions/valid-file (0.00s) - --- PASS: TestGetOSVersions/invalid-file (0.00s) -=== RUN TestOsversionsCfgPath -=== RUN TestOsversionsCfgPath/default - osversions_test.go:205: osversionCfgPath(): "/var/www/files/osversions.json" -=== RUN TestOsversionsCfgPath/override - osversions_test.go:205: osversionCfgPath(): "/this/is/not/the/default/osversions.json" -=== RUN TestOsversionsCfgPath/override-cwd - osversions_test.go:205: osversionCfgPath(): "osversions.json" ---- PASS: TestOsversionsCfgPath (0.00s) - --- PASS: TestOsversionsCfgPath/default (0.00s) - --- PASS: TestOsversionsCfgPath/override (0.00s) - --- PASS: TestOsversionsCfgPath/override-cwd (0.00s) -=== RUN TestParseResolve -=== RUN TestParseResolve/empty - resolve_test.go:92: parseResolve(): [] -=== RUN TestParseResolve/comments - resolve_test.go:92: parseResolve(): [] -=== RUN TestParseResolve/invalid_IPs - resolve_test.go:92: parseResolve(): [] -=== RUN TestParseResolve/single - resolve_test.go:92: parseResolve(): [127.0.0.1] -=== RUN TestParseResolve/multi - resolve_test.go:92: parseResolve(): [127.0.0.1 192.168.1.10 beef::1 ::0] ---- PASS: TestParseResolve (0.00s) - --- PASS: TestParseResolve/empty (0.00s) - --- PASS: TestParseResolve/comments (0.00s) - --- PASS: TestParseResolve/invalid_IPs (0.00s) - --- PASS: TestParseResolve/single (0.00s) - --- PASS: TestParseResolve/multi (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/iso (cached) -=== RUN TestLoginWithEmptyCredentials ---- PASS: TestLoginWithEmptyCredentials (0.00s) -=== RUN TestVerifyUrlOnWhiteList ---- PASS: TestVerifyUrlOnWhiteList (0.00s) -=== RUN TestTemplateRender - login_test.go:117: From: no-reply@test.quest - To: em@i.l - Content-Type: text/html - Subject: TO API Unit Tests Password Reset Request - - - - - TO API Unit Tests Password Reset Request - - - - -
-

Someone has requested to change your password for the TO API Unit Tests. If you requested this change, please click the link below and change your password. Otherwise, you can disregard this email.

-

Click to Reset Your Password

-
-
-

Thank you,
- The TO API Unit Tests Team

-
- - ---- PASS: TestTemplateRender (0.00s) -=== RUN TestLogout ---- PASS: TestLogout (0.00s) -=== RUN TestRegistrationTemplateRender - register_test.go:57: From: no-reply@test.quest - To: em@i.l - Content-Type: text/html - Subject: test New User Registration - - - - - test New User Registration - - - - -
-

A new account has been created for you on the test Portal. In the - test Portal, you'll find a dashboard that provides access to all of your - Delivery Services.

-

Click here to finish your registration

-
-
-

Thank you,
- The test Team

-
- - ---- PASS: TestRegistrationTemplateRender (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/login (cached) -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/logs [no test files] -=== RUN TestGetMonitoringServers ---- PASS: TestGetMonitoringServers (0.00s) -=== RUN TestGetProfileWithParams ---- PASS: TestGetProfileWithParams (0.00s) -=== RUN TestGetCachegroups ---- PASS: TestGetCachegroups (0.00s) -=== RUN TestGetProfiles ---- PASS: TestGetProfiles (0.00s) -=== RUN TestGetDeliveryServices ---- PASS: TestGetDeliveryServices (0.00s) -=== RUN TestGetConfig ---- PASS: TestGetConfig (0.00s) -=== RUN TestGetMonitoringJSON ---- PASS: TestGetMonitoringJSON (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/monitoring (cached) -=== RUN TestReadOrigins ---- PASS: TestReadOrigins (0.00s) -=== RUN TestFuncs ---- PASS: TestFuncs (0.00s) -=== RUN TestInterfaces ---- PASS: TestInterfaces (0.00s) -=== RUN TestValidate ---- PASS: TestValidate (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/origin (cached) -=== RUN TestGetParameters ---- PASS: TestGetParameters (0.00s) -=== RUN TestInterfaces ---- PASS: TestInterfaces (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/parameter (cached) -=== RUN TestGetPhysLocations ---- PASS: TestGetPhysLocations (0.00s) -=== RUN TestInterfaces ---- PASS: TestInterfaces (0.00s) -=== RUN TestValidate ---- PASS: TestValidate (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/physlocation (cached) -=== RUN TestPingHandler ---- PASS: TestPingHandler (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/ping (cached) -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/plugin [no test files] -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/plugins [no test files] -=== RUN TestCopyProfileInvalidExistingProfile -=== RUN TestCopyProfileInvalidExistingProfile/multiple_profiles_with_existing_name_returned -=== RUN TestCopyProfileInvalidExistingProfile/existing_profile_does_not_exist ---- PASS: TestCopyProfileInvalidExistingProfile (0.00s) - --- PASS: TestCopyProfileInvalidExistingProfile/multiple_profiles_with_existing_name_returned (0.00s) - --- PASS: TestCopyProfileInvalidExistingProfile/existing_profile_does_not_exist (0.00s) -=== RUN TestCopyNewProfileExists ---- PASS: TestCopyNewProfileExists (0.00s) -=== RUN TestCopyProfile ---- PASS: TestCopyProfile (0.00s) -=== RUN TestGetExportProfileResponse -=== RUN TestGetExportProfileResponse/Success:_Read_export_profile_successful - profile_export_test.go:74: Starting test scenario: Success: Read export profile successful -=== RUN TestGetExportProfileResponse/Success:_Read_export_profile_with_no_parameters_successful - profile_export_test.go:74: Starting test scenario: Success: Read export profile with no parameters successful -=== RUN TestGetExportProfileResponse/Failure:_Storage_error_reading_profile - profile_export_test.go:74: Starting test scenario: Failure: Storage error reading profile ---- PASS: TestGetExportProfileResponse (0.00s) - --- PASS: TestGetExportProfileResponse/Success:_Read_export_profile_successful (0.00s) - --- PASS: TestGetExportProfileResponse/Success:_Read_export_profile_with_no_parameters_successful (0.00s) - --- PASS: TestGetExportProfileResponse/Failure:_Storage_error_reading_profile (0.00s) -=== RUN TestGetImportProfile -=== RUN TestGetImportProfile/Success:_Import_profile_successful - profile_import_test.go:86: Starting test scenario: Success: Import profile successful -=== RUN TestGetImportProfile/Failure:_Import_profile_didn't_insert_row - profile_import_test.go:86: Starting test scenario: Failure: Import profile didn't insert row -=== RUN TestGetImportProfile/Failure:_Import_profile_storage_error - profile_import_test.go:86: Starting test scenario: Failure: Import profile storage error ---- PASS: TestGetImportProfile (0.00s) - --- PASS: TestGetImportProfile/Success:_Import_profile_successful (0.00s) - --- PASS: TestGetImportProfile/Failure:_Import_profile_didn't_insert_row (0.00s) - --- PASS: TestGetImportProfile/Failure:_Import_profile_storage_error (0.00s) -=== RUN TestGetImportProfileParameters -=== RUN TestGetImportProfileParameters/Success:_All_import_parameters_new - profile_import_test.go:292: Starting test scenario: Success: All import parameters new -=== RUN TestGetImportProfileParameters/Success:_All_parameters_exisiting - profile_import_test.go:292: Starting test scenario: Success: All parameters exisiting -=== RUN TestGetImportProfileParameters/Success:_Mix_of_existing/new_parameters - profile_import_test.go:292: Starting test scenario: Success: Mix of existing/new parameters -=== RUN TestGetImportProfileParameters/Success:_Dup_of_existing - profile_import_test.go:292: Starting test scenario: Success: Dup of existing -=== RUN TestGetImportProfileParameters/Fail:_Storage_error_selecting_param - profile_import_test.go:292: Starting test scenario: Fail: Storage error selecting param -=== RUN TestGetImportProfileParameters/Fail:_Storage_error_inserting_param - profile_import_test.go:292: Starting test scenario: Fail: Storage error inserting param ---- PASS: TestGetImportProfileParameters (0.00s) - --- PASS: TestGetImportProfileParameters/Success:_All_import_parameters_new (0.00s) - --- PASS: TestGetImportProfileParameters/Success:_All_parameters_exisiting (0.00s) - --- PASS: TestGetImportProfileParameters/Success:_Mix_of_existing/new_parameters (0.00s) - --- PASS: TestGetImportProfileParameters/Success:_Dup_of_existing (0.00s) - --- PASS: TestGetImportProfileParameters/Fail:_Storage_error_selecting_param (0.00s) - --- PASS: TestGetImportProfileParameters/Fail:_Storage_error_inserting_param (0.00s) -=== RUN TestGetProfiles ---- PASS: TestGetProfiles (0.00s) -=== RUN TestInterfaces ---- PASS: TestInterfaces (0.00s) -=== RUN TestValidate ---- PASS: TestValidate (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/profile (cached) -=== RUN TestGetProfileParameters ---- PASS: TestGetProfileParameters (0.00s) -=== RUN TestInterfaces ---- PASS: TestInterfaces (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/profileparameter (cached) -=== RUN TestReadRegions ---- PASS: TestReadRegions (0.00s) -=== RUN TestInterfaces ---- PASS: TestInterfaces (0.00s) -=== RUN TestValidation - regions_test.go:137: Got expected error validating region with no division: region 'division' is required ---- PASS: TestValidation (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/region (cached) -=== RUN TestFuncs ---- PASS: TestFuncs (0.00s) -=== RUN TestInterfaces ---- PASS: TestInterfaces (0.00s) -=== RUN TestValidate ---- PASS: TestValidate (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/role (cached) -=== RUN TestCompileRoutes - routing_test.go:160: testing path GET api/2.0/cdns - routing_test.go:160: testing path POST api/1.4/users/login - routing_test.go:160: testing path POST api/3.0/cdns - routing_test.go:160: testing path POST api/3.0/users - routing_test.go:160: testing path PUT api/3.0/deliveryservices/3 - routing_test.go:160: testing path DELETE api/3.0/servers/777 - routing_test.go:160: testing path GET api/3.0/cdns/1 - routing_test.go:160: testing path GET /api/1.1/about - routing_test.go:160: testing path GET api/3.0/notatypeweknowabout - routing_test.go:160: testing path GET api/99999.99999/cdns - routing_test.go:160: testing path GET blahblah/api/3.0/cdns - routing_test.go:160: testing path GET internal/api/2.0/federations.json - routing_test.go:160: testing path GET api/3.0/servers - routing_test.go:160: testing path GET api/2.0/servers ---- PASS: TestCompileRoutes (0.01s) -=== RUN TestRoutes ---- PASS: TestRoutes (0.00s) -=== RUN TestCreateRouteMap ---- PASS: TestCreateRouteMap (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/routing (cached) -=== RUN TestWrapHeaders ---- PASS: TestWrapHeaders (0.00s) -=== RUN TestWrapPanicRecover ---- PASS: TestWrapPanicRecover (0.00s) -=== RUN TestGzip ---- PASS: TestGzip (0.00s) -=== RUN TestWrapAuth ---- PASS: TestWrapAuth (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/routing/middleware (cached) -=== RUN TestGetDetailServers ---- PASS: TestGetDetailServers (0.00s) -=== RUN TestAssignDsesToServer ---- PASS: TestAssignDsesToServer (0.00s) -=== RUN TestInterfaces ---- PASS: TestInterfaces (0.00s) -=== RUN TestUpdateServer ---- PASS: TestUpdateServer (0.00s) -=== RUN TestGetServersByCachegroup ---- PASS: TestGetServersByCachegroup (0.00s) -=== RUN TestGetMidServers ---- PASS: TestGetMidServers (0.00s) -=== RUN TestV3Validations - servers_test.go:578: Got expected error validating server with no interfaces: a server must have at least one interface - servers_test.go:592: Got expected error validating server with nil interfaces: a server must have at least one interface - servers_test.go:609: Got expected error validating server with an MTU < 1280: 'interface 'eth0' mtu' must be at least 1280 - servers_test.go:625: Got expected error validating server with no IP addresses: 'interface 'eth0' ipAddresses' cannot be blank, a server must have at least one service address - servers_test.go:640: Got expected error validating server with nil IP addresses: 'interface 'eth0' ipAddresses' cannot be blank, a server must have at least one service address - servers_test.go:661: Got expected error validating server with no service addresses: a server must have at least one service address - servers_test.go:675: Got expected error validating server with too many interfaces with service addresses: interfaces: address '127.0.0.1/32' of interface 'eth0' is marked as a service address, but an IPv4 service address appears earlier in the list - servers_test.go:695: Got expected error validating server with no service addresses: interfaces: address '1.2.3.4/1' of interface 'eth0' is marked as a service address, but an IPv4 service address appears earlier in the list ---- PASS: TestV3Validations (0.00s) -=== RUN TestGetServerUpdateStatus ---- PASS: TestGetServerUpdateStatus (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/server (cached) -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/servercapability [no test files] -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/servercheck [no test files] -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/servercheck/extensions [no test files] -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/servicecategory [no test files] -=== RUN TestFuncs ---- PASS: TestFuncs (0.00s) -=== RUN TestInterfaces ---- PASS: TestInterfaces (0.00s) -=== RUN TestValidate ---- PASS: TestValidate (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/staticdnsentry (cached) -=== RUN TestReadStatuses ---- PASS: TestReadStatuses (0.00s) -=== RUN TestInterfaces ---- PASS: TestInterfaces (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/status (cached) -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/steering [no test files] -=== RUN TestInvalidSteeringTargetType ---- PASS: TestInvalidSteeringTargetType (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/steeringtargets (cached) -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/swaggerdocs/v13 [no test files] -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/swaggerdocs/v13/swaggerspec-server [no test files] -=== RUN TestGetSystemInfo ---- PASS: TestGetSystemInfo (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/systeminfo (cached) -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/tenant [no test files] -=== RUN TestErrorCode ---- PASS: TestErrorCode (0.00s) -=== RUN TestErrorContextPanicMode ---- PASS: TestErrorContextPanicMode (0.00s) -=== RUN TestErrorContextMapping ---- PASS: TestErrorContextMapping (0.00s) -=== RUN TestErrorContextMisc ---- PASS: TestErrorContextMisc (0.00s) -=== RUN TestErrorContext ---- PASS: TestErrorContext (0.00s) -=== RUN TestColsFromStructByTag ---- PASS: TestColsFromStructByTag (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/test (cached) -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/tocookie [no test files] -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/topology [no test files] -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/topology/topology_validation [no test files] -=== RUN TestTSConfigFromRequest ---- PASS: TestTSConfigFromRequest (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/trafficstats (cached) -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/trafficvault [no test files] -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/trafficvault/backends [no test files] -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/trafficvault/backends/disabled [no test files] -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/trafficvault/backends/postgres [no test files] -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/trafficvault/backends/postgres/hashicorpvault [no test files] -=== RUN TestFetchObjectValues ---- PASS: TestFetchObjectValues (0.00s) -=== RUN TestSaveObject ---- PASS: TestSaveObject (0.00s) -=== RUN TestDeleteObject ---- PASS: TestDeleteObject (0.00s) -=== RUN TestSetTLSVersion ---- PASS: TestSetTLSVersion (0.00s) -=== RUN TestGetRiakCluster ---- PASS: TestGetRiakCluster (0.10s) -=== RUN TestUnmarshalGoodRiakConfig ---- PASS: TestUnmarshalGoodRiakConfig (0.00s) -=== RUN TestUnmarshalBadRiakConfig ---- PASS: TestUnmarshalBadRiakConfig (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/trafficvault/backends/riaksvc (cached) -=== RUN TestGetType ---- PASS: TestGetType (0.00s) -=== RUN TestInterfaces ---- PASS: TestInterfaces (0.00s) -=== RUN TestUpdateInvalidType ---- PASS: TestUpdateInvalidType (0.00s) -=== RUN TestDeleteInvalidType ---- PASS: TestDeleteInvalidType (0.00s) -=== RUN TestCreateInvalidType ---- PASS: TestCreateInvalidType (0.00s) -=== RUN TestValidate ---- PASS: TestValidate (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/types (cached) -=== RUN TestValidateURIKeyset ---- PASS: TestValidateURIKeyset (0.00s) -PASS -ok github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/urisigning (cached) -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/user [no test files] -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/util/ims [no test files] -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/util/monitorhlp [no test files] -? github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/vault [no test files] -? github.com/apache/trafficcontrol/traffic_ops/v2-client [no test files] -? github.com/apache/trafficcontrol/traffic_ops/v3-client [no test files] -? github.com/apache/trafficcontrol/traffic_ops/v4-client [no test files] -FAIL From 8244ba081691ef74e46e98e392c27209ad6f716e Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Fri, 27 Aug 2021 13:24:37 -0600 Subject: [PATCH 23/35] Remove unnecessary log package from test file --- traffic_ops/testing/api/v4/jobs_test.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/traffic_ops/testing/api/v4/jobs_test.go b/traffic_ops/testing/api/v4/jobs_test.go index 6e617fdc70..d4353289f4 100644 --- a/traffic_ops/testing/api/v4/jobs_test.go +++ b/traffic_ops/testing/api/v4/jobs_test.go @@ -22,7 +22,6 @@ import ( "testing" "time" - "github.com/apache/trafficcontrol/lib/go-log" "github.com/apache/trafficcontrol/lib/go-tc" client "github.com/apache/trafficcontrol/traffic_ops/v4-client" ) @@ -353,13 +352,7 @@ func GetTestInvalidationJobs(t *testing.T) { } func GetTestJobsByValidData(t *testing.T) { - log.Infof("GetTestJobsByValidData") - toJobs, _, err := TOSession.GetInvalidationJobs(client.RequestOptions{}) - log.Infof("toJobs len: %d", len(toJobs.Response)) - for _, job := range toJobs.Response { - log.Infof("Job: %v\n", job) - } if err != nil { t.Fatalf("error getting jobs %v - alerts: %+v", err, toJobs.Alerts) } From 07ea25ae8f34424d8321a4865f7c1caebe67e0b6 Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Mon, 30 Aug 2021 15:41:49 -0600 Subject: [PATCH 24/35] Remove load_schema calls The load_schema calls are being performed after migrations have occured. This is resulting in create_tables.sql attempting to be applied to a DB state that can conflict. If a table exists, is modified by the migrations then it may not match the original schema as defined in create_tables.sql. This can result in a failure when the script is run. --- traffic_ops_db/test/docker/run-db-test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/traffic_ops_db/test/docker/run-db-test.sh b/traffic_ops_db/test/docker/run-db-test.sh index a4b712b453..befdde1876 100755 --- a/traffic_ops_db/test/docker/run-db-test.sh +++ b/traffic_ops_db/test/docker/run-db-test.sh @@ -97,6 +97,7 @@ if [[ "$old_db_version" -eq 0 ]]; then ./db/admin --env=production reset || { echo "DB reset failed!"; exit 1; } fi +# applies migrations then performs seeding and patching ./db/admin --env=production upgrade || { echo "DB upgrade failed!"; exit 1; } new_db_version=$(get_current_db_version) From 4feec53ad6746d75b1aade675f32b4b7b0a03e23 Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Tue, 28 Sep 2021 18:26:27 -0600 Subject: [PATCH 25/35] Rebase and fix conflicts --- cache-config/t3cutil/toreq/clientfuncs.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cache-config/t3cutil/toreq/clientfuncs.go b/cache-config/t3cutil/toreq/clientfuncs.go index 408c8601c5..188426e8a0 100644 --- a/cache-config/t3cutil/toreq/clientfuncs.go +++ b/cache-config/t3cutil/toreq/clientfuncs.go @@ -515,11 +515,7 @@ func (cl *TOClient) GetDeliveryServiceRegexes(reqHdr http.Header) ([]tc.Delivery return regexes, reqInf, nil } -<<<<<<< HEAD -func (cl *TOClient) GetJobs(reqHdr http.Header, cdnName string) ([]tc.InvalidationJob, toclientlib.ReqInf, error) { -======= -func (cl *TOClient) GetJobs(reqHdr http.Header) ([]tc.InvalidationJobV40, toclientlib.ReqInf, error) { ->>>>>>> Update T3C to account for new InvalidationJob type +func (cl *TOClient) GetJobs(reqHdr http.Header, cdnName string) ([]tc.InvalidationJobV40, toclientlib.ReqInf, error) { if cl.c == nil { oldJobs, inf, err := cl.old.GetJobs() jobs, err := atscfg.JobsToInvalidationJobsV40(oldJobs) From 55927dfdb0d68495671a43dc1415e2ef399935fd Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Fri, 8 Oct 2021 11:23:49 -0600 Subject: [PATCH 26/35] Utilize major version alias (v4 instead of v40) throughout --- .../t3c-generate/cfgfile/cfgfile_test.go | 6 +- cache-config/t3cutil/getdatacfg.go | 2 +- cache-config/t3cutil/toreq/clientfuncs.go | 8 +-- lib/go-atscfg/atscfg.go | 12 ++-- lib/go-atscfg/regexrevalidatedotconfig.go | 6 +- .../regexrevalidatedotconfig_test.go | 2 +- lib/go-tc/invalidationjobs.go | 22 ++++--- traffic_ops/testing/api/v4/jobs_test.go | 4 +- .../invalidationjobs/invalidationjobs.go | 64 +++++++++---------- .../traffic_ops_golang/routing/routes.go | 2 +- traffic_ops/v4-client/job.go | 8 +-- 11 files changed, 70 insertions(+), 66 deletions(-) diff --git a/cache-config/t3c-generate/cfgfile/cfgfile_test.go b/cache-config/t3c-generate/cfgfile/cfgfile_test.go index e82e2d09a2..867e7f7f38 100644 --- a/cache-config/t3c-generate/cfgfile/cfgfile_test.go +++ b/cache-config/t3c-generate/cfgfile/cfgfile_test.go @@ -356,9 +356,9 @@ func randParam() *tc.Parameter { } } -func randJob() *tc.InvalidationJobV40 { +func randJob() *tc.InvalidationJobV4 { now := time.Now() - return &tc.InvalidationJobV40{ + return &tc.InvalidationJobV4{ AssetURL: randStr(), CreatedBy: randStr(), StartTime: &now, @@ -491,7 +491,7 @@ func MakeFakeTOData() *t3cutil.ConfigData { }, DeliveryServiceServers: dss, Server: sv0, - Jobs: []tc.InvalidationJobV40{ + Jobs: []tc.InvalidationJobV4{ *randJob(), *randJob(), }, diff --git a/cache-config/t3cutil/getdatacfg.go b/cache-config/t3cutil/getdatacfg.go index 7c23615802..847049b949 100644 --- a/cache-config/t3cutil/getdatacfg.go +++ b/cache-config/t3cutil/getdatacfg.go @@ -69,7 +69,7 @@ type ConfigData struct { Server *atscfg.Server `json:"server,omitempty"` // Jobs must be all Jobs on the server's CDN. May include jobs on other CDNs. - Jobs []tc.InvalidationJobV40 `json:"jobs,omitempty"` + Jobs []tc.InvalidationJobV4 `json:"jobs,omitempty"` // CDN must be the CDN of the server. CDN *tc.CDN `json:"cdn,omitempty"` diff --git a/cache-config/t3cutil/toreq/clientfuncs.go b/cache-config/t3cutil/toreq/clientfuncs.go index 188426e8a0..664d54bc77 100644 --- a/cache-config/t3cutil/toreq/clientfuncs.go +++ b/cache-config/t3cutil/toreq/clientfuncs.go @@ -515,17 +515,17 @@ func (cl *TOClient) GetDeliveryServiceRegexes(reqHdr http.Header) ([]tc.Delivery return regexes, reqInf, nil } -func (cl *TOClient) GetJobs(reqHdr http.Header, cdnName string) ([]tc.InvalidationJobV40, toclientlib.ReqInf, error) { +func (cl *TOClient) GetJobs(reqHdr http.Header, cdnName string) ([]tc.InvalidationJobV4, toclientlib.ReqInf, error) { if cl.c == nil { oldJobs, inf, err := cl.old.GetJobs() - jobs, err := atscfg.JobsToInvalidationJobsV40(oldJobs) + jobs, err := atscfg.JobsToInvalidationJobsV4(oldJobs) if err != nil { return nil, inf, errors.New("converting old []tc.Job to []tc.InvalidationJob: " + err.Error()) } return jobs, inf, err } - jobs := []tc.InvalidationJobV40{} + jobs := []tc.InvalidationJobV4{} reqInf := toclientlib.ReqInf{} err := torequtil.GetRetry(cl.NumRetries, "jobs_cdn_"+cdnName, &jobs, func(obj interface{}) error { opts := *ReqOpts(reqHdr) @@ -535,7 +535,7 @@ func (cl *TOClient) GetJobs(reqHdr http.Header, cdnName string) ([]tc.Invalidati if err != nil { return errors.New("getting jobs from Traffic Ops '" + torequtil.MaybeIPStr(reqInf.RemoteAddr) + "': " + err.Error()) } - jobs := obj.(*[]tc.InvalidationJobV40) + jobs := obj.(*[]tc.InvalidationJobV4) *jobs = toJobs.Response reqInf = toReqInf return nil diff --git a/lib/go-atscfg/atscfg.go b/lib/go-atscfg/atscfg.go index e17febfe60..f25200de09 100644 --- a/lib/go-atscfg/atscfg.go +++ b/lib/go-atscfg/atscfg.go @@ -697,8 +697,8 @@ func JobToInvalidationJob(jb tc.Job) (tc.InvalidationJob, error) { }, nil } -func JobsToInvalidationJobsV40(oldJobs []tc.Job) ([]tc.InvalidationJobV40, error) { - jobs := make([]tc.InvalidationJobV40, len(oldJobs), len(oldJobs)) +func JobsToInvalidationJobsV4(oldJobs []tc.Job) ([]tc.InvalidationJobV4, error) { + jobs := make([]tc.InvalidationJobV4, len(oldJobs), len(oldJobs)) err := error(nil) for i, oldJob := range oldJobs { jobs[i], err = JobToInvalidationJobV40(oldJob) @@ -709,16 +709,16 @@ func JobsToInvalidationJobsV40(oldJobs []tc.Job) ([]tc.InvalidationJobV40, error return jobs, nil } -func JobToInvalidationJobV40(jb tc.Job) (tc.InvalidationJobV40, error) { +func JobToInvalidationJobV40(jb tc.Job) (tc.InvalidationJobV4, error) { startTime := tc.Time{} if err := json.Unmarshal([]byte(`"`+jb.StartTime+`"`), &startTime); err != nil { - return tc.InvalidationJobV40{}, errors.New("unmarshalling time: " + err.Error()) + return tc.InvalidationJobV4{}, errors.New("unmarshalling time: " + err.Error()) } ttl, err := strconv.Atoi(strings.TrimSuffix(strings.TrimPrefix(jb.Parameters, "TTL:"), "h")) if err != nil { - return tc.InvalidationJobV40{}, errors.New("unmarshalling ttl: " + err.Error()) + return tc.InvalidationJobV4{}, errors.New("unmarshalling ttl: " + err.Error()) } - return tc.InvalidationJobV40{ + return tc.InvalidationJobV4{ AssetURL: util.StrPtr(jb.AssetURL), CreatedBy: util.StrPtr(jb.CreatedBy), DeliveryService: util.StrPtr(jb.DeliveryService), diff --git a/lib/go-atscfg/regexrevalidatedotconfig.go b/lib/go-atscfg/regexrevalidatedotconfig.go index 2b10035063..dec2c25738 100644 --- a/lib/go-atscfg/regexrevalidatedotconfig.go +++ b/lib/go-atscfg/regexrevalidatedotconfig.go @@ -57,7 +57,7 @@ func MakeRegexRevalidateDotConfig( server *Server, deliveryServices []DeliveryService, globalParams []tc.Parameter, - jobs []tc.InvalidationJobV40, + jobs []tc.InvalidationJobV4, opt *RegexRevalidateDotConfigOpts, ) (Cfg, error) { if opt == nil { @@ -80,7 +80,7 @@ func MakeRegexRevalidateDotConfig( dsNames[*ds.XMLID] = struct{}{} } - dsJobs := []tc.InvalidationJobV40{} + dsJobs := []tc.InvalidationJobV4{} for _, job := range jobs { if job.DeliveryService == nil { warnings = append(warnings, "got job from Traffic Ops with a nil DeliveryService! Skipping!") @@ -150,7 +150,7 @@ func (jb jobsSort) Less(i, j int) bool { // - are "purge" jobs // - have a start_time+ttl > now. That is, jobs that haven't expired yet. // Returns the filtered jobs, and any warnings. -func filterJobs(tcJobs []tc.InvalidationJobV40, maxReval time.Duration, minTTL time.Duration) ([]revalJob, []string) { +func filterJobs(tcJobs []tc.InvalidationJobV4, maxReval time.Duration, minTTL time.Duration) ([]revalJob, []string) { warnings := []string{} jobMap := map[string]revalJob{} diff --git a/lib/go-atscfg/regexrevalidatedotconfig_test.go b/lib/go-atscfg/regexrevalidatedotconfig_test.go index 3fa2194470..4d9ca535a0 100644 --- a/lib/go-atscfg/regexrevalidatedotconfig_test.go +++ b/lib/go-atscfg/regexrevalidatedotconfig_test.go @@ -45,7 +45,7 @@ func TestMakeRegexRevalidateDotConfig(t *testing.T) { "unrelated": {"unrelated0", "unrelated1"}, }) - jobs := []tc.InvalidationJobV40{ + jobs := []tc.InvalidationJobV4{ { AssetURL: util.StrPtr("assetURL0"), StartTime: util.TimePtr(time.Now().Add(42*24*time.Hour + time.Hour)), diff --git a/lib/go-tc/invalidationjobs.go b/lib/go-tc/invalidationjobs.go index b26fa5b248..b4bc668885 100644 --- a/lib/go-tc/invalidationjobs.go +++ b/lib/go-tc/invalidationjobs.go @@ -468,11 +468,14 @@ const REFETCH = "REFETCH" // InvalidationJobsResponse is the type of a response from Traffic Ops to a // request made to its /jobs API endpoint for v 4.0+ -type InvalidationJobsResponseV40 struct { - Response []InvalidationJobV40 `json:"response"` +type InvalidationJobsResponseV4 struct { + Response []InvalidationJobV4 `json:"response"` Alerts } +// InvalidationJobCreateV4 is an alias for the InvalidationJobCreateV40 struct used for the latest minor version associated with api major version 4. +type InvalidationJobCreateV4 InvalidationJobCreateV40 + // InvalidationJobCreateV40 represents user input intending to create a content invalidation job. type InvalidationJobCreateV40 struct { // The Delivery Service XML-ID for which the Invalidation Job is to be applied. @@ -495,7 +498,7 @@ type InvalidationJobCreateV40 struct { // Validates the fields submitted for an InvalidationJobCreateV40. These errors // are ultimately returned to the user -func (job *InvalidationJobCreateV40) Validate(tx *sql.Tx) error { +func (job *InvalidationJobCreateV4) Validate(tx *sql.Tx) error { errs := []string{} err := validation.ValidateStruct(job, validation.Field(&job.DeliveryService, validation.Required), @@ -539,6 +542,9 @@ func (job *InvalidationJobCreateV40) Validate(tx *sql.Tx) error { return nil } +// InvalidationJobV4 is an alias for the InvalidationJobV4 struct used for the latest minor version associated with api major version 4. +type InvalidationJobV4 InvalidationJobV40 + // InvalidationJobV40 represents a content invalidation job as returned by the API. // Also used for Update calls. type InvalidationJobV40 struct { @@ -551,7 +557,7 @@ type InvalidationJobV40 struct { StartTime *time.Time `json:"startTime"` } -func (job InvalidationJobV40) String() string { +func (job InvalidationJobV4) String() string { var ID, AssetURL, CreatedBy, DeliveryService, TTLHours, InvalidationType, StartTime string if job.ID != nil { ID = strconv.FormatUint(*job.ID, 10) @@ -589,7 +595,7 @@ func (job InvalidationJobV40) String() string { // Validate checks that the InvalidationJob is valid, by ensuring all of its fields are well-defined. // // This returns an error describing any and all problematic fields encountered during validation. -func (job *InvalidationJobV40) Validate() error { +func (job *InvalidationJobV4) Validate() error { errs := []string{} err := validation.ValidateStruct(job, validation.Field(&job.DeliveryService, validation.Required), @@ -626,7 +632,7 @@ func (job *InvalidationJobV40) Validate() error { } // validateDeliveryService ensures the supplied (required) Delivery Service XML ID exists -func (job *InvalidationJobCreateV40) validateDeliveryService(tx *sql.Tx) error { +func (job *InvalidationJobCreateV4) validateDeliveryService(tx *sql.Tx) error { var exists bool row := tx.QueryRow(`SELECT EXISTS(SELECT * FROM deliveryservice WHERE xml_id=$1)`, job.DeliveryService) if err := row.Scan(&exists); err != nil { @@ -639,7 +645,7 @@ func (job *InvalidationJobCreateV40) validateDeliveryService(tx *sql.Tx) error { } // validateTLLHours ensures the supplied TTL hours is within acceptable limits -func (job *InvalidationJobCreateV40) validateTLLHours(tx *sql.Tx) error { +func (job *InvalidationJobCreateV4) validateTLLHours(tx *sql.Tx) error { var maxDays uint err := tx.QueryRow(`SELECT value FROM parameter WHERE name='maxRevalDurationDays' AND config_file='regex_revalidate.config'`).Scan(&maxDays) maxHours := maxDays * 24 @@ -657,7 +663,7 @@ func RefetchAllowed(tx *sql.Tx) bool { return refetchEnabled } -func (job *InvalidationJobCreateV40) GetDSIDfromDSXMLID(tx *sql.Tx) (uint, error) { +func (job *InvalidationJobCreateV4) GetDSIDfromDSXMLID(tx *sql.Tx) (uint, error) { var dsID uint row := tx.QueryRow(`SELECT id FROM deliveryservice WHERE xml_id=$1`, job.DeliveryService) if err := row.Scan(&dsID); err != nil { diff --git a/traffic_ops/testing/api/v4/jobs_test.go b/traffic_ops/testing/api/v4/jobs_test.go index d4353289f4..dd6dda94dc 100644 --- a/traffic_ops/testing/api/v4/jobs_test.go +++ b/traffic_ops/testing/api/v4/jobs_test.go @@ -132,7 +132,7 @@ func JobCollisionWarningTest(t *testing.T) { t.Fatalf("unable to get invalidation jobs: %v - alerts: %+v", err, jobs.Alerts) } - var realJob *tc.InvalidationJobV40 + var realJob *tc.InvalidationJobV4 for i, job := range jobs.Response { if job.StartTime == nil || job.DeliveryService == nil || job.CreatedBy == nil { t.Error("Traffic Ops returned a representation of a content invalidation Job that had null or undefined Start Time and/or Delivery Service and/or Created By") @@ -941,7 +941,7 @@ func UpdateTestJobsInvalidDS(t *testing.T) { t.Fatalf("unable to get invalidation jobs: %v - alerts: %+v", err, jobs.Alerts) } - var realJob tc.InvalidationJobV40 + var realJob tc.InvalidationJobV4 for i, job := range jobs.Response { if job.StartTime == nil || job.DeliveryService == nil || job.CreatedBy == nil { t.Error("Traffic Ops returned a representation of a content invalidation Job that had null or undefined Start Time and/or Delivery Service and/or Created By") diff --git a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go index 0062925972..9b51604c15 100644 --- a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go +++ b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go @@ -48,9 +48,9 @@ type InvalidationJob struct { tc.InvalidationJob } -type InvalidationJobV40 struct { +type InvalidationJobV4 struct { api.APIInfoImpl `json:"-"` - tc.InvalidationJobV40 + tc.InvalidationJobV4 } // Deprecated, only to be used with versions below 4.0 @@ -92,7 +92,7 @@ RETURNING ` // Almost the same as insertQuery, but returns appropriate values for API 4.0+ -const insertQueryV40 = ` +const insertQueryV4 = ` INSERT INTO job ( ttl_hr, asset_url, @@ -177,7 +177,7 @@ RETURNING asset_url, ` // Almost the same as updateQuery, but returns appropriate values for API 4.0+ -const updateQueryV40 = ` +const updateQueryV4 = ` UPDATE job SET asset_url=$1, ttl_hr=$2, @@ -220,7 +220,7 @@ WHERE job.id=$1 ` // Almost the same as putInfoQuery, but returns appropriate values for API 4.0+ -const putInfoQueryV40 = ` +const putInfoQueryV4 = ` SELECT job.id AS id, tm_user.username AS createdBy, job.job_user AS createdByID, @@ -260,7 +260,7 @@ RETURNING job.asset_url, ` // Almost the same as deleteQuery, but returns appropriate values for API 4.0+ -const deleteQueryV40 = ` +const deleteQueryV4 = ` DELETE FROM job WHERE job.id=$1 @@ -287,9 +287,9 @@ type apiResponse struct { Response tc.InvalidationJob `json:"response,omitempty"` } -type apiResponseV40 struct { - Alerts []tc.Alert `json:"alerts,omitempty"` - Response tc.InvalidationJobV40 `json:"response,omitempty"` +type apiResponseV4 struct { + Alerts []tc.Alert `json:"alerts,omitempty"` + Response tc.InvalidationJobV4 `json:"response,omitempty"` } func selectMaxLastUpdatedQuery(where string) string { @@ -316,7 +316,7 @@ JOIN deliveryservice ds ON job.job_deliveryservice = ds.id ` // Almost the same as readQuery, but returns appropriate values for API 4.0+ -const readQueryV40 = ` +const readQueryV4 = ` SELECT job.id, asset_url, u.username as createdBy, @@ -331,7 +331,7 @@ JOIN deliveryservice ds ON job.job_deliveryservice = ds.id // Used by GET requests to `/jobs`, simply returns a filtered list of // content invalidation jobs according to the provided query parameters. -func (job *InvalidationJobV40) Read(h http.Header, useIMS bool) ([]interface{}, error, error, int, *time.Time) { +func (job *InvalidationJobV4) Read(h http.Header, useIMS bool) ([]interface{}, error, error, int, *time.Time) { var maxTime time.Time var runSecond bool queryParamsToSQLCols := map[string]dbhelpers.WhereColumnInfo{ @@ -372,7 +372,7 @@ func (job *InvalidationJobV40) Read(h http.Header, useIMS bool) ([]interface{}, log.Debugln("Non IMS request") } - query := readQueryV40 + where + orderBy + pagination + query := readQueryV4 + where + orderBy + pagination log.Debugln("generated job query: " + query) log.Debugf("executing with values: %++v\n", queryValues) @@ -384,7 +384,7 @@ func (job *InvalidationJobV40) Read(h http.Header, useIMS bool) ([]interface{}, defer rows.Close() for rows.Next() { - job := tc.InvalidationJobV40{} + job := tc.InvalidationJobV4{} err := rows.Scan(&job.ID, &job.AssetURL, &job.CreatedBy, @@ -511,7 +511,7 @@ func (job *InvalidationJob) Read(h http.Header, useIMS bool) ([]interface{}, err // Used by POST requests to `/jobs`, creates a new content invalidation job // from the provided request body. -func CreateV40(w http.ResponseWriter, r *http.Request) { +func CreateV4(w http.ResponseWriter, r *http.Request) { inf, userErr, sysErr, errCode := api.NewInfo(r, nil, nil) if userErr != nil || sysErr != nil { api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr) @@ -519,7 +519,7 @@ func CreateV40(w http.ResponseWriter, r *http.Request) { } defer inf.Close() - job := tc.InvalidationJobCreateV40{} + job := tc.InvalidationJobCreateV4{} if err := json.NewDecoder(r.Body).Decode(&job); err != nil { api.HandleErr(w, r, inf.Tx.Tx, http.StatusBadRequest, errors.New("Unable to parse Invalidation Job"), fmt.Errorf("parsing jobs/ POST: %v", err)) return @@ -539,12 +539,11 @@ func CreateV40(w http.ResponseWriter, r *http.Request) { resp, err := json.Marshal(response) if err != nil { - api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, fmt.Errorf("Encoding bad request response: %v", err)) + api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, fmt.Errorf("encoding bad request response: %v", err)) return } - w.WriteHeader(http.StatusBadRequest) - api.WriteAndLogErr(w, r, append(resp, '\n')) + api.HandleErr(w, r, inf.Tx.Tx, http.StatusBadRequest, errors.New(string(append(resp, '\n'))), fmt.Errorf("error validating job: %v", err)) return } @@ -569,7 +568,7 @@ func CreateV40(w http.ResponseWriter, r *http.Request) { api.HandleErr(w, r, inf.Tx.Tx, errCode, nil, sysErr) } - row := inf.Tx.Tx.QueryRow(insertQueryV40, + row := inf.Tx.Tx.QueryRow(insertQueryV4, job.TTLHours, dsid, // Used in inner select for deliveryservice job.Regex, @@ -579,7 +578,7 @@ func CreateV40(w http.ResponseWriter, r *http.Request) { dsid, job.InvalidationType) // Defaults for all api versions below 4.0 - result := tc.InvalidationJobV40{} + result := tc.InvalidationJobV4{} err = row.Scan( &result.ID, &result.AssetURL, @@ -600,7 +599,7 @@ func CreateV40(w http.ResponseWriter, r *http.Request) { } conflicts := tc.ValidateJobUniqueness(inf.Tx.Tx, dsid, *result.StartTime, *result.AssetURL, *result.TTLHours) - response := apiResponseV40{ + response := apiResponseV4{ make([]tc.Alert, len(conflicts)+1), result, } @@ -688,8 +687,7 @@ func Create(w http.ResponseWriter, r *http.Request) { return } - w.WriteHeader(http.StatusBadRequest) - api.WriteAndLogErr(w, r, append(resp, '\n')) + api.HandleErr(w, r, inf.Tx.Tx, http.StatusBadRequest, errors.New(string(append(resp, '\n'))), fmt.Errorf("error validating job: %v", err)) return } @@ -805,7 +803,7 @@ func Create(w http.ResponseWriter, r *http.Request) { // Used by PUT requests to `/jobs`, replaces an existing content invalidation job // with the provided request body. -func UpdateV40(w http.ResponseWriter, r *http.Request) { +func UpdateV4(w http.ResponseWriter, r *http.Request) { inf, userErr, sysErr, errCode := api.NewInfo(r, nil, nil) if userErr != nil || sysErr != nil { api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr) @@ -816,8 +814,8 @@ func UpdateV40(w http.ResponseWriter, r *http.Request) { var oFQDN string var dsid uint var uid uint - job := tc.InvalidationJobV40{} - row := inf.Tx.Tx.QueryRow(putInfoQueryV40, inf.Params["id"]) + job := tc.InvalidationJobV4{} + row := inf.Tx.Tx.QueryRow(putInfoQueryV4, inf.Params["id"]) err := row.Scan(&job.ID, &job.CreatedBy, &uid, @@ -864,7 +862,7 @@ func UpdateV40(w http.ResponseWriter, r *http.Request) { return } - input := tc.InvalidationJobV40{} + input := tc.InvalidationJobV4{} if err := json.NewDecoder(r.Body).Decode(&input); err != nil { userErr = fmt.Errorf("Unable to parse input: %v", err) sysErr = fmt.Errorf("parsing input to PUT jobs?id=%s: %v", inf.Params["id"], err) @@ -923,7 +921,7 @@ func UpdateV40(w http.ResponseWriter, r *http.Request) { } } - row = inf.Tx.Tx.QueryRow(updateQueryV40, + row = inf.Tx.Tx.QueryRow(updateQueryV4, input.AssetURL, input.TTLHours, input.StartTime, @@ -949,7 +947,7 @@ func UpdateV40(w http.ResponseWriter, r *http.Request) { } conflicts := tc.ValidateJobUniqueness(inf.Tx.Tx, dsid, *input.StartTime, *input.AssetURL, *input.TTLHours) - response := apiResponseV40{ + response := apiResponseV4{ make([]tc.Alert, len(conflicts)+1), job, } @@ -1173,7 +1171,7 @@ func Update(w http.ResponseWriter, r *http.Request) { } // Used by DELETE requests to `/jobs`, deletes an existing content invalidation job -func DeleteV40(w http.ResponseWriter, r *http.Request) { +func DeleteV4(w http.ResponseWriter, r *http.Request) { inf, userErr, sysErr, errCode := api.NewInfo(r, []string{"id"}, []string{"id"}) if userErr != nil || sysErr != nil { api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr) @@ -1220,8 +1218,8 @@ func DeleteV40(w http.ResponseWriter, r *http.Request) { return } - result := tc.InvalidationJobV40{} - row = inf.Tx.Tx.QueryRow(deleteQueryV40, inf.Params["id"]) + result := tc.InvalidationJobV4{} + row = inf.Tx.Tx.QueryRow(deleteQueryV4, inf.Params["id"]) err := row.Scan( &result.ID, &result.AssetURL, @@ -1244,7 +1242,7 @@ func DeleteV40(w http.ResponseWriter, r *http.Request) { return } - response := apiResponseV40{[]tc.Alert{ + response := apiResponseV4{[]tc.Alert{ {Text: "Content invalidation job was deleted", Level: tc.SuccessLevel.String()}, }, result, diff --git a/traffic_ops/traffic_ops_golang/routing/routes.go b/traffic_ops/traffic_ops_golang/routing/routes.go index 78676c1247..c6112c2471 100644 --- a/traffic_ops/traffic_ops_golang/routing/routes.go +++ b/traffic_ops/traffic_ops_golang/routing/routes.go @@ -216,7 +216,7 @@ func Routes(d ServerData) ([]Route, http.Handler, error) { {api.Version{Major: 4, Minor: 0}, http.MethodGet, `logs/newcount/?$`, logs.GetNewCount, auth.PrivLevelReadOnly, []string{"LOG:READ"}, Authenticated, nil, 44058330123}, //Content invalidation jobs - {api.Version{Major: 4, Minor: 0}, http.MethodGet, `jobs/?$`, api.ReadHandler(&invalidationjobs.InvalidationJobV40{}), auth.PrivLevelReadOnly, nil, Authenticated, nil, 49667820413}, + {api.Version{Major: 4, Minor: 0}, http.MethodGet, `jobs/?$`, api.ReadHandler(&invalidationjobs.InvalidationJobV4{}), auth.PrivLevelReadOnly, nil, Authenticated, nil, 49667820413}, {api.Version{Major: 4, Minor: 0}, http.MethodDelete, `jobs/?$`, invalidationjobs.DeleteV40, auth.PrivLevelPortal, nil, Authenticated, nil, 4167807763}, {api.Version{Major: 4, Minor: 0}, http.MethodPut, `jobs/?$`, invalidationjobs.UpdateV40, auth.PrivLevelPortal, nil, Authenticated, nil, 4861342263}, {api.Version{Major: 4, Minor: 0}, http.MethodPost, `jobs/?`, invalidationjobs.CreateV40, auth.PrivLevelPortal, nil, Authenticated, nil, 404509553}, diff --git a/traffic_ops/v4-client/job.go b/traffic_ops/v4-client/job.go index a2ba5bd66a..7f492cf95f 100644 --- a/traffic_ops/v4-client/job.go +++ b/traffic_ops/v4-client/job.go @@ -28,7 +28,7 @@ import ( const apiJobs = "/jobs" // CreateInvalidationJob creates the passed Content Invalidation Job. -func (to *Session) CreateInvalidationJob(job tc.InvalidationJobCreateV40, opts RequestOptions) (tc.Alerts, toclientlib.ReqInf, error) { +func (to *Session) CreateInvalidationJob(job tc.InvalidationJobCreateV4, opts RequestOptions) (tc.Alerts, toclientlib.ReqInf, error) { var alerts tc.Alerts reqInf, err := to.post(apiJobs, opts, job, &alerts) return alerts, reqInf, err @@ -48,7 +48,7 @@ func (to *Session) DeleteInvalidationJob(jobID uint64, opts RequestOptions) (tc. // UpdateInvalidationJob updates the passed Content Invalidation Job (it is // expected to have an ID). -func (to *Session) UpdateInvalidationJob(job tc.InvalidationJobV40, opts RequestOptions) (tc.Alerts, toclientlib.ReqInf, error) { +func (to *Session) UpdateInvalidationJob(job tc.InvalidationJobV4, opts RequestOptions) (tc.Alerts, toclientlib.ReqInf, error) { var alerts tc.Alerts if job.ID == nil { return alerts, toclientlib.ReqInf{}, errors.New("job has a nil ID") @@ -63,8 +63,8 @@ func (to *Session) UpdateInvalidationJob(job tc.InvalidationJobV40, opts Request // GetInvalidationJobs returns a list of Content Invalidation Jobs visible to // your Tenant. -func (to *Session) GetInvalidationJobs(opts RequestOptions) (tc.InvalidationJobsResponseV40, toclientlib.ReqInf, error) { - var data tc.InvalidationJobsResponseV40 +func (to *Session) GetInvalidationJobs(opts RequestOptions) (tc.InvalidationJobsResponseV4, toclientlib.ReqInf, error) { + var data tc.InvalidationJobsResponseV4 reqInf, err := to.get(apiJobs, opts, &data) return data, reqInf, err } From 901ffc992891bb98808b9cce140d165789f828ed Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Fri, 8 Oct 2021 13:16:29 -0600 Subject: [PATCH 27/35] Address feedback from review. Use concrete values instead of pointers in DB call. Use explicit ignore for returned error in db call. Keep routes major+minor aware. --- lib/go-atscfg/regexrevalidatedotconfig.go | 8 ++-- lib/go-tc/invalidationjobs.go | 16 ++++---- lib/go-util/ptr.go | 8 ++-- traffic_ops/testing/api/v4/jobs_test.go | 40 +++++++++---------- .../testing/api/v4/traffic_control_test.go | 2 +- .../invalidationjobs/invalidationjobs.go | 6 +-- 6 files changed, 39 insertions(+), 41 deletions(-) diff --git a/lib/go-atscfg/regexrevalidatedotconfig.go b/lib/go-atscfg/regexrevalidatedotconfig.go index dec2c25738..abe74cd0ef 100644 --- a/lib/go-atscfg/regexrevalidatedotconfig.go +++ b/lib/go-atscfg/regexrevalidatedotconfig.go @@ -183,9 +183,7 @@ func filterJobs(tcJobs []tc.InvalidationJobV4, maxReval time.Duration, minTTL ti continue } - // process the __REFETCH__ keyword - // There is no InvalidationType string before TO v4.0, so pass empty string - jobType, assetURL := ProcessRefetch(*tcJob.InvalidationType, *tcJob.AssetURL) + jobType, assetURL := processRefetch(*tcJob.InvalidationType, *tcJob.AssetURL) purgeEnd := tcJob.StartTime.Add(ttl) @@ -206,10 +204,10 @@ func filterJobs(tcJobs []tc.InvalidationJobV4, maxReval time.Duration, minTTL ti const MISS = "MISS" const STALE = "STALE" -// ProcessRefetch determines the type of Invalidation, returns the corresponding jobtype +// processRefetch determines the type of Invalidation, returns the corresponding jobtype // and "cleans" the regex URL for the asset to be invalidated. REFETCH trumps REFRESH, // whether in the AssetURL or as InvalidationType -func ProcessRefetch(invalidationType, assetURL string) (string, string) { +func processRefetch(invalidationType, assetURL string) (string, string) { if (len(invalidationType) > 0 && invalidationType == tc.REFETCH) || strings.HasSuffix(assetURL, RefetchSuffix) { assetURL = strings.TrimSuffix(assetURL, RefetchSuffix) diff --git a/lib/go-tc/invalidationjobs.go b/lib/go-tc/invalidationjobs.go index b4bc668885..ff04d90a62 100644 --- a/lib/go-tc/invalidationjobs.go +++ b/lib/go-tc/invalidationjobs.go @@ -247,9 +247,9 @@ func (job *InvalidationJobInput) Validate(tx *sql.Tx) error { } type compareJob struct { - AssetURL *string - TTLHours *uint - StartTime *time.Time + AssetURL string + TTLHours uint + StartTime time.Time } // ValidateJobUniqueness returns a message describing each overlap between @@ -282,14 +282,14 @@ WHERE job.job_deliveryservice = $1 if err != nil { continue } - if !strings.HasSuffix(*testJob.AssetURL, assetURL) { + if !strings.HasSuffix(testJob.AssetURL, assetURL) { continue } - if *testJob.TTLHours == 0 { + if testJob.TTLHours == 0 { continue } testJobStart := testJob.StartTime - testJobEnd := testJobStart.Add(time.Hour * time.Duration(*testJob.TTLHours)) + testJobEnd := testJobStart.Add(time.Hour * time.Duration(testJob.TTLHours)) jobEnd := jobStart.Add(time.Hour * time.Duration(ttlHours)) // jobStart in testJob range if (testJobStart.Before(jobStart) && jobStart.Before(testJobEnd)) || @@ -298,7 +298,7 @@ WHERE job.job_deliveryservice = $1 // job range encaspulates testJob range (testJobEnd.Before(jobEnd) && jobStart.Before(jobStart)) { errs = append(errs, fmt.Sprintf("Invalidation request duplicate found for %v, start:%v end:%v", - *testJob.AssetURL, testJobStart, testJobEnd)) + testJob.AssetURL, testJobStart, testJobEnd)) } } } @@ -659,7 +659,7 @@ func (job *InvalidationJobCreateV4) validateTLLHours(tx *sql.Tx) error { func RefetchAllowed(tx *sql.Tx) bool { refetchEnabled := false // Silently ignore paramter error - tx.QueryRow(`SELECT value FROM parameter WHERE name='refetch_enabled' AND config_file='global'`).Scan(&refetchEnabled) + _ = tx.QueryRow(`SELECT value FROM parameter WHERE name='refetch_enabled' AND config_file='global'`).Scan(&refetchEnabled) return refetchEnabled } diff --git a/lib/go-util/ptr.go b/lib/go-util/ptr.go index 94e2b79183..e25ecbc85b 100644 --- a/lib/go-util/ptr.go +++ b/lib/go-util/ptr.go @@ -1,7 +1,3 @@ -package util - -import "time" - /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -21,6 +17,10 @@ import "time" * under the License. */ +package util + +import "time" + func StrPtr(str string) *string { return &str } diff --git a/traffic_ops/testing/api/v4/jobs_test.go b/traffic_ops/testing/api/v4/jobs_test.go index dd6dda94dc..8b49e00464 100644 --- a/traffic_ops/testing/api/v4/jobs_test.go +++ b/traffic_ops/testing/api/v4/jobs_test.go @@ -62,7 +62,7 @@ func CreateTestJobs(t *testing.T) { } for _, job := range testData.InvalidationJobs { - request := tc.InvalidationJobCreateV40{ + request := tc.InvalidationJobCreateV4{ DeliveryService: job.DeliveryService, Regex: job.Regex, StartTime: job.StartTime, @@ -85,7 +85,7 @@ func JobCollisionWarningTest(t *testing.T) { } xmlID := *testData.DeliveryServices[0].XMLID - firstJob := tc.InvalidationJobCreateV40{ + firstJob := tc.InvalidationJobCreateV4{ DeliveryService: xmlID, Regex: `/\.*([A-Z]0?)`, TTLHours: 16, @@ -98,7 +98,7 @@ func JobCollisionWarningTest(t *testing.T) { t.Fatalf("Unexpected error creating a content invalidation Job: %v - alerts: %+v", err, resp.Alerts) } - newJob := tc.InvalidationJobCreateV40{ + newJob := tc.InvalidationJobCreateV4{ DeliveryService: firstJob.DeliveryService, Regex: firstJob.Regex, TTLHours: firstJob.TTLHours, @@ -586,7 +586,7 @@ func CreateTestJobsInvalidDS(t *testing.T) { testData.InvalidationJobs[0] = job //Invalid DS XML ID (Does not exist) - request := tc.InvalidationJobCreateV40{ + request := tc.InvalidationJobCreateV4{ DeliveryService: "invalid", Regex: job.Regex, StartTime: job.StartTime, @@ -602,7 +602,7 @@ func CreateTestJobsInvalidDS(t *testing.T) { } //Missing DS - request = tc.InvalidationJobCreateV40{ + request = tc.InvalidationJobCreateV4{ Regex: job.Regex, StartTime: job.StartTime, TTLHours: job.TTLHours, @@ -617,7 +617,7 @@ func CreateTestJobsInvalidDS(t *testing.T) { } //Empty DS - request = tc.InvalidationJobCreateV40{ + request = tc.InvalidationJobCreateV4{ DeliveryService: "", Regex: job.Regex, StartTime: job.StartTime, @@ -641,7 +641,7 @@ func CreateTestJobsAlreadyExistTTL(t *testing.T) { job.StartTime = time.Now().Add(time.Minute).UTC() testData.InvalidationJobs[0] = job - request := tc.InvalidationJobCreateV40{ + request := tc.InvalidationJobCreateV4{ DeliveryService: job.DeliveryService, Regex: job.Regex, StartTime: job.StartTime, @@ -664,7 +664,7 @@ func CreateTestJobsWithPastDate(t *testing.T) { job := testData.InvalidationJobs[0] job.StartTime = dt.AddDate(0, 0, -1) testData.InvalidationJobs[0] = job - request := tc.InvalidationJobCreateV40{ + request := tc.InvalidationJobCreateV4{ DeliveryService: job.DeliveryService, Regex: job.Regex, StartTime: job.StartTime, @@ -685,7 +685,7 @@ func CreateTestJobsWithPastDate(t *testing.T) { job = testData.InvalidationJobs[0] job.StartTime = dt.AddDate(0, 0, -1) testData.InvalidationJobs[0] = job - request = tc.InvalidationJobCreateV40{ + request = tc.InvalidationJobCreateV4{ DeliveryService: job.DeliveryService, Regex: job.Regex, StartTime: job.StartTime, @@ -706,7 +706,7 @@ func CreateTestJobsWithPastDate(t *testing.T) { job = testData.InvalidationJobs[0] job.StartTime = dt.AddDate(0, 0, -5) testData.InvalidationJobs[0] = job - request = tc.InvalidationJobCreateV40{ + request = tc.InvalidationJobCreateV4{ DeliveryService: job.DeliveryService, Regex: job.Regex, StartTime: job.StartTime, @@ -725,7 +725,7 @@ func CreateTestJobsWithPastDate(t *testing.T) { job = testData.InvalidationJobs[0] job.StartTime = time.Unix(1, 0) testData.InvalidationJobs[0] = job - request = tc.InvalidationJobCreateV40{ + request = tc.InvalidationJobCreateV4{ DeliveryService: job.DeliveryService, Regex: job.Regex, StartTime: job.StartTime, @@ -751,7 +751,7 @@ func CreateTestJobsWithFutureDate(t *testing.T) { job := testData.InvalidationJobs[0] job.StartTime = dt.AddDate(0, 0, 1) testData.InvalidationJobs[0] = job - request := tc.InvalidationJobCreateV40{ + request := tc.InvalidationJobCreateV4{ DeliveryService: job.DeliveryService, Regex: job.Regex, StartTime: job.StartTime, @@ -772,7 +772,7 @@ func CreateTestJobsWithFutureDate(t *testing.T) { job = testData.InvalidationJobs[0] job.StartTime = dt.AddDate(0, 0, 1) testData.InvalidationJobs[0] = job - request = tc.InvalidationJobCreateV40{ + request = tc.InvalidationJobCreateV4{ DeliveryService: job.DeliveryService, Regex: job.Regex, StartTime: job.StartTime, @@ -793,7 +793,7 @@ func CreateTestJobsWithFutureDate(t *testing.T) { job = testData.InvalidationJobs[0] job.StartTime = dt.AddDate(0, 0, 1) testData.InvalidationJobs[0] = job - request = tc.InvalidationJobCreateV40{ + request = tc.InvalidationJobCreateV4{ DeliveryService: job.DeliveryService, Regex: job.Regex, StartTime: job.StartTime, @@ -815,7 +815,7 @@ func CreateJobsMissingDate(t *testing.T) { } //Missing start date job := testData.InvalidationJobs[0] - request := tc.InvalidationJobCreateV40{ + request := tc.InvalidationJobCreateV4{ DeliveryService: job.DeliveryService, Regex: job.Regex, TTLHours: job.TTLHours, @@ -841,7 +841,7 @@ func CreateJobsMissingRegex(t *testing.T) { job := testData.InvalidationJobs[0] job.StartTime = dt.AddDate(0, 0, 1) testData.InvalidationJobs[0] = job - request := tc.InvalidationJobCreateV40{ + request := tc.InvalidationJobCreateV4{ DeliveryService: job.DeliveryService, TTLHours: job.TTLHours, InvalidationType: job.InvalidationType, @@ -856,7 +856,7 @@ func CreateJobsMissingRegex(t *testing.T) { //Empty Regex job.Regex = "" - request = tc.InvalidationJobCreateV40{ + request = tc.InvalidationJobCreateV4{ DeliveryService: job.DeliveryService, Regex: job.Regex, TTLHours: job.TTLHours, @@ -882,7 +882,7 @@ func CreateJobsMissingTtl(t *testing.T) { job := testData.InvalidationJobs[0] job.StartTime = dt.AddDate(0, 0, 1) testData.InvalidationJobs[0] = job - request := tc.InvalidationJobCreateV40{ + request := tc.InvalidationJobCreateV4{ DeliveryService: job.DeliveryService, Regex: job.Regex, InvalidationType: job.InvalidationType, @@ -897,7 +897,7 @@ func CreateJobsMissingTtl(t *testing.T) { //Invalid TTL job.TTLHours = 0 - request = tc.InvalidationJobCreateV40{ + request = tc.InvalidationJobCreateV4{ DeliveryService: job.DeliveryService, Regex: job.Regex, TTLHours: job.TTLHours, @@ -922,7 +922,7 @@ func UpdateTestJobsInvalidDS(t *testing.T) { } xmlID := *testData.DeliveryServices[0].XMLID startTime := time.Now().Add(time.Hour) - firstJob := tc.InvalidationJobCreateV40{ + firstJob := tc.InvalidationJobCreateV4{ DeliveryService: xmlID, Regex: `/\.*([A-Z]0?)`, TTLHours: 16, diff --git a/traffic_ops/testing/api/v4/traffic_control_test.go b/traffic_ops/testing/api/v4/traffic_control_test.go index 20052c9879..783f7751dc 100644 --- a/traffic_ops/testing/api/v4/traffic_control_test.go +++ b/traffic_ops/testing/api/v4/traffic_control_test.go @@ -56,5 +56,5 @@ type TrafficControl struct { SteeringTargets []tc.SteeringTargetNullable `json:"steeringTargets"` Serverchecks []tc.ServercheckRequestNullable `json:"serverchecks"` Users []tc.UserV4 `json:"users"` - InvalidationJobs []tc.InvalidationJobCreateV40 `json:"invalidationJobs"` + InvalidationJobs []tc.InvalidationJobCreateV4 `json:"invalidationJobs"` } diff --git a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go index 9b51604c15..5d5c69a2a1 100644 --- a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go +++ b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go @@ -511,7 +511,7 @@ func (job *InvalidationJob) Read(h http.Header, useIMS bool) ([]interface{}, err // Used by POST requests to `/jobs`, creates a new content invalidation job // from the provided request body. -func CreateV4(w http.ResponseWriter, r *http.Request) { +func CreateV40(w http.ResponseWriter, r *http.Request) { inf, userErr, sysErr, errCode := api.NewInfo(r, nil, nil) if userErr != nil || sysErr != nil { api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr) @@ -803,7 +803,7 @@ func Create(w http.ResponseWriter, r *http.Request) { // Used by PUT requests to `/jobs`, replaces an existing content invalidation job // with the provided request body. -func UpdateV4(w http.ResponseWriter, r *http.Request) { +func UpdateV40(w http.ResponseWriter, r *http.Request) { inf, userErr, sysErr, errCode := api.NewInfo(r, nil, nil) if userErr != nil || sysErr != nil { api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr) @@ -1171,7 +1171,7 @@ func Update(w http.ResponseWriter, r *http.Request) { } // Used by DELETE requests to `/jobs`, deletes an existing content invalidation job -func DeleteV4(w http.ResponseWriter, r *http.Request) { +func DeleteV40(w http.ResponseWriter, r *http.Request) { inf, userErr, sysErr, errCode := api.NewInfo(r, []string{"id"}, []string{"id"}) if userErr != nil || sysErr != nil { api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr) From 483a834aabcd1c0d334862bb158baa7c7ce57d3f Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Fri, 8 Oct 2021 16:38:39 -0600 Subject: [PATCH 28/35] Address SQL string literal formatting issues. Fix Major V4 naming --- lib/go-atscfg/atscfg.go | 4 +- .../invalidationjobs/invalidationjobs.go | 200 +++++++++--------- 2 files changed, 104 insertions(+), 100 deletions(-) diff --git a/lib/go-atscfg/atscfg.go b/lib/go-atscfg/atscfg.go index f25200de09..a0440fc7ce 100644 --- a/lib/go-atscfg/atscfg.go +++ b/lib/go-atscfg/atscfg.go @@ -701,7 +701,7 @@ func JobsToInvalidationJobsV4(oldJobs []tc.Job) ([]tc.InvalidationJobV4, error) jobs := make([]tc.InvalidationJobV4, len(oldJobs), len(oldJobs)) err := error(nil) for i, oldJob := range oldJobs { - jobs[i], err = JobToInvalidationJobV40(oldJob) + jobs[i], err = JobToInvalidationJobV4(oldJob) if err != nil { return nil, errors.New("converting old tc.Job to tc.InvalidationJob: " + err.Error()) } @@ -709,7 +709,7 @@ func JobsToInvalidationJobsV4(oldJobs []tc.Job) ([]tc.InvalidationJobV4, error) return jobs, nil } -func JobToInvalidationJobV40(jb tc.Job) (tc.InvalidationJobV4, error) { +func JobToInvalidationJobV4(jb tc.Job) (tc.InvalidationJobV4, error) { startTime := tc.Time{} if err := json.Unmarshal([]byte(`"`+jb.StartTime+`"`), &startTime); err != nil { return tc.InvalidationJobV4{}, errors.New("unmarshalling time: " + err.Error()) diff --git a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go index 5d5c69a2a1..0581bb68ad 100644 --- a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go +++ b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go @@ -79,13 +79,15 @@ VALUES ( ) RETURNING asset_url, - (SELECT deliveryservice.xml_id - FROM deliveryservice - WHERE deliveryservice.id=job_deliveryservice) AS deliveryservice, + ( + SELECT deliveryservice.xml_id + FROM deliveryservice + WHERE deliveryservice.id=job_deliveryservice) AS deliveryservice, id, - (SELECT tm_user.username - FROM tm_user - WHERE tm_user.id=job_user) AS createdBy, + ( + SELECT tm_user.username + FROM tm_user + WHERE tm_user.id=job_user) AS createdBy, 'PURGE' AS keyword, CONCAT('TTL:', ttl_hr, 'h') AS parameters, start_time @@ -118,12 +120,14 @@ VALUES ( RETURNING id, asset_url, - (SELECT tm_user.username + ( + SELECT tm_user.username FROM tm_user WHERE tm_user.id=job_user) AS createdBy, - (SELECT deliveryservice.xml_id - FROM deliveryservice - WHERE deliveryservice.id=job_deliveryservice) AS deliveryServiceXML, + ( + SELECT deliveryservice.xml_id + FROM deliveryservice + WHERE deliveryservice.id=job_deliveryservice) AS deliveryServiceXML, ttl_hr as ttlHrs, invalidation_type as invalidationType, start_time as startTime @@ -160,20 +164,20 @@ SET asset_url=$1, start_time=$3 WHERE job.id=$4 RETURNING asset_url, - ( - SELECT tm_user.username - FROM tm_user - WHERE tm_user.id=job.job_user - ) AS created_by, - ( - SELECT deliveryservice.xml_id - FROM deliveryservice - WHERE deliveryservice.id=job.job_deliveryservice - ) AS delivery_service, - job.id, - 'PURGE' as keyword, - CONCAT('TTL:', ttl_hr, 'h') AS parameters, - start_time + ( + SELECT tm_user.username + FROM tm_user + WHERE tm_user.id=job.job_user + ) AS created_by, + ( + SELECT deliveryservice.xml_id + FROM deliveryservice + WHERE deliveryservice.id=job.job_deliveryservice + ) AS delivery_service, + job.id, + 'PURGE' as keyword, + CONCAT('TTL:', ttl_hr, 'h') AS parameters, + start_time ` // Almost the same as updateQuery, but returns appropriate values for API 4.0+ @@ -185,33 +189,33 @@ SET asset_url=$1, invalidation_type=$4 WHERE job.id=$5 RETURNING asset_url, - ( - SELECT tm_user.username - FROM tm_user - WHERE tm_user.id=job.job_user - ) AS created_by, - ( - SELECT deliveryservice.xml_id - FROM deliveryservice - WHERE deliveryservice.id=job.job_deliveryservice - ) AS delivery_service, - job.id, - ttl_hr, - start_time, - invalidation_type + ( + SELECT tm_user.username + FROM tm_user + WHERE tm_user.id=job.job_user + ) AS created_by, + ( + SELECT deliveryservice.xml_id + FROM deliveryservice + WHERE deliveryservice.id=job.job_deliveryservice + ) AS delivery_service, + job.id, + ttl_hr, + start_time, + invalidation_type ` // Deprecated, only to be used with versions below 4.0 const putInfoQuery = ` SELECT job.id AS id, - tm_user.username AS createdBy, - job.job_user AS createdByID, - job.job_deliveryservice AS dsid, - deliveryservice.xml_id AS dsxmlid, - job.asset_url AS assetURL, - CONCAT('TTL:', ttl_hr, 'h') AS parameters, - job.start_time AS start_time, - origin.protocol || '://' || origin.fqdn || rtrim(concat(':', origin.port), ':') AS OFQDN + tm_user.username AS createdBy, + job.job_user AS createdByID, + job.job_deliveryservice AS dsid, + deliveryservice.xml_id AS dsxmlid, + job.asset_url AS assetURL, + CONCAT('TTL:', ttl_hr, 'h') AS parameters, + job.start_time AS start_time, + origin.protocol || '://' || origin.fqdn || rtrim(concat(':', origin.port), ':') AS OFQDN FROM job INNER JOIN origin ON origin.deliveryservice=job.job_deliveryservice AND origin.is_primary INNER JOIN tm_user ON tm_user.id=job.job_user @@ -222,15 +226,15 @@ WHERE job.id=$1 // Almost the same as putInfoQuery, but returns appropriate values for API 4.0+ const putInfoQueryV4 = ` SELECT job.id AS id, - tm_user.username AS createdBy, - job.job_user AS createdByID, - job.job_deliveryservice AS dsid, - deliveryservice.xml_id AS dsxmlid, - job.asset_url AS assetURL, - job.ttl_hr AS ttlhrs, - job.start_time AS start_time, - job.invalidation_type as invalidationType, - origin.protocol || '://' || origin.fqdn || rtrim(concat(':', origin.port), ':') AS OFQDN + tm_user.username AS createdBy, + job.job_user AS createdByID, + job.job_deliveryservice AS dsid, + deliveryservice.xml_id AS dsxmlid, + job.asset_url AS assetURL, + job.ttl_hr AS ttlhrs, + job.start_time AS start_time, + job.invalidation_type as invalidationType, + origin.protocol || '://' || origin.fqdn || rtrim(concat(':', origin.port), ':') AS OFQDN FROM job INNER JOIN origin ON origin.deliveryservice=job.job_deliveryservice AND origin.is_primary INNER JOIN tm_user ON tm_user.id=job.job_user @@ -244,19 +248,20 @@ DELETE FROM job WHERE job.id=$1 RETURNING job.asset_url, - ( SELECT tm_user.username - FROM tm_user - WHERE tm_user.id=job.job_user - ) AS created_by, - ( - SELECT deliveryservice.xml_id - FROM deliveryservice - WHERE deliveryservice.id=job.job_deliveryservice - ) AS deliveryservice, - job.id, - 'PURGE' as keyword, - CONCAT('TTL:', ttl_hr, 'h') AS parameters, - job.start_time + ( + SELECT tm_user.username + FROM tm_user + WHERE tm_user.id=job.job_user + ) AS created_by, + ( + SELECT deliveryservice.xml_id + FROM deliveryservice + WHERE deliveryservice.id=job.job_deliveryservice + ) AS deliveryservice, + job.id, + 'PURGE' as keyword, + CONCAT('TTL:', ttl_hr, 'h') AS parameters, + job.start_time ` // Almost the same as deleteQuery, but returns appropriate values for API 4.0+ @@ -265,21 +270,21 @@ DELETE FROM job WHERE job.id=$1 RETURNING - job.id, - job.asset_url, - ( - SELECT tm_user.username - FROM tm_user - WHERE tm_user.id=job.job_user - ) AS created_by, - ( - SELECT deliveryservice.xml_id - FROM deliveryservice - WHERE deliveryservice.id=job.job_deliveryservice - ) AS deliveryservice, - ttl_hr, - job.invalidation_type, - job.start_time + job.id, + job.asset_url, + ( + SELECT tm_user.username + FROM tm_user + WHERE tm_user.id=job.job_user + ) AS created_by, + ( + SELECT deliveryservice.xml_id + FROM deliveryservice + WHERE deliveryservice.id=job.job_deliveryservice + ) AS deliveryservice, + ttl_hr, + job.invalidation_type, + job.start_time ` type apiResponse struct { @@ -304,12 +309,12 @@ func selectMaxLastUpdatedQuery(where string) string { // Deprecated, only to be used with versions below 4.0 const readQuery = ` SELECT job.id, - 'PURGE' AS keyword, - ttl_hr, - asset_url, - start_time, - u.username as createdBy, - ds.xml_id as dsId + 'PURGE' AS keyword, + ttl_hr, + asset_url, + start_time, + u.username as createdBy, + ds.xml_id as dsId FROM job JOIN tm_user u ON job.job_user = u.id JOIN deliveryservice ds ON job.job_deliveryservice = ds.id @@ -318,12 +323,12 @@ JOIN deliveryservice ds ON job.job_deliveryservice = ds.id // Almost the same as readQuery, but returns appropriate values for API 4.0+ const readQueryV4 = ` SELECT job.id, - asset_url, - u.username as createdBy, - ds.xml_id, - ttl_hr, - invalidation_type, - start_time + asset_url, + u.username as createdBy, + ds.xml_id, + ttl_hr, + invalidation_type, + start_time FROM job JOIN tm_user u ON job.job_user = u.id JOIN deliveryservice ds ON job.job_deliveryservice = ds.id @@ -385,14 +390,13 @@ func (job *InvalidationJobV4) Read(h http.Header, useIMS bool) ([]interface{}, e for rows.Next() { job := tc.InvalidationJobV4{} - err := rows.Scan(&job.ID, + if err := rows.Scan(&job.ID, &job.AssetURL, &job.CreatedBy, &job.DeliveryService, &job.TTLHours, &job.InvalidationType, - &job.StartTime) - if err != nil { + &job.StartTime); err != nil { return nil, nil, fmt.Errorf("parsing db response: %v", err), http.StatusInternalServerError, nil } From fdd65e8eca08ac7b1c9ae7c68fbe6130fd4b4f35 Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Mon, 11 Oct 2021 14:34:50 -0600 Subject: [PATCH 29/35] Fix alignment with SQL string literals. Ensure additional filtering for InvalidationJob Read is added --- .../2021081321025500_refetch.down.sql | 4 ++- .../invalidationjobs/invalidationjobs.go | 28 +++++++++++++++---- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/traffic_ops/app/db/migrations/2021081321025500_refetch.down.sql b/traffic_ops/app/db/migrations/2021081321025500_refetch.down.sql index f39fc30d83..452b1bab62 100644 --- a/traffic_ops/app/db/migrations/2021081321025500_refetch.down.sql +++ b/traffic_ops/app/db/migrations/2021081321025500_refetch.down.sql @@ -121,7 +121,9 @@ UPDATE public.job SET agent = 1; UPDATE public.job -SET status = 1; +SET status = ( + SELECT id FROM public.job_status WHERE name = 'PENDING' +); ALTER TABLE public.job ALTER COLUMN status SET NOT NULL; diff --git a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go index 0581bb68ad..915080be55 100644 --- a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go +++ b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go @@ -184,8 +184,8 @@ RETURNING asset_url, const updateQueryV4 = ` UPDATE job SET asset_url=$1, - ttl_hr=$2, - start_time=$3, + ttl_hr=$2, + start_time=$3, invalidation_type=$4 WHERE job.id=$5 RETURNING asset_url, @@ -258,7 +258,7 @@ RETURNING job.asset_url, FROM deliveryservice WHERE deliveryservice.id=job.job_deliveryservice ) AS deliveryservice, - job.id, + job.id, 'PURGE' as keyword, CONCAT('TTL:', ttl_hr, 'h') AS parameters, job.start_time @@ -359,10 +359,28 @@ func (job *InvalidationJobV4) Read(h http.Header, useIMS bool) ([]interface{}, e if err != nil { return nil, nil, fmt.Errorf("getting accessible tenants for user - %v", err), http.StatusInternalServerError, nil } + cdn := "" + if cdnName, ok := job.APIInfo().Params["cdn"]; ok { + queryValues["cdn"] = cdnName + cdn = ` AND ds.cdn_id = (SELECT id FROM cdn WHERE name = :cdn) ` + } + maxDays := "" + if _, ok := job.APIInfo().Params["maxRevalDurationDays"]; ok { + // jobs started within the last $maxRevalDurationDays days (defaulting to 90 days if the parameter doesn't exist) + maxDays = ` AND job.start_time >= NOW() - CAST( + (SELECT COALESCE( + (SELECT value + FROM parameter + WHERE name = 'maxRevalDurationDays' + AND config_file = 'regex_revalidate.config' + LIMIT 1), + '90')) + || ' days' AS INTERVAL) ` + } if len(where) > 0 { - where += " AND ds.tenant_id = ANY(:tenants) " + where += " AND ds.tenant_id = ANY(:tenants) " + maxDays + cdn } else { - where = dbhelpers.BaseWhere + " ds.tenant_id = ANY(:tenants) " + where = dbhelpers.BaseWhere + " ds.tenant_id = ANY(:tenants) " + maxDays + cdn } queryValues["tenants"] = pq.Array(accessibleTenants) From d0fe631dc0d2e58bc92aabaa80e2e19625bee8f4 Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Tue, 12 Oct 2021 16:58:04 -0600 Subject: [PATCH 30/35] Refactor business logic from shared library go-tc library into TO golang. Remove references to Perl in README.md test file Move common database call(s) to dbhelper Change pointer fields to concrete values --- .../t3c-generate/cfgfile/cfgfile_test.go | 14 +- lib/go-atscfg/atscfg.go | 21 +- lib/go-atscfg/regexrevalidatedotconfig.go | 35 +-- .../regexrevalidatedotconfig_test.go | 84 +++---- lib/go-tc/invalidationjobs.go | 180 ++------------- traffic_ops/testing/api/README.md | 23 +- traffic_ops/testing/api/v4/jobs_test.go | 112 +++++----- traffic_ops/testing/api/v4/tc-fixtures.json | 28 +++ .../dbhelpers/db_helpers.go | 12 + .../invalidationjobs/invalidationjobs.go | 207 ++++++++++++++---- traffic_ops/v4-client/job.go | 6 +- 11 files changed, 347 insertions(+), 375 deletions(-) diff --git a/cache-config/t3c-generate/cfgfile/cfgfile_test.go b/cache-config/t3c-generate/cfgfile/cfgfile_test.go index 867e7f7f38..05d8012085 100644 --- a/cache-config/t3c-generate/cfgfile/cfgfile_test.go +++ b/cache-config/t3c-generate/cfgfile/cfgfile_test.go @@ -359,13 +359,13 @@ func randParam() *tc.Parameter { func randJob() *tc.InvalidationJobV4 { now := time.Now() return &tc.InvalidationJobV4{ - AssetURL: randStr(), - CreatedBy: randStr(), - StartTime: &now, - ID: randUint64(), - DeliveryService: randStr(), - TTLHours: randUint(), - InvalidationType: util.StrPtr(tc.REFRESH), + AssetURL: *randStr(), + CreatedBy: *randStr(), + StartTime: now, + ID: *randUint64(), + DeliveryService: *randStr(), + TTLHours: *randUint(), + InvalidationType: tc.REFRESH, } } diff --git a/lib/go-atscfg/atscfg.go b/lib/go-atscfg/atscfg.go index a0440fc7ce..fc1c8d9efa 100644 --- a/lib/go-atscfg/atscfg.go +++ b/lib/go-atscfg/atscfg.go @@ -22,7 +22,6 @@ package atscfg import ( "encoding/json" "errors" - "fmt" "net" "sort" "strconv" @@ -655,12 +654,6 @@ func makeErr(warnings []string, err string) error { return errors.New(`(warnings: ` + strings.Join(warnings, `, `) + `) ` + err) } -// makeErrf is a convenience for formatting errors for makeErr. -// todo also unused, maybe remove? -func makeErrf(warnings []string, format string, v ...interface{}) error { - return makeErr(warnings, fmt.Sprintf(format, v...)) -} - // DeliveryServiceServer is a compact version of DeliveryServiceServer. // The Delivery Service Servers is massive on large CDNs not using Topologies, compacting it in JSON and dropping the timestamp drastically reduces the size. // The t3c apps will also drop any DSS from Traffic Ops with null values, which are invalid and useless, to avoid pointers and further reduce size. @@ -719,12 +712,12 @@ func JobToInvalidationJobV4(jb tc.Job) (tc.InvalidationJobV4, error) { return tc.InvalidationJobV4{}, errors.New("unmarshalling ttl: " + err.Error()) } return tc.InvalidationJobV4{ - AssetURL: util.StrPtr(jb.AssetURL), - CreatedBy: util.StrPtr(jb.CreatedBy), - DeliveryService: util.StrPtr(jb.DeliveryService), - ID: util.Uint64Ptr(uint64(jb.ID)), - TTLHours: util.UIntPtr(uint(ttl)), - InvalidationType: util.StrPtr(tc.REFRESH), - StartTime: &startTime.Time, + AssetURL: jb.AssetURL, + CreatedBy: jb.CreatedBy, + DeliveryService: jb.DeliveryService, + ID: uint64(jb.ID), + TTLHours: uint(ttl), + InvalidationType: tc.REFRESH, + StartTime: startTime.Time, }, nil } diff --git a/lib/go-atscfg/regexrevalidatedotconfig.go b/lib/go-atscfg/regexrevalidatedotconfig.go index abe74cd0ef..5639e910a3 100644 --- a/lib/go-atscfg/regexrevalidatedotconfig.go +++ b/lib/go-atscfg/regexrevalidatedotconfig.go @@ -20,7 +20,6 @@ package atscfg */ import ( - "fmt" "sort" "strconv" "strings" @@ -82,11 +81,11 @@ func MakeRegexRevalidateDotConfig( dsJobs := []tc.InvalidationJobV4{} for _, job := range jobs { - if job.DeliveryService == nil { - warnings = append(warnings, "got job from Traffic Ops with a nil DeliveryService! Skipping!") + if job.DeliveryService == "" { + warnings = append(warnings, "got job from Traffic Ops with an empty DeliveryService! Skipping!") continue } - if _, ok := dsNames[*job.DeliveryService]; !ok { + if _, ok := dsNames[job.DeliveryService]; !ok { continue } dsJobs = append(dsJobs, job) @@ -106,8 +105,7 @@ func MakeRegexRevalidateDotConfig( maxReval := time.Duration(maxDays) * time.Hour * 24 - cfgJobs, jobWarns := filterJobs(dsJobs, maxReval, RegexRevalidateMinTTL) - warnings = append(warnings, jobWarns...) + cfgJobs := filterJobs(dsJobs, maxReval, RegexRevalidateMinTTL) txt := makeHdrComment(opt.HdrComment) for _, job := range cfgJobs { @@ -144,34 +142,26 @@ func (jb jobsSort) Less(i, j int) bool { } // filterJobs returns only jobs which: -// - have a non-null deliveryservice -// - have parameters of the form TTL:%dh +// - have a non-empty deliveryservice // - have a start time later than (now + maxReval days). That is, we don't query jobs older than maxReval in the past. -// - are "purge" jobs // - have a start_time+ttl > now. That is, jobs that haven't expired yet. -// Returns the filtered jobs, and any warnings. -func filterJobs(tcJobs []tc.InvalidationJobV4, maxReval time.Duration, minTTL time.Duration) ([]revalJob, []string) { - warnings := []string{} +// Returns the filtered jobs. +func filterJobs(tcJobs []tc.InvalidationJobV4, maxReval time.Duration, minTTL time.Duration) []revalJob { jobMap := map[string]revalJob{} for _, tcJob := range tcJobs { - if tcJob.DeliveryService == nil || *tcJob.DeliveryService == "" { + if tcJob.DeliveryService == "" { continue } - ttl := time.Duration(*tcJob.TTLHours) * time.Hour + ttl := time.Duration(tcJob.TTLHours) * time.Hour if ttl > maxReval { ttl = maxReval } else if ttl < minTTL { ttl = minTTL } - if tcJob.StartTime == nil { - warnings = append(warnings, fmt.Sprintf("job %+v had nil start time, config generation skipping!\n", tcJob)) - continue - } - if tcJob.StartTime.Add(maxReval).Before(time.Now()) { continue } @@ -179,11 +169,8 @@ func filterJobs(tcJobs []tc.InvalidationJobV4, maxReval time.Duration, minTTL ti if tcJob.StartTime.Add(ttl).Before(time.Now()) { continue } - if tcJob.AssetURL == nil { - continue - } - jobType, assetURL := processRefetch(*tcJob.InvalidationType, *tcJob.AssetURL) + jobType, assetURL := processRefetch(tcJob.InvalidationType, tcJob.AssetURL) purgeEnd := tcJob.StartTime.Add(ttl) @@ -198,7 +185,7 @@ func filterJobs(tcJobs []tc.InvalidationJobV4, maxReval time.Duration, minTTL ti } sort.Sort(jobsSort(newJobs)) - return newJobs, warnings + return newJobs } const MISS = "MISS" diff --git a/lib/go-atscfg/regexrevalidatedotconfig_test.go b/lib/go-atscfg/regexrevalidatedotconfig_test.go index 4d9ca535a0..272a3c9249 100644 --- a/lib/go-atscfg/regexrevalidatedotconfig_test.go +++ b/lib/go-atscfg/regexrevalidatedotconfig_test.go @@ -47,60 +47,60 @@ func TestMakeRegexRevalidateDotConfig(t *testing.T) { jobs := []tc.InvalidationJobV4{ { - AssetURL: util.StrPtr("assetURL0"), - StartTime: util.TimePtr(time.Now().Add(42*24*time.Hour + time.Hour)), - DeliveryService: util.StrPtr("myds"), - CreatedBy: util.StrPtr("me"), - ID: util.UInt64Ptr(42), - TTLHours: util.UIntPtr(14), - InvalidationType: util.StrPtr(tc.REFRESH), + AssetURL: "assetURL0", + StartTime: time.Now().Add(42*24*time.Hour + time.Hour), + DeliveryService: "myds", + CreatedBy: "me", + ID: 42, + TTLHours: 14, + InvalidationType: tc.REFRESH, }, { - AssetURL: util.StrPtr("expiredassetURL0"), - StartTime: util.TimePtr(time.Now().Add(-24 * time.Hour)), - DeliveryService: util.StrPtr("expiredmyds"), - CreatedBy: util.StrPtr("expiredme"), - ID: util.UInt64Ptr(42), - TTLHours: util.UIntPtr(14), - InvalidationType: util.StrPtr(tc.REFRESH), + AssetURL: "expiredassetURL0", + StartTime: time.Now().Add(-24 * time.Hour), + DeliveryService: "expiredmyds", + CreatedBy: "expiredme", + ID: 42, + TTLHours: 14, + InvalidationType: tc.REFRESH, }, { - AssetURL: util.StrPtr("refetchasset##REFETCH##"), - StartTime: util.TimePtr(time.Now().Add(24 * time.Hour)), - DeliveryService: util.StrPtr("myds"), - CreatedBy: util.StrPtr("want_refetch"), - ID: util.UInt64Ptr(42), - TTLHours: util.UIntPtr(24), - InvalidationType: util.StrPtr(tc.REFETCH), + AssetURL: "refetchasset##REFETCH##", + StartTime: time.Now().Add(24 * time.Hour), + DeliveryService: "myds", + CreatedBy: "want_refetch", + ID: 42, + TTLHours: 24, + InvalidationType: tc.REFETCH, }, { - AssetURL: util.StrPtr("refetchtype"), - StartTime: util.TimePtr(time.Now().Add(24 * time.Hour)), - DeliveryService: util.StrPtr("myds"), - CreatedBy: util.StrPtr("want_refetch"), - ID: util.UInt64Ptr(42), - TTLHours: util.UIntPtr(24), - InvalidationType: util.StrPtr(tc.REFETCH), + AssetURL: "refetchtype", + StartTime: time.Now().Add(24 * time.Hour), + DeliveryService: "myds", + CreatedBy: "want_refetch", + ID: 42, + TTLHours: 24, + InvalidationType: tc.REFETCH, }, { // Mixed assetURL and invalidation type. REFETCH should trump REFRESH // for backwards compatibility - AssetURL: util.StrPtr("shouldbeREFETCH##REFETCH##"), - StartTime: util.TimePtr(time.Now().Add(24 * time.Hour)), - DeliveryService: util.StrPtr("myds"), - CreatedBy: util.StrPtr("want_refetch"), - ID: util.UInt64Ptr(42), - TTLHours: util.UIntPtr(24), - InvalidationType: util.StrPtr(tc.REFRESH), + AssetURL: "shouldbeREFETCH##REFETCH##", + StartTime: time.Now().Add(24 * time.Hour), + DeliveryService: "myds", + CreatedBy: "want_refetch", + ID: 42, + TTLHours: 24, + InvalidationType: tc.REFRESH, }, { - AssetURL: util.StrPtr("refreshasset##REFRESH##"), - StartTime: util.TimePtr(time.Now().Add(24 * time.Hour)), - DeliveryService: util.StrPtr("myds"), - CreatedBy: util.StrPtr("want_refresh"), - ID: util.UInt64Ptr(42), - TTLHours: util.UIntPtr(24), - InvalidationType: util.StrPtr(tc.REFRESH), + AssetURL: "refreshasset##REFRESH##", + StartTime: time.Now().Add(24 * time.Hour), + DeliveryService: "myds", + CreatedBy: "want_refresh", + ID: 42, + TTLHours: 24, + InvalidationType: tc.REFRESH, }, } diff --git a/lib/go-tc/invalidationjobs.go b/lib/go-tc/invalidationjobs.go index ff04d90a62..a1d1488de3 100644 --- a/lib/go-tc/invalidationjobs.go +++ b/lib/go-tc/invalidationjobs.go @@ -496,181 +496,33 @@ type InvalidationJobCreateV40 struct { InvalidationType string `json:"invalidationType"` } -// Validates the fields submitted for an InvalidationJobCreateV40. These errors -// are ultimately returned to the user -func (job *InvalidationJobCreateV4) Validate(tx *sql.Tx) error { - errs := []string{} - err := validation.ValidateStruct(job, - validation.Field(&job.DeliveryService, validation.Required), - validation.Field(&job.Regex, validation.Required, validation.NewStringRule(func(s string) bool { - return strings.HasPrefix(s, `\/`) || strings.HasPrefix(s, "/") - }, `must start with '/' (or '\/')`)), - validation.Field(&job.StartTime, validation.Required), - validation.Field(&job.TTLHours, validation.Required), - validation.Field(&job.InvalidationType, validation.Required, validation.NewStringRule(func(s string) bool { - return s == REFRESH || s == REFETCH - }, fmt.Sprintf("must be either %s or %s (case sensitive)", REFRESH, REFETCH))), - ) - if err != nil { - errs = append(errs, err.Error()) - } - - if err := job.validateDeliveryService(tx); err != nil { - errs = append(errs, "Delivery Service is invalid: "+err.Error()) - } - - if _, err := regexp.Compile(job.Regex); err != nil { - errs = append(errs, "regex: is not a valid Regular Expression: "+err.Error()) - } - - if job.StartTime.Before(time.Now()) { - errs = append(errs, "startTime: must be in the future") - } - - if err := job.validateTLLHours(tx); err != nil { - errs = append(errs, "TTL is invalid: "+err.Error()) - } - - if job.InvalidationType == REFETCH && !RefetchAllowed(tx) { - errs = append(errs, "InvalidationType is invalid") - } - - if len(errs) > 0 { - return errors.New(strings.Join(errs, ", ")) - } - - return nil -} - // InvalidationJobV4 is an alias for the InvalidationJobV4 struct used for the latest minor version associated with api major version 4. type InvalidationJobV4 InvalidationJobV40 // InvalidationJobV40 represents a content invalidation job as returned by the API. // Also used for Update calls. type InvalidationJobV40 struct { - ID *uint64 `json:"id"` - AssetURL *string `json:"assetUrl"` - CreatedBy *string `json:"createdBy"` - DeliveryService *string `json:"deliveryService"` - TTLHours *uint `json:"ttlHours"` - InvalidationType *string `json:"invalidationType"` - StartTime *time.Time `json:"startTime"` + ID uint64 `json:"id"` + AssetURL string `json:"assetUrl"` + CreatedBy string `json:"createdBy"` + DeliveryService string `json:"deliveryService"` + TTLHours uint `json:"ttlHours"` + InvalidationType string `json:"invalidationType"` + StartTime time.Time `json:"startTime"` } func (job InvalidationJobV4) String() string { - var ID, AssetURL, CreatedBy, DeliveryService, TTLHours, InvalidationType, StartTime string - if job.ID != nil { - ID = strconv.FormatUint(*job.ID, 10) - } else { - ID = "-1" - } - if job.AssetURL != nil { - AssetURL = *job.AssetURL - } - if job.CreatedBy != nil { - CreatedBy = *job.CreatedBy - } - if job.DeliveryService != nil { - DeliveryService = *job.DeliveryService - } - if job.TTLHours != nil { - TTLHours = strconv.FormatUint(uint64(*job.TTLHours), 10) - } - if job.InvalidationType != nil { - InvalidationType = *job.InvalidationType - } - if job.StartTime != nil { - StartTime = job.StartTime.String() - } + + ID := strconv.FormatUint(job.ID, 10) + TTLHours := strconv.FormatUint(uint64(job.TTLHours), 10) + StartTime := job.StartTime.String() + return fmt.Sprintf("{\nID:\t\t\t\t\t%s,\nAssetURL:\t\t\t%s,\nCreatedBy:\t\t\t%s,\nDeliveryService:\t%s,\nTTLHours:\t\t\t%s,\nInvalidationType:\t%s,\nStartTime:\t\t\t%s,\n}", ID, - AssetURL, - CreatedBy, - DeliveryService, + job.AssetURL, + job.CreatedBy, + job.DeliveryService, TTLHours, - InvalidationType, + job.InvalidationType, StartTime) } - -// Validate checks that the InvalidationJob is valid, by ensuring all of its fields are well-defined. -// -// This returns an error describing any and all problematic fields encountered during validation. -func (job *InvalidationJobV4) Validate() error { - errs := []string{} - err := validation.ValidateStruct(job, - validation.Field(&job.DeliveryService, validation.Required), - validation.Field(&job.AssetURL, validation.Required, is.URL), - validation.Field(&job.CreatedBy, validation.Required), - validation.Field(&job.ID, validation.Required), - validation.Field(&job.TTLHours, validation.Required), - validation.Field(&job.InvalidationType, validation.Required, validation.NewStringRule(func(s string) bool { - return s == REFRESH || s == REFETCH - }, fmt.Sprintf("must be either %s or %s (case sensitive)", REFRESH, REFETCH))), - ) - - if err != nil { - errs = append(errs, err.Error()) - } - - if job.StartTime == nil { - return errors.New(strings.Join(append(errs, "startTime: cannot be blank"), ", ")) - } - - if job.StartTime.After(time.Now().Add(twoDays)) { - errs = append(errs, "startTime: must be within two days from now") - } - - if job.StartTime.Before(time.Now()) { - errs = append(errs, "startTime: cannot be in the past") - } - - if len(errs) > 0 { - return errors.New(strings.Join(errs, ", ")) - } - - return nil -} - -// validateDeliveryService ensures the supplied (required) Delivery Service XML ID exists -func (job *InvalidationJobCreateV4) validateDeliveryService(tx *sql.Tx) error { - var exists bool - row := tx.QueryRow(`SELECT EXISTS(SELECT * FROM deliveryservice WHERE xml_id=$1)`, job.DeliveryService) - if err := row.Scan(&exists); err != nil { - if err == sql.ErrNoRows { - return fmt.Errorf("No DeliveryService exists matching identifier: %v", job.DeliveryService) - } - return errors.New("Unknown error occurred") - } - return nil -} - -// validateTLLHours ensures the supplied TTL hours is within acceptable limits -func (job *InvalidationJobCreateV4) validateTLLHours(tx *sql.Tx) error { - var maxDays uint - err := tx.QueryRow(`SELECT value FROM parameter WHERE name='maxRevalDurationDays' AND config_file='regex_revalidate.config'`).Scan(&maxDays) - maxHours := maxDays * 24 - if err == nil && uint(job.TTLHours) > maxHours { // silently ignore other errors too - return fmt.Errorf("cannot exceed %s", strconv.FormatUint(uint64(maxHours), 10)) - } - return nil -} - -// RefetchAllowed checks whether Refetch is allowed and enabled as a global paramter -func RefetchAllowed(tx *sql.Tx) bool { - refetchEnabled := false - // Silently ignore paramter error - _ = tx.QueryRow(`SELECT value FROM parameter WHERE name='refetch_enabled' AND config_file='global'`).Scan(&refetchEnabled) - return refetchEnabled -} - -func (job *InvalidationJobCreateV4) GetDSIDfromDSXMLID(tx *sql.Tx) (uint, error) { - var dsID uint - row := tx.QueryRow(`SELECT id FROM deliveryservice WHERE xml_id=$1`, job.DeliveryService) - if err := row.Scan(&dsID); err != nil { - if err == sql.ErrNoRows { - return 0, fmt.Errorf("No DeliveryService exists matching identifier: %v", job.DeliveryService) - } - return 0, errors.New("Unknown error occurred") - } - return dsID, nil -} diff --git a/traffic_ops/testing/api/README.md b/traffic_ops/testing/api/README.md index 87537ea2db..45a2bbf75a 100644 --- a/traffic_ops/testing/api/README.md +++ b/traffic_ops/testing/api/README.md @@ -61,22 +61,13 @@ In order to run the tests you will need the following: For more info see: http://trafficcontrol.apache.org/docs/latest/development/traffic_ops.html?highlight=reset -3. A running Traffic Ops instance running with the `secure` (https) and is pointing to the `to_test` - database by running in `MOJO_MODE=test` which will point to your `to_test` database. - To get your to_test database setup do the following: - - `$ export MOJO_MODE=test` - - `$ cd trafficcontrol/traffic_ops/app` - - `$ bin/start.pl --secure` - -4. A running Traffic Ops Golang proxy pointing to the to_test database. - `$ cd trafficcontrol/traffic_ops/traffic_ops_golang` - `$ cp ../app/conf/cdn.conf $HOME/cdn.conf` - change `traffic_ops_golang->port` to 8443 - - `$ go build && ./traffic_ops_golang -cfg $HOME/cdn.conf -dbcfg ../app/conf/test/database.conf` +3. A running Traffic Ops Golang instance pointing to the `to_test` database. + + ```shell + $ cd trafficcontrol/traffic_ops/traffic_ops_golang + $ cp ../app/conf/cdn.conf $HOME/cdn.conf # change `traffic_ops_golang->port` to 8443 + $ go build && ./traffic_ops_golang -cfg $HOME/cdn.conf -dbcfg ../app/conf/test/database.conf + ``` In your local development environment, if the above command fails for an error similar to `ERROR: traffic_ops_golang.go:193: 2020-04-10T10:55:53.190298-06:00: cannot open /etc/pki/tls/certs/localhost.crt for read: open /etc/pki/tls/certs/localhost.crt: no such file or directory` diff --git a/traffic_ops/testing/api/v4/jobs_test.go b/traffic_ops/testing/api/v4/jobs_test.go index 8b49e00464..6ed696d3c0 100644 --- a/traffic_ops/testing/api/v4/jobs_test.go +++ b/traffic_ops/testing/api/v4/jobs_test.go @@ -134,23 +134,19 @@ func JobCollisionWarningTest(t *testing.T) { var realJob *tc.InvalidationJobV4 for i, job := range jobs.Response { - if job.StartTime == nil || job.DeliveryService == nil || job.CreatedBy == nil { - t.Error("Traffic Ops returned a representation of a content invalidation Job that had null or undefined Start Time and/or Delivery Service and/or Created By") - continue - } - diff := newJob.StartTime.Sub(*job.StartTime) - if *job.DeliveryService == xmlID && *job.CreatedBy == "admin" && diff < time.Second { + diff := newJob.StartTime.Sub(job.StartTime) + if job.DeliveryService == xmlID && job.CreatedBy == "admin" && diff < time.Second { realJob = &jobs.Response[i] break } } - if realJob == nil || realJob.ID == nil || *realJob.ID == 0 { + if realJob == nil || realJob.ID == 0 { t.Fatal("could not find new job") } time := firstJob.StartTime.Add(time.Hour * 2) - realJob.StartTime = &time + realJob.StartTime = time alerts, _, err = TOSession.UpdateInvalidationJob(*realJob, client.RequestOptions{}) if err != nil { t.Fatalf("expected invalidation job update to succeed: %v - alerts: %+v", err, alerts.Alerts) @@ -253,10 +249,8 @@ func GetTestJobsQueryParams(t *testing.T) { } foundOne := false for _, j := range toJobs.Response { - if j.DeliveryService == nil { - t.Error("expected: non-nil DeliveryService pointer, actual: nil") - } else if *j.DeliveryService != "ds2" { - t.Errorf("expected: DeliveryService == ds2, actual: DeliveryService == %s", *j.DeliveryService) + if j.DeliveryService != "ds2" { + t.Errorf("expected: DeliveryService == ds2, actual: DeliveryService == %s", j.DeliveryService) } else { foundOne = true } @@ -280,18 +274,18 @@ func GetTestJobs(t *testing.T) { for _, testJob := range testData.InvalidationJobs { found := false for j, toJob := range toJobs.Response { - if toJob.DeliveryService == nil { - t.Errorf("to job (index %v) has nil delivery service", j) + if toJob.DeliveryService == "" { + t.Errorf("to job (index %v) has empty delivery service", j) continue } - if toJob.AssetURL == nil { - t.Errorf("to job (index %v) has nil asset url", j) + if toJob.AssetURL == "" { + t.Errorf("to job (index %v) has empty asset url", j) continue } - if *toJob.DeliveryService != testJob.DeliveryService { + if toJob.DeliveryService != testJob.DeliveryService { continue } - if !strings.HasSuffix(*toJob.AssetURL, testJob.Regex) { + if !strings.HasSuffix(toJob.AssetURL, testJob.Regex) { continue } toJobTime := toJob.StartTime.Round(time.Minute) @@ -332,10 +326,10 @@ func GetTestInvalidationJobs(t *testing.T) { for _, testJob := range testData.InvalidationJobs { found := false for _, toJob := range jobs.Response { - if *toJob.DeliveryService != testJob.DeliveryService { + if toJob.DeliveryService != testJob.DeliveryService { continue } - if !strings.HasSuffix(*toJob.AssetURL, testJob.Regex) { + if !strings.HasSuffix(toJob.AssetURL, testJob.Regex) { continue } if !toJob.StartTime.Round(time.Minute).Equal(testJob.StartTime.Round(time.Minute)) { @@ -368,9 +362,9 @@ func GetTestJobsByValidData(t *testing.T) { invalidationType := jobs.InvalidationType //Get Jobs by Asset URL - if len(*assetUrl) > 0 { + if len(assetUrl) > 0 { opts := client.NewRequestOptions() - opts.QueryParameters.Set("assetUrl", *assetUrl) + opts.QueryParameters.Set("assetUrl", assetUrl) toJobs, _, _ = TOSession.GetInvalidationJobs(opts) if len(toJobs.Response) < 1 { t.Errorf("Expected atleast one Jobs response for GET Jobs by Asset URL, but found %d ", len(toJobs.Response)) @@ -380,9 +374,9 @@ func GetTestJobsByValidData(t *testing.T) { } //Get Jobs by CreatedBy - if len(*createdBy) > 1 { + if len(createdBy) > 1 { opts := client.NewRequestOptions() - opts.QueryParameters.Set("createdBy", *createdBy) + opts.QueryParameters.Set("createdBy", createdBy) toJobs, _, _ = TOSession.GetInvalidationJobs(opts) if len(toJobs.Response) < 1 { t.Errorf("Expected atleast one Jobs response for GET Jobs by CreatedBy, but found %d ", len(toJobs.Response)) @@ -392,21 +386,21 @@ func GetTestJobsByValidData(t *testing.T) { } //Get Jobs by ID - if *id >= 1 { + if id >= 1 { opts := client.NewRequestOptions() - opts.QueryParameters.Set("id", strconv.FormatUint(uint64(*id), 10)) + opts.QueryParameters.Set("id", strconv.FormatUint(uint64(id), 10)) toJobs, _, _ = TOSession.GetInvalidationJobs(opts) if len(toJobs.Response) != 1 { t.Errorf("Expected only one Jobs response for GET Jobs by ID, but found %d ", len(toJobs.Response)) } } else { - t.Errorf("ID Field is empty, so can't test get jobs %d", *id) + t.Errorf("ID Field is empty, so can't test get jobs %d", id) } //Get Jobs by Invalidation Type - if len(*invalidationType) > 1 { + if len(invalidationType) > 1 { opts := client.NewRequestOptions() - opts.QueryParameters.Set("InvalidationType", *invalidationType) + opts.QueryParameters.Set("InvalidationType", invalidationType) toJobs, _, _ = TOSession.GetInvalidationJobs(opts) if len(toJobs.Response) < 1 { t.Errorf("Expected atleast one Jobs response for GET Jobs by keyword, but found %d ", len(toJobs.Response)) @@ -416,9 +410,9 @@ func GetTestJobsByValidData(t *testing.T) { } //Get Delivery Service ID by Name - if dsName != nil && len(*dsName) > 0 { + if len(dsName) > 0 { opts := client.NewRequestOptions() - opts.QueryParameters.Set("xmlId", *dsName) + opts.QueryParameters.Set("xmlId", dsName) toDSes, _, _ := TOSession.GetDeliveryServices(opts) if len(toDSes.Response) > 0 { dsId := toDSes.Response[0].ID @@ -486,8 +480,8 @@ func GetTestJobsByValidData(t *testing.T) { t.Errorf("GET /jobs by maxRevalDurationDays - expected at least 1 job") } for _, j := range maxRevalJobs.Response { - if time.Since(*j.StartTime) > time.Duration(maxRevalDurationDays)*24*time.Hour { - t.Errorf("GET /jobs by maxRevalDurationDays returned job that is older than %d days: {%s, %s, %v}", maxRevalDurationDays, *j.DeliveryService, *j.AssetURL, *j.StartTime) + if time.Since(j.StartTime) > time.Duration(maxRevalDurationDays)*24*time.Hour { + t.Errorf("GET /jobs by maxRevalDurationDays returned job that is older than %d days: {%s, %s, %v}", maxRevalDurationDays, j.DeliveryService, j.AssetURL, j.StartTime) } } @@ -512,8 +506,8 @@ func GetTestJobsByValidData(t *testing.T) { t.Errorf("GET /jobs by cdn - expected at least 1 job") } for _, j := range cdnJobs.Response { - if dsToCDN[*j.DeliveryService] != cdn { - t.Errorf("GET /jobs by cdn returned job that does not belong to CDN %s: {%s, %s, %v}", cdn, *j.DeliveryService, *j.AssetURL, *j.StartTime) + if dsToCDN[j.DeliveryService] != cdn { + t.Errorf("GET /jobs by cdn returned job that does not belong to CDN %s: {%s, %s, %v}", cdn, j.DeliveryService, j.AssetURL, j.StartTime) } } } @@ -943,25 +937,21 @@ func UpdateTestJobsInvalidDS(t *testing.T) { var realJob tc.InvalidationJobV4 for i, job := range jobs.Response { - if job.StartTime == nil || job.DeliveryService == nil || job.CreatedBy == nil { - t.Error("Traffic Ops returned a representation of a content invalidation Job that had null or undefined Start Time and/or Delivery Service and/or Created By") - continue - } - diff := firstJob.StartTime.Sub(*job.StartTime) - if *job.DeliveryService == xmlID && *job.CreatedBy == "admin" && diff < time.Second { + diff := firstJob.StartTime.Sub(job.StartTime) + if job.DeliveryService == xmlID && job.CreatedBy == "admin" && diff < time.Second { realJob = jobs.Response[i] break } } - if realJob.ID == nil || *realJob.ID == 0 { + if realJob.ID == 0 { t.Fatal("could not find new job") } //update existing jobs with new ds id originalJob := realJob newTime := startTime.Add(time.Hour * 2) - originalJob.StartTime = &newTime - originalJob.DeliveryService = testData.DeliveryServices[1].XMLID + originalJob.StartTime = newTime + originalJob.DeliveryService = *testData.DeliveryServices[1].XMLID alerts, reqInf, err := TOSession.UpdateInvalidationJob(originalJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected Cannot change 'deliveryService' of existing invalidation job! - alerts: %+v", alerts.Alerts) @@ -973,7 +963,7 @@ func UpdateTestJobsInvalidDS(t *testing.T) { //update existing jobs with invalid ds id invalidDsIdJob := realJob invalidDsId := "abcd" - invalidDsIdJob.DeliveryService = &invalidDsId + invalidDsIdJob.DeliveryService = invalidDsId alerts, reqInf, err = TOSession.UpdateInvalidationJob(invalidDsIdJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected Cannot change 'deliveryService' of existing invalidation job! - alerts: %+v", alerts.Alerts) @@ -985,7 +975,7 @@ func UpdateTestJobsInvalidDS(t *testing.T) { //update existing jobs with blank ds id blankDsIdJob := realJob blankDsId := "" - blankDsIdJob.DeliveryService = &blankDsId + blankDsIdJob.DeliveryService = blankDsId alerts, reqInf, err = TOSession.UpdateInvalidationJob(blankDsIdJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected deliveryService: cannot be blank. - alerts: %+v", alerts.Alerts) @@ -997,7 +987,7 @@ func UpdateTestJobsInvalidDS(t *testing.T) { //update existing jobs with asset url not starts with origin.infra invalidAssetURLJob := realJob assetURL := "http://google.com" - invalidAssetURLJob.AssetURL = &assetURL + invalidAssetURLJob.AssetURL = assetURL alerts, reqInf, err = TOSession.UpdateInvalidationJob(invalidAssetURLJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected Cannot set asset URL that does not start with Delivery Service origin URL: http://origin.infra.ciab.test. - alerts: %+v", alerts.Alerts) @@ -1009,7 +999,7 @@ func UpdateTestJobsInvalidDS(t *testing.T) { //update existing jobs with blank asset url blankAssetURLJob := realJob assetURL = "" - blankAssetURLJob.AssetURL = &assetURL + blankAssetURLJob.AssetURL = assetURL alerts, reqInf, err = TOSession.UpdateInvalidationJob(blankAssetURLJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected assetUrl: cannot be blank. alerts: %+v", alerts.Alerts) @@ -1021,7 +1011,7 @@ func UpdateTestJobsInvalidDS(t *testing.T) { //update existing jobs with blank created by blankCreatedByJob := realJob createdBy := "" - blankCreatedByJob.CreatedBy = &createdBy + blankCreatedByJob.CreatedBy = createdBy alerts, reqInf, err = TOSession.UpdateInvalidationJob(blankCreatedByJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected createdBy: cannot be blank. alerts: %+v", alerts.Alerts) @@ -1033,7 +1023,7 @@ func UpdateTestJobsInvalidDS(t *testing.T) { //update existing jobs created by createdByJob := realJob createdBy = "operator" - createdByJob.CreatedBy = &createdBy + createdByJob.CreatedBy = createdBy alerts, reqInf, err = TOSession.UpdateInvalidationJob(createdByJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected Cannot change 'createdBy' of existing invalidation jobs!. alerts: %+v", alerts.Alerts) @@ -1045,7 +1035,7 @@ func UpdateTestJobsInvalidDS(t *testing.T) { //update existing jobs with blank invalidation types blankInvalidationTypeJob := realJob invalidationType := "" - blankInvalidationTypeJob.InvalidationType = &invalidationType + blankInvalidationTypeJob.InvalidationType = invalidationType alerts, reqInf, err = TOSession.UpdateInvalidationJob(blankInvalidationTypeJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected parameters: cannot be blank. alerts: %+v", alerts.Alerts) @@ -1059,7 +1049,7 @@ func UpdateTestJobsInvalidDS(t *testing.T) { dt := time.Now() dt.Format("2019-10-12T07:20:50.52Z") addThreeDays := dt.AddDate(0, 0, 3) - startDateFutureJob.StartTime = &addThreeDays + startDateFutureJob.StartTime = addThreeDays alerts, reqInf, err = TOSession.UpdateInvalidationJob(startDateFutureJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected startTime: must be within two days from now. alerts: %+v", alerts.Alerts) @@ -1073,7 +1063,7 @@ func UpdateTestJobsInvalidDS(t *testing.T) { dt = time.Now() dt.Format("2019-06-18 21:28:31") minusThreeDays := dt.AddDate(0, 0, -3) - pastStartDateJob.StartTime = &minusThreeDays + pastStartDateJob.StartTime = minusThreeDays alerts, reqInf, err = TOSession.UpdateInvalidationJob(pastStartDateJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected startTime: cannot be in the past. alerts: %+v", alerts.Alerts) @@ -1087,7 +1077,7 @@ func UpdateTestJobsInvalidDS(t *testing.T) { dt = time.Now() dt.Format("2019-10-12T07:20:50.52Z") minusOneDay := dt.AddDate(0, 0, -1) - pastStartDateJob.StartTime = &minusOneDay + pastStartDateJob.StartTime = minusOneDay alerts, reqInf, err = TOSession.UpdateInvalidationJob(pastStartDateJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected startTime: cannot be in the past. alerts: %+v", alerts.Alerts) @@ -1099,7 +1089,7 @@ func UpdateTestJobsInvalidDS(t *testing.T) { //update jobs with UNIX Format past start date pastStartDateJob = realJob unixTimeFormat := time.Unix(1, 0) - pastStartDateJob.StartTime = &unixTimeFormat + pastStartDateJob.StartTime = unixTimeFormat alerts, reqInf, err = TOSession.UpdateInvalidationJob(pastStartDateJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected startTime: cannot be in the past. alerts: %+v", alerts.Alerts) @@ -1113,7 +1103,7 @@ func UpdateTestJobsInvalidDS(t *testing.T) { dt = time.Now() dt.Format("2020-03-11 14:12:20-06") oneLessDay := dt.AddDate(0, 0, -1) - pastStartDateJob.StartTime = &oneLessDay + pastStartDateJob.StartTime = oneLessDay alerts, reqInf, err = TOSession.UpdateInvalidationJob(pastStartDateJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected startTime: cannot be in the past. alerts: %+v", alerts.Alerts) @@ -1127,7 +1117,7 @@ func UpdateTestJobsInvalidDS(t *testing.T) { dt = time.Now() dt.Format("2019-10-12T07:20:50.52Z") oneMoreDay := dt.AddDate(0, 0, 1) - startDateFutureJob.StartTime = &oneMoreDay + startDateFutureJob.StartTime = oneMoreDay alerts, reqInf, err = TOSession.UpdateInvalidationJob(startDateFutureJob, client.RequestOptions{}) if err != nil { t.Fatalf("Expected Content invalidation job updated. alerts: %+v", alerts.Alerts) @@ -1141,7 +1131,7 @@ func UpdateTestJobsInvalidDS(t *testing.T) { dt = time.Now() dt.Format(".000") andAnotherDay := dt.AddDate(0, 0, 1) - startDateFutureJob.StartTime = &andAnotherDay + startDateFutureJob.StartTime = andAnotherDay alerts, reqInf, err = TOSession.UpdateInvalidationJob(startDateFutureJob, client.RequestOptions{}) if err != nil { t.Fatalf("Expected Content invalidation job updated. alerts: %+v", alerts.Alerts) @@ -1155,7 +1145,7 @@ func UpdateTestJobsInvalidDS(t *testing.T) { dt = time.Now() dt.Format("2020-03-11 14:12:20-06") addEvenMore := dt.AddDate(0, 0, 1) - startDateFutureJob.StartTime = &addEvenMore + startDateFutureJob.StartTime = addEvenMore alerts, reqInf, err = TOSession.UpdateInvalidationJob(startDateFutureJob, client.RequestOptions{}) if err != nil { t.Fatalf("Expected Content invalidation job updated. alerts: %+v", alerts.Alerts) @@ -1168,7 +1158,7 @@ func UpdateTestJobsInvalidDS(t *testing.T) { newIdJob := realJob var b uint64 = 1111 var a *uint64 = &b - newIdJob.ID = a + newIdJob.ID = *a alerts, reqInf, err = TOSession.UpdateInvalidationJob(newIdJob, client.RequestOptions{}) if err == nil { t.Fatalf("Expected Cannot change an invalidation job 'id'! - alerts: %+v", alerts.Alerts) @@ -1192,7 +1182,7 @@ func DeleteTestJobs(t *testing.T) { id := jobs.ID //Delete Jobs by valid id - alerts, reqInf, err := TOSession.DeleteInvalidationJob(uint64(*id), client.RequestOptions{}) + alerts, reqInf, err := TOSession.DeleteInvalidationJob(uint64(id), client.RequestOptions{}) if err != nil { t.Errorf("Expected Content invalidation job was deleted Error - %v, Alerts %v", err, alerts.Alerts) } diff --git a/traffic_ops/testing/api/v4/tc-fixtures.json b/traffic_ops/testing/api/v4/tc-fixtures.json index da13b832c6..7a3f6ab96c 100644 --- a/traffic_ops/testing/api/v4/tc-fixtures.json +++ b/traffic_ops/testing/api/v4/tc-fixtures.json @@ -5695,6 +5695,34 @@ "startTime": "2100-06-19T13:57:51Z", "ttlHours": 144, "invalidationType" : "REFRESH" + }, + { + "deliveryService": "ds1", + "regex": "/oldest", + "startTime": "1970-01-01T12:00:00Z", + "ttlHours": 2160, + "invalidationType" : "REFRESH" + }, + { + "deliveryService": "ds1", + "regex": "/older", + "startTime": "1970-01-01T12:00:00Z", + "ttlHours": 2160, + "invalidationType" : "REFRESH" + }, + { + "deliveryService": "ds1", + "regex": "/old", + "startTime": "1970-01-01T12:00:00Z", + "ttlHours": 2160, + "invalidationType" : "REFRESH" + }, + { + "deliveryService": "ds-forked-topology", + "regex": "/this-ds-is-in-cdn2", + "startTime": "2100-06-19T13:57:51Z", + "ttlHours": 2160, + "invalidationType" : "REFRESH" } ], "statsSummaries": [ diff --git a/traffic_ops/traffic_ops_golang/dbhelpers/db_helpers.go b/traffic_ops/traffic_ops_golang/dbhelpers/db_helpers.go index 0205acffa3..efd1e06057 100644 --- a/traffic_ops/traffic_ops_golang/dbhelpers/db_helpers.go +++ b/traffic_ops/traffic_ops_golang/dbhelpers/db_helpers.go @@ -504,6 +504,18 @@ func GetDSNameFromID(tx *sql.Tx, id int) (tc.DeliveryServiceName, bool, error) { return name, true, nil } +// GetDSNameFromID loads the DeliveryService's xml_id from the database, from the ID. Returns whether the delivery service was found, and any error. +func GetDSIDFromXMLID(tx *sql.Tx, xmlID string) (int, bool, error) { + var id int + if err := tx.QueryRow(`SELECT id FROM deliveryservice WHERE xml_id = $1`, xmlID).Scan(&id); err != nil { + if err == sql.ErrNoRows { + return id, false, nil + } + return id, false, fmt.Errorf("querying ID for delivery service XMLID '%v': %v", xmlID, err) + } + return id, true, nil +} + // GetDSCDNIdFromID loads the DeliveryService's cdn ID from the database, from the delivery service ID. Returns whether the delivery service was found, and any error. func GetDSCDNIdFromID(tx *sql.Tx, dsID int) (int, bool, error) { var cdnID int diff --git a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go index 915080be55..c21e94dce0 100644 --- a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go +++ b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go @@ -25,14 +25,11 @@ import ( "errors" "fmt" "net/http" + "regexp" "strconv" "strings" "time" - "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/util/ims" - - "github.com/lib/pq" - "github.com/apache/trafficcontrol/lib/go-log" "github.com/apache/trafficcontrol/lib/go-rfc" "github.com/apache/trafficcontrol/lib/go-tc" @@ -40,6 +37,11 @@ import ( "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/api" "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/dbhelpers" "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/tenant" + "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/util/ims" + + validation "github.com/go-ozzo/ozzo-validation" + "github.com/go-ozzo/ozzo-validation/is" + "github.com/lib/pq" ) // Deprecated, only to be used with versions below 4.0 @@ -549,7 +551,7 @@ func CreateV40(w http.ResponseWriter, r *http.Request) { // Check if request object is valid w.Header().Set(rfc.ContentType, rfc.ApplicationJSON) - if err := job.Validate(inf.Tx.Tx); err != nil { + if err := validateJobCreateV4(job, inf.Tx.Tx); err != nil { response := tc.Alerts{ Alerts: []tc.Alert{ { @@ -583,12 +585,17 @@ func CreateV40(w http.ResponseWriter, r *http.Request) { } // DS existence was already verified in the Validate() function - dsid, err := job.GetDSIDfromDSXMLID(inf.Tx.Tx) + dsid, exists, err := dbhelpers.GetDSIDFromXMLID(inf.Tx.Tx, job.DeliveryService) if err != nil { sysErr = fmt.Errorf("failed to match XML ID to int ID for Delivery Service %s: %v", job.DeliveryService, err) errCode = http.StatusInternalServerError api.HandleErr(w, r, inf.Tx.Tx, errCode, nil, sysErr) } + if !exists { + userErr = fmt.Errorf("delivery service \"%v\" does not exist", job.DeliveryService) + errCode = http.StatusNotFound + api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) + } row := inf.Tx.Tx.QueryRow(insertQueryV4, job.TTLHours, @@ -615,12 +622,12 @@ func CreateV40(w http.ResponseWriter, r *http.Request) { return } - if err := setRevalFlags(dsid, inf.Tx.Tx); err != nil { + if err := setRevalFlags(uint(dsid), inf.Tx.Tx); err != nil { api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, fmt.Errorf("setting reval flags: %v", err)) return } - conflicts := tc.ValidateJobUniqueness(inf.Tx.Tx, dsid, *result.StartTime, *result.AssetURL, *result.TTLHours) + conflicts := tc.ValidateJobUniqueness(inf.Tx.Tx, uint(dsid), result.StartTime, result.AssetURL, result.TTLHours) response := apiResponseV4{ make([]tc.Alert, len(conflicts)+1), result, @@ -633,9 +640,9 @@ func CreateV40(w http.ResponseWriter, r *http.Request) { } response.Alerts[len(conflicts)] = tc.Alert{ Text: fmt.Sprintf("Invalidation (%s) request created for %v, start:%v end %v", - *result.InvalidationType, - *result.AssetURL, - *result.StartTime, + result.InvalidationType, + result.AssetURL, + result.StartTime, result.StartTime.Add(time.Hour*time.Duration(job.TTLHours))), Level: tc.SuccessLevel.String(), } @@ -651,7 +658,12 @@ func CreateV40(w http.ResponseWriter, r *http.Request) { return } - w.Header().Set(http.CanonicalHeaderKey("location"), fmt.Sprintf("%s://%s/api/%d.%d/jobs?id=%d", inf.Config.URL.Scheme, r.Host, inf.Version.Major, inf.Version.Minor, *result.ID)) + w.Header().Set(http.CanonicalHeaderKey("location"), fmt.Sprintf("%s://%s/api/%d.%d/jobs?id=%d", + inf.Config.URL.Scheme, + r.Host, + inf.Version.Major, + inf.Version.Minor, + result.ID)) w.WriteHeader(http.StatusOK) api.WriteAndLogErr(w, r, append(resp, '\n')) @@ -662,11 +674,11 @@ func CreateV40(w http.ResponseWriter, r *http.Request) { changeLogMsg := fmt.Sprintf("%s content invalidation job %s- ID: %d DSXMLID: %s ASSET_URL: '%s' TTLHRs: %d INVALIDATION: %s", api.Created, duplicate, - *result.ID, - *result.DeliveryService, - *result.AssetURL, - *result.TTLHours, - *result.InvalidationType, + result.ID, + result.DeliveryService, + result.AssetURL, + result.TTLHours, + result.InvalidationType, ) api.CreateChangeLogRawTx(api.ApiChange, changeLogMsg, @@ -872,8 +884,8 @@ func UpdateV40(w http.ResponseWriter, r *http.Request) { return } - if ok, err := IsUserAuthorizedToModifyJobsMadeByUsername(inf, *job.CreatedBy); err != nil { - sysErr = fmt.Errorf("Checking user permissions against user %s: %v", *job.CreatedBy, err) + if ok, err := IsUserAuthorizedToModifyJobsMadeByUsername(inf, job.CreatedBy); err != nil { + sysErr = fmt.Errorf("Checking user permissions against user %s: %v", job.CreatedBy, err) errCode = http.StatusInternalServerError api.HandleErr(w, r, inf.Tx.Tx, errCode, nil, sysErr) return @@ -893,12 +905,12 @@ func UpdateV40(w http.ResponseWriter, r *http.Request) { return } - if err := input.Validate(); err != nil { + if err := validateInvalidationJobV4(input); err != nil { api.HandleErr(w, r, inf.Tx.Tx, http.StatusBadRequest, err, nil) return } - if !strings.HasPrefix(*input.AssetURL, oFQDN) { + if !strings.HasPrefix(input.AssetURL, oFQDN) { userErr = fmt.Errorf("Cannot set asset URL that does not start with Delivery Service origin URL: %s", oFQDN) errCode = http.StatusBadRequest api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) @@ -913,29 +925,29 @@ func UpdateV40(w http.ResponseWriter, r *http.Request) { return } - if *job.DeliveryService != *input.DeliveryService { + if job.DeliveryService != input.DeliveryService { userErr = errors.New("Cannot change 'deliveryService' of existing invalidation job!") errCode = http.StatusConflict api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) return } - if *job.CreatedBy != *input.CreatedBy { + if job.CreatedBy != input.CreatedBy { userErr = errors.New("Cannot change 'createdBy' of existing invalidation jobs!") errCode = http.StatusConflict api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) return } - if *job.ID != *input.ID { + if job.ID != input.ID { userErr = errors.New("Cannot change an invalidation job 'id'!") errCode = http.StatusConflict api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) return } - if *job.InvalidationType != *input.InvalidationType { - if *input.InvalidationType == tc.REFETCH && !tc.RefetchAllowed(inf.Tx.Tx) { + if job.InvalidationType != input.InvalidationType { + if input.InvalidationType == tc.REFETCH && !refetchAllowed(inf.Tx.Tx) { userErr = errors.New("Invalidation Type REFRESH is disallowed") errCode = http.StatusBadRequest api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, nil) @@ -948,7 +960,7 @@ func UpdateV40(w http.ResponseWriter, r *http.Request) { input.TTLHours, input.StartTime, input.InvalidationType, - *job.ID) + job.ID) err = row.Scan(&job.AssetURL, &job.CreatedBy, &job.DeliveryService, @@ -963,12 +975,12 @@ func UpdateV40(w http.ResponseWriter, r *http.Request) { return } - if err = setRevalFlags(*job.DeliveryService, inf.Tx.Tx); err != nil { + if err = setRevalFlags(job.DeliveryService, inf.Tx.Tx); err != nil { api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, fmt.Errorf("Setting reval flags: %v", err)) return } - conflicts := tc.ValidateJobUniqueness(inf.Tx.Tx, dsid, *input.StartTime, *input.AssetURL, *input.TTLHours) + conflicts := tc.ValidateJobUniqueness(inf.Tx.Tx, dsid, input.StartTime, input.AssetURL, input.TTLHours) response := apiResponseV4{ make([]tc.Alert, len(conflicts)+1), job, @@ -981,10 +993,10 @@ func UpdateV40(w http.ResponseWriter, r *http.Request) { } response.Alerts[len(conflicts)] = tc.Alert{ Text: fmt.Sprintf("Invalidation request created for %s, start: %v end: %v invalidation type: %v", - *job.AssetURL, + job.AssetURL, job.StartTime, - job.StartTime.Add(time.Hour*time.Duration(*job.TTLHours)), - *job.InvalidationType), + job.StartTime.Add(time.Hour*time.Duration(job.TTLHours)), + job.InvalidationType), Level: tc.SuccessLevel.String(), } @@ -1001,11 +1013,11 @@ func UpdateV40(w http.ResponseWriter, r *http.Request) { changeLogMsg := fmt.Sprintf("%s content invalidation job - ID: %d DSXMLID: %s ASSET_URL: '%s' TTLHRs: %d INVALIDATION: %s", api.Updated, - *input.ID, - *input.DeliveryService, - *input.AssetURL, - *input.TTLHours, - *input.InvalidationType, + input.ID, + input.DeliveryService, + input.AssetURL, + input.TTLHours, + input.InvalidationType, ) api.CreateChangeLogRawTx(api.ApiChange, changeLogMsg, @@ -1282,11 +1294,11 @@ func DeleteV40(w http.ResponseWriter, r *http.Request) { changeLogMsg := fmt.Sprintf("%s content invalidation job - ID: %d DSXMLID: %s ASSET_URL: '%s' TTLHRs: %d INVALIDATION: %s", api.Deleted, - *result.ID, - *result.DeliveryService, - *result.AssetURL, - *result.TTLHours, - *result.InvalidationType, + result.ID, + result.DeliveryService, + result.AssetURL, + result.TTLHours, + result.InvalidationType, ) api.CreateChangeLogRawTx(api.ApiChange, changeLogMsg, @@ -1393,6 +1405,117 @@ func Delete(w http.ResponseWriter, r *http.Request) { api.CreateChangeLogRawTx(api.ApiChange, api.Deleted+" content invalidation job - ID: "+strconv.FormatUint(*result.ID, 10)+" DS: "+*result.DeliveryService+" URL: '"+*result.AssetURL+"' Params: '"+*result.Parameters+"'", inf.User, inf.Tx.Tx) } +// Validates the fields submitted for an InvalidationJobCreateV40. These errors +// are ultimately returned to the user +func validateJobCreateV4(job tc.InvalidationJobCreateV4, tx *sql.Tx) error { + errs := []string{} + err := validation.ValidateStruct(&job, + validation.Field(&job.DeliveryService, validation.Required), + validation.Field(&job.Regex, validation.Required, validation.NewStringRule(func(s string) bool { + return strings.HasPrefix(s, `\/`) || strings.HasPrefix(s, "/") + }, `must start with '/' (or '\/')`)), + validation.Field(&job.StartTime, validation.Required), + validation.Field(&job.TTLHours, validation.Required), + validation.Field(&job.InvalidationType, validation.Required, validation.NewStringRule(func(s string) bool { + return s == tc.REFRESH || s == tc.REFETCH + }, fmt.Sprintf("must be either %s or %s (case sensitive)", tc.REFRESH, tc.REFETCH))), + ) + if err != nil { + errs = append(errs, err.Error()) + } + + if _, _, err := dbhelpers.GetDSIDFromXMLID(tx, job.DeliveryService); err != nil { + errs = append(errs, "Delivery Service is invalid: "+err.Error()) + } + + if _, err := regexp.Compile(job.Regex); err != nil { + errs = append(errs, "regex: is not a valid Regular Expression: "+err.Error()) + } + + if job.StartTime.Before(time.Now()) { + errs = append(errs, "startTime: must be in the future") + } + + if valid, err := validateTLLHours(job.TTLHours, tx); !valid { + if err != nil { + errs = append(errs, "TTL is invalid: "+err.Error()) + } else { + errs = append(errs, "TTL is invalid") + } + } + + if job.InvalidationType == tc.REFETCH && !refetchAllowed(tx) { + errs = append(errs, "InvalidationType is invalid") + } + + if len(errs) > 0 { + return errors.New(strings.Join(errs, ", ")) + } + + return nil +} + +// validateInvalidationJobV4 checks that the InvalidationJob is valid, by ensuring all of its fields are well-defined. +// This returns an error describing any and all problematic fields encountered during validation. +func validateInvalidationJobV4(job tc.InvalidationJobV4) error { + errs := []string{} + err := validation.ValidateStruct(&job, + validation.Field(&job.DeliveryService, validation.Required), + validation.Field(&job.AssetURL, validation.Required, is.URL), + validation.Field(&job.CreatedBy, validation.Required), + validation.Field(&job.ID, validation.Required), + validation.Field(&job.TTLHours, validation.Required), + validation.Field(&job.StartTime, validation.Required), + validation.Field(&job.InvalidationType, validation.Required, validation.NewStringRule(func(s string) bool { + return s == tc.REFRESH || s == tc.REFETCH + }, fmt.Sprintf("must be either %s or %s (case sensitive)", tc.REFRESH, tc.REFETCH))), + ) + + if err != nil { + errs = append(errs, err.Error()) + } + + if job.StartTime.After(time.Now().Add(time.Hour * 48)) { + errs = append(errs, "startTime: must be within two days from now") + } + + if job.StartTime.Before(time.Now()) { + errs = append(errs, "startTime: cannot be in the past") + } + + if len(errs) > 0 { + return errors.New(strings.Join(errs, ", ")) + } + + return nil +} + +// validateTLLHours ensures the supplied TTL hours is within acceptable limits +func validateTLLHours(ttlHours uint32, tx *sql.Tx) (bool, error) { + var maxDays uint + err := tx.QueryRow(`SELECT value FROM parameter WHERE name='maxRevalDurationDays' AND config_file='regex_revalidate.config'`).Scan(&maxDays) + maxHours := maxDays * 24 + if err != nil { + log.Errorf("error querying \"maxRevalDurationDays\" parameter: %v", err) + return false, nil // sent to the user, hide server error + } + if err == nil && uint(ttlHours) > maxHours { + return false, fmt.Errorf("cannot exceed %s", strconv.FormatUint(uint64(maxHours), 10)) + } + return true, nil +} + +// refetchAllowed checks whether Refetch is allowed and enabled in the parameter table +func refetchAllowed(tx *sql.Tx) bool { + refetchEnabled := false + err := tx.QueryRow(`SELECT 'true' = lower(trim(p.value)) FROM "parameter" p WHERE p.name='refetch_enabled' AND p.config_file='global'`).Scan(&refetchEnabled) + if err != nil { + log.Errorf("error querying \"refetch_enabled\" from parameter: %v", err) + return refetchEnabled // sent to the user, hide server error + } + return refetchEnabled +} + // API versions below 4.0 allowed for either the Delivery Service ID (uint) OR Delivery Service XML-ID (string). // This can be refactored once api versions below 4.0 are removed to take a Delivery Service XML-ID (string), rather // than an empty interface {}. diff --git a/traffic_ops/v4-client/job.go b/traffic_ops/v4-client/job.go index 7f492cf95f..5898f8a115 100644 --- a/traffic_ops/v4-client/job.go +++ b/traffic_ops/v4-client/job.go @@ -16,7 +16,6 @@ package client */ import ( - "errors" "net/url" "strconv" @@ -50,13 +49,10 @@ func (to *Session) DeleteInvalidationJob(jobID uint64, opts RequestOptions) (tc. // expected to have an ID). func (to *Session) UpdateInvalidationJob(job tc.InvalidationJobV4, opts RequestOptions) (tc.Alerts, toclientlib.ReqInf, error) { var alerts tc.Alerts - if job.ID == nil { - return alerts, toclientlib.ReqInf{}, errors.New("job has a nil ID") - } if opts.QueryParameters == nil { opts.QueryParameters = url.Values{} } - opts.QueryParameters.Set("id", strconv.FormatUint(*job.ID, 10)) + opts.QueryParameters.Set("id", strconv.FormatUint(job.ID, 10)) reqInf, err := to.put(apiJobs, opts, job, &alerts) return alerts, reqInf, err } From d9a29c45790421106207f4921858515e9d895e65 Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Thu, 14 Oct 2021 10:24:46 -0600 Subject: [PATCH 31/35] Add Refetch API tests. Make the id query param required for updates. Fix the named parameter colon escape. Ensure helper function still remains in atscfg library. --- lib/go-atscfg/atscfg.go | 7 + lib/go-tc/enum.go | 13 ++ traffic_ops/testing/api/README.md | 21 ++- traffic_ops/testing/api/v4/jobs_test.go | 138 +++++++++++++++--- traffic_ops/testing/api/v4/tc-fixtures.json | 16 ++ .../testing/api/v4/traffic_control_test.go | 1 + .../invalidationjobs/invalidationjobs.go | 13 +- 7 files changed, 175 insertions(+), 34 deletions(-) diff --git a/lib/go-atscfg/atscfg.go b/lib/go-atscfg/atscfg.go index fc1c8d9efa..5a83b6caab 100644 --- a/lib/go-atscfg/atscfg.go +++ b/lib/go-atscfg/atscfg.go @@ -22,6 +22,7 @@ package atscfg import ( "encoding/json" "errors" + "fmt" "net" "sort" "strconv" @@ -654,6 +655,12 @@ func makeErr(warnings []string, err string) error { return errors.New(`(warnings: ` + strings.Join(warnings, `, `) + `) ` + err) } +// makeErrf is a convenience for formatting errors for makeErr. +// todo also unused, maybe remove? +func makeErrf(warnings []string, format string, v ...interface{}) error { + return makeErr(warnings, fmt.Sprintf(format, v...)) +} + // DeliveryServiceServer is a compact version of DeliveryServiceServer. // The Delivery Service Servers is massive on large CDNs not using Topologies, compacting it in JSON and dropping the timestamp drastically reduces the size. // The t3c apps will also drop any DSS from Traffic Ops with null values, which are invalid and useless, to avoid pointers and further reduce size. diff --git a/lib/go-tc/enum.go b/lib/go-tc/enum.go index ec910d7a57..3bb9da6316 100644 --- a/lib/go-tc/enum.go +++ b/lib/go-tc/enum.go @@ -219,6 +219,19 @@ type ParameterName string // fetched by t3c. const UseRevalPendingParameterName = ParameterName("use_reval_pending") +// RefetchEnabled is the name of a Parameter used to determine if the +// Refetch feature is enabled. If enabled, this allows a consumer of the TO API +// to submit Refetch InvalidationJob types. These will subsequently be treated +// as a MISS by cache servers. Previously, the only capability was Refresh +// which was, in turn, treated as a STALE by cache servers. This value should +// be used with caution, since coupled with regex, could cause significant +// performance impacts by implementing cache servers if used incorrectly. +// +// Note that there is no guarantee that a Parameter with this name exists in +// Traffic Ops at any given time, and while it's implementation relies on +// a boolean Value, this it not gauranteed either. +const RefetchEnabled = ParameterName("refetch_enabled") + // ConfigFileName is a Parameter ConfigFile value. // // This has no additional attached semantics, and so while it is known to most diff --git a/traffic_ops/testing/api/README.md b/traffic_ops/testing/api/README.md index 45a2bbf75a..fdf57422f8 100644 --- a/traffic_ops/testing/api/README.md +++ b/traffic_ops/testing/api/README.md @@ -31,9 +31,10 @@ In order to run the tests you will need the following: To get your to_test database setup do the following: - `$ cd trafficcontrol/traffic_ops/app` - - `$ db/admin --env=test reset` + ```shell + $ cd trafficcontrol/traffic_ops/app + $ db/admin --env=test reset + ``` NOTE on passwords: Check that the passwords defined defined for your `to_test` database match @@ -45,6 +46,7 @@ In order to run the tests you will need the following: Note that for the database to successfully set up the tables and run the migrations, you will the `db/admin` tool. To test if `db/admin` ran all migrations successfully, you can run the following command from the `traffic_ops/app` directory: + ```shell db/admin -env=test dbversion ``` @@ -74,7 +76,7 @@ In order to run the tests you will need the following: you might not have the right certificates at the right locations. Follow the procedure listed [here](https://traffic-control-cdn.readthedocs.io/en/latest/admin/traffic_ops.html#id12) to fix it. ## Running the API Tests -The integration tests are run using `go test`, however, there are some flags that need to be provided in order for the tests to work. +The integration tests are run using `go test` from the traffic_ops/testing/api/ directory, however, there are some flags that need to be provided in order for the tests to work. The flags are: @@ -85,9 +87,16 @@ The flags are: * test_data - traffic control * run - Go runtime flag for executing a specific test case -Example command to run the tests: -`TO_URL=https://localhost:8443 go test -v -cfg=traffic-ops-test.conf -run TestCDNs` +Example command to run the tests: +```shell +$ TO_URL=https://localhost:8443 go test -v -cfg=traffic-ops-test.conf -run TestCDNs +``` + +or, since the cfg file location is inferred, the call could be shortened to test a specific API version with something like: +```shell +$ go test -v run TestJobs ./v4 +``` * It can take several minutes for the API tests to complete, so using the `-v` flag is recommended to see progress.* diff --git a/traffic_ops/testing/api/v4/jobs_test.go b/traffic_ops/testing/api/v4/jobs_test.go index 6ed696d3c0..ffd314daea 100644 --- a/traffic_ops/testing/api/v4/jobs_test.go +++ b/traffic_ops/testing/api/v4/jobs_test.go @@ -16,6 +16,7 @@ package v4 */ import ( + "fmt" "net/http" "strconv" "strings" @@ -28,25 +29,27 @@ import ( func TestJobs(t *testing.T) { WithObjs(t, []TCObj{CDNs, Types, Tenants, Parameters, Profiles, Statuses, Divisions, Regions, PhysLocations, CacheGroups, Servers, Topologies, ServiceCategories, DeliveryServices}, func() { - CreateTestJobs(t) - CreateTestInvalidationJobs(t) - CreateTestInvalidJob(t) - GetTestJobsQueryParams(t) - GetTestJobs(t) - GetTestInvalidationJobs(t) - JobCollisionWarningTest(t) - GetTestJobsByValidData(t) - GetTestJobsByInvalidData(t) - CreateTestJobsInvalidDS(t) - CreateTestJobsAlreadyExistTTL(t) - CreateTestJobsWithPastDate(t) - CreateTestJobsWithFutureDate(t) - CreateJobsMissingDate(t) - CreateJobsMissingRegex(t) - CreateJobsMissingTtl(t) - UpdateTestJobsInvalidDS(t) - DeleteTestJobs(t) - DeleteTestJobsByInvalidId(t) + // CreateTestJobs(t) + // CreateTestInvalidationJobs(t) + // CreateTestInvalidJob(t) + // GetTestJobsQueryParams(t) + // GetTestJobs(t) + // GetTestInvalidationJobs(t) + // JobCollisionWarningTest(t) + // GetTestJobsByValidData(t) + // GetTestJobsByInvalidData(t) + // CreateTestJobsInvalidDS(t) + // CreateTestJobsAlreadyExistTTL(t) + // CreateTestJobsWithPastDate(t) + // CreateTestJobsWithFutureDate(t) + // CreateJobsMissingDate(t) + // CreateJobsMissingRegex(t) + // CreateJobsMissingTtl(t) + // UpdateTestJobsInvalidDS(t) + // DeleteTestJobs(t) + // DeleteTestJobsByInvalidId(t) + CreateRefetchJobParameterFail(t) + CreateRefetchJobParameterSuccess(t) }) } @@ -1204,3 +1207,100 @@ func DeleteTestJobsByInvalidId(t *testing.T) { t.Errorf("Expected status code 404, got %v", reqInf.StatusCode) } } + +func CreateRefetchJobParameterFail(t *testing.T) { + + // Ensure clean slate for parameters + err := clearRefetchEnabledParameter() + if err != nil { + t.Fatal(err) + } + + // Attempt to create Refetch job w/o refetch_enabled + job := testData.InvalidationJobsRefetch[0] + createJob := tc.InvalidationJobCreateV4{ + DeliveryService: job.DeliveryService, + Regex: job.Regex, + TTLHours: job.TTLHours, + StartTime: time.Now().Add(time.Hour).UTC(), + InvalidationType: job.InvalidationType, + } + + _, _, err = TOSession.CreateInvalidationJob(createJob, client.RequestOptions{}) + if err == nil { + t.Fatalf("expected error preventing the creation of the Refetch Invalidation Job.") + } + +} + +func CreateRefetchJobParameterSuccess(t *testing.T) { + + // Ensure clean slate for parameters + err := clearRefetchEnabledParameter() + if err != nil { + t.Fatal(err) + } + + // Create refetch_enabled parameter + param := tc.Parameter{ + ConfigFile: string(tc.GlobalConfigFileName), + Name: string(tc.RefetchEnabled), + Secure: false, + Value: "true", + } + + paramsResp, _, err := TOSession.CreateParameter(param, client.RequestOptions{}) + if err != nil { + t.Fatalf("error creating RefetchEnabled parameter. err: %v \n alerts: %v", err, paramsResp.Alerts) + } + + // Create Refetch jobs + for _, job := range testData.InvalidationJobsRefetch { + createJob := tc.InvalidationJobCreateV4{ + DeliveryService: job.DeliveryService, + Regex: job.Regex, + TTLHours: job.TTLHours, + StartTime: time.Now().Add(time.Hour).UTC(), + InvalidationType: job.InvalidationType, + } + + createResp, _, err := TOSession.CreateInvalidationJob(createJob, client.RequestOptions{}) + if err != nil { + t.Fatalf("error posting Refetch Invalidation Job. err: %v \n alerts: %v", err, createResp.Alerts) + } + } + + // Get all jobs + jobsResp, _, err := TOSession.GetInvalidationJobs(client.RequestOptions{}) + if err != nil { + t.Fatalf("error requesting Invalidation Jobs. err: %v \n alerts: %v", err, jobsResp.Alerts) + } + + // Ensure expected created refetch jobs matches actual + var refetchJobs int + for _, job := range jobsResp.Response { + if job.InvalidationType == tc.REFETCH { + refetchJobs++ + } + } + + if refetchJobs != len(testData.InvalidationJobsRefetch) { + t.Fatalf("failed to verify creation of Refetch Invalidation Jobs. Refetch Job count: %v Expected job count: %v", refetchJobs, len(testData.InvalidationJobsRefetch)) + } + +} + +func clearRefetchEnabledParameter() error { + // Ensure Parameter is not set + paramsResp, _, err := TOSession.GetParameters(client.RequestOptions{}) + if err != nil { + return fmt.Errorf("error retrieving parameters. err: %v \n alerts: %v", err, paramsResp.Alerts) + } + + for _, param := range paramsResp.Response { + if param.Name == string(tc.RefetchEnabled) { + TOSession.DeleteParameter(param.ID, client.RequestOptions{}) + } + } + return nil +} diff --git a/traffic_ops/testing/api/v4/tc-fixtures.json b/traffic_ops/testing/api/v4/tc-fixtures.json index 7a3f6ab96c..9d04c57ded 100644 --- a/traffic_ops/testing/api/v4/tc-fixtures.json +++ b/traffic_ops/testing/api/v4/tc-fixtures.json @@ -5725,6 +5725,22 @@ "invalidationType" : "REFRESH" } ], + "invalidationJobsRefetch": [ + { + "deliveryService": "ds1", + "regex": "/.*", + "startTime": "2100-06-19T13:57:51Z", + "ttlHours": 72, + "invalidationType" : "REFETCH" + }, + { + "deliveryService": "ds1", + "regex": "/foo", + "startTime": "2100-06-19T13:57:51Z", + "ttlHours": 36, + "invalidationType" : "REFETCH" + } + ], "statsSummaries": [ { "cdnName": "cdn1", diff --git a/traffic_ops/testing/api/v4/traffic_control_test.go b/traffic_ops/testing/api/v4/traffic_control_test.go index 783f7751dc..e1d68ec69f 100644 --- a/traffic_ops/testing/api/v4/traffic_control_test.go +++ b/traffic_ops/testing/api/v4/traffic_control_test.go @@ -57,4 +57,5 @@ type TrafficControl struct { Serverchecks []tc.ServercheckRequestNullable `json:"serverchecks"` Users []tc.UserV4 `json:"users"` InvalidationJobs []tc.InvalidationJobCreateV4 `json:"invalidationJobs"` + InvalidationJobsRefetch []tc.InvalidationJobCreateV4 `json:"invalidationJobsRefetch"` } diff --git a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go index c21e94dce0..3b5761c874 100644 --- a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go +++ b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go @@ -312,7 +312,7 @@ func selectMaxLastUpdatedQuery(where string) string { const readQuery = ` SELECT job.id, 'PURGE' AS keyword, - ttl_hr, + CONCAT('TTL::', ttl_hr, 'h') AS parameters, asset_url, start_time, u.username as createdBy, @@ -517,12 +517,6 @@ func (job *InvalidationJob) Read(h http.Header, useIMS bool) ([]interface{}, err return nil, nil, fmt.Errorf("parsing db response: %v", err), http.StatusInternalServerError, nil } - // NamedQuery (nor prepare statements) cannot handle string functions in SELECT such as: - // CONCAT('TTL:', ttl_hr, 'h') AS parameters - // So it is necessary to modify the value after it's been queried. - fmtTTL := fmt.Sprintf("TTL:%sh", *j.Parameters) - j.Parameters = &fmtTTL - returnable = append(returnable, j) } @@ -838,7 +832,7 @@ func Create(w http.ResponseWriter, r *http.Request) { // Used by PUT requests to `/jobs`, replaces an existing content invalidation job // with the provided request body. func UpdateV40(w http.ResponseWriter, r *http.Request) { - inf, userErr, sysErr, errCode := api.NewInfo(r, nil, nil) + inf, userErr, sysErr, errCode := api.NewInfo(r, []string{"id"}, []string{"id"}) if userErr != nil || sysErr != nil { api.HandleErr(w, r, inf.Tx.Tx, errCode, userErr, sysErr) return @@ -1508,7 +1502,8 @@ func validateTLLHours(ttlHours uint32, tx *sql.Tx) (bool, error) { // refetchAllowed checks whether Refetch is allowed and enabled in the parameter table func refetchAllowed(tx *sql.Tx) bool { refetchEnabled := false - err := tx.QueryRow(`SELECT 'true' = lower(trim(p.value)) FROM "parameter" p WHERE p.name='refetch_enabled' AND p.config_file='global'`).Scan(&refetchEnabled) + err := tx.QueryRow(`SELECT 'true' = lower(trim(p.value)) FROM "parameter" p WHERE p.name=$1 AND p.config_file=$2`, + tc.RefetchEnabled, tc.GlobalConfigFileName).Scan(&refetchEnabled) if err != nil { log.Errorf("error querying \"refetch_enabled\" from parameter: %v", err) return refetchEnabled // sent to the user, hide server error From 8069c9750af2a1344bc29b58c3c90054df8e78d0 Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Thu, 14 Oct 2021 15:38:32 -0600 Subject: [PATCH 32/35] Use type alias to prevent mass changes. When a change is done in go-tc that causes a new struct, use a type alias to ensure the change is localized and minimized throughout go-atscfg and t3c --- .../t3c-generate/cfgfile/cfgfile_test.go | 6 +- cache-config/t3cutil/getdatacfg.go | 2 +- cache-config/t3cutil/toreq/clientfuncs.go | 10 +-- cache-config/t3cutil/toreq/conversions.go | 4 + lib/go-atscfg/atscfg.go | 83 +++++++++++-------- lib/go-atscfg/regexrevalidatedotconfig.go | 6 +- .../regexrevalidatedotconfig_test.go | 2 +- 7 files changed, 65 insertions(+), 48 deletions(-) diff --git a/cache-config/t3c-generate/cfgfile/cfgfile_test.go b/cache-config/t3c-generate/cfgfile/cfgfile_test.go index 05d8012085..de0e79ed8e 100644 --- a/cache-config/t3c-generate/cfgfile/cfgfile_test.go +++ b/cache-config/t3c-generate/cfgfile/cfgfile_test.go @@ -356,9 +356,9 @@ func randParam() *tc.Parameter { } } -func randJob() *tc.InvalidationJobV4 { +func randJob() *atscfg.InvalidationJob { now := time.Now() - return &tc.InvalidationJobV4{ + return &atscfg.InvalidationJob{ AssetURL: *randStr(), CreatedBy: *randStr(), StartTime: now, @@ -491,7 +491,7 @@ func MakeFakeTOData() *t3cutil.ConfigData { }, DeliveryServiceServers: dss, Server: sv0, - Jobs: []tc.InvalidationJobV4{ + Jobs: []atscfg.InvalidationJob{ *randJob(), *randJob(), }, diff --git a/cache-config/t3cutil/getdatacfg.go b/cache-config/t3cutil/getdatacfg.go index 847049b949..5e3a280894 100644 --- a/cache-config/t3cutil/getdatacfg.go +++ b/cache-config/t3cutil/getdatacfg.go @@ -69,7 +69,7 @@ type ConfigData struct { Server *atscfg.Server `json:"server,omitempty"` // Jobs must be all Jobs on the server's CDN. May include jobs on other CDNs. - Jobs []tc.InvalidationJobV4 `json:"jobs,omitempty"` + Jobs []atscfg.InvalidationJob `json:"jobs,omitempty"` // CDN must be the CDN of the server. CDN *tc.CDN `json:"cdn,omitempty"` diff --git a/cache-config/t3cutil/toreq/clientfuncs.go b/cache-config/t3cutil/toreq/clientfuncs.go index 664d54bc77..7785e48b9a 100644 --- a/cache-config/t3cutil/toreq/clientfuncs.go +++ b/cache-config/t3cutil/toreq/clientfuncs.go @@ -515,17 +515,17 @@ func (cl *TOClient) GetDeliveryServiceRegexes(reqHdr http.Header) ([]tc.Delivery return regexes, reqInf, nil } -func (cl *TOClient) GetJobs(reqHdr http.Header, cdnName string) ([]tc.InvalidationJobV4, toclientlib.ReqInf, error) { +func (cl *TOClient) GetJobs(reqHdr http.Header, cdnName string) ([]atscfg.InvalidationJob, toclientlib.ReqInf, error) { if cl.c == nil { oldJobs, inf, err := cl.old.GetJobs() - jobs, err := atscfg.JobsToInvalidationJobsV4(oldJobs) + jobs, err := atscfg.JobsToInvalidationJobs(oldJobs) if err != nil { return nil, inf, errors.New("converting old []tc.Job to []tc.InvalidationJob: " + err.Error()) } return jobs, inf, err } - jobs := []tc.InvalidationJobV4{} + jobs := []atscfg.InvalidationJob{} reqInf := toclientlib.ReqInf{} err := torequtil.GetRetry(cl.NumRetries, "jobs_cdn_"+cdnName, &jobs, func(obj interface{}) error { opts := *ReqOpts(reqHdr) @@ -535,8 +535,8 @@ func (cl *TOClient) GetJobs(reqHdr http.Header, cdnName string) ([]tc.Invalidati if err != nil { return errors.New("getting jobs from Traffic Ops '" + torequtil.MaybeIPStr(reqInf.RemoteAddr) + "': " + err.Error()) } - jobs := obj.(*[]tc.InvalidationJobV4) - *jobs = toJobs.Response + jobs := obj.(*[]atscfg.InvalidationJob) + *jobs = jobsToLatest(toJobs.Response) reqInf = toReqInf return nil }) diff --git a/cache-config/t3cutil/toreq/conversions.go b/cache-config/t3cutil/toreq/conversions.go index b73d74a614..3cd368714b 100644 --- a/cache-config/t3cutil/toreq/conversions.go +++ b/cache-config/t3cutil/toreq/conversions.go @@ -36,3 +36,7 @@ func serverToLatest(oldSv *tc.ServerV40) (*atscfg.Server, error) { func dsesToLatest(dses []tc.DeliveryServiceV40) []atscfg.DeliveryService { return atscfg.V40ToDeliveryServices(dses) } + +func jobsToLatest(jobs []tc.InvalidationJobV4) []atscfg.InvalidationJob { + return atscfg.ToInvalidationJobs(jobs) +} diff --git a/lib/go-atscfg/atscfg.go b/lib/go-atscfg/atscfg.go index 5a83b6caab..a1c16b5132 100644 --- a/lib/go-atscfg/atscfg.go +++ b/lib/go-atscfg/atscfg.go @@ -29,7 +29,6 @@ import ( "strings" "github.com/apache/trafficcontrol/lib/go-tc" - "github.com/apache/trafficcontrol/lib/go-util" ) const InvalidID = -1 @@ -59,6 +58,11 @@ type Server tc.ServerV40 // but to only have to change it here, and the places where breaking symbol changes were made. type DeliveryService tc.DeliveryServiceV40 +// InvalidationJob is a tc.InvalidationJob for the latest lib/go-tc and traffic_ops/vx-client type. +// This allows atscfg to not have to change the type everywhere it's used, every time ATC changes the base type, +// but to only have to change it here, and the places where breaking symbol changes were made. +type InvalidationJob tc.InvalidationJobV4 + // ToDeliveryServices converts a slice of the latest lib/go-tc and traffic_ops/vx-client type to the local alias. func ToDeliveryServices(dses []tc.DeliveryServiceV40) []DeliveryService { ad := []DeliveryService{} @@ -77,6 +81,15 @@ func V40ToDeliveryServices(dses []tc.DeliveryServiceV40) []DeliveryService { return ad } +// ToInvalidationJobs converts a slice of the latest lib/go-tc and traffic_ops/vx-client type to the local alias. +func ToInvalidationJobs(jobs []tc.InvalidationJobV4) []InvalidationJob { + aj := []InvalidationJob{} + for _, job := range jobs { + aj = append(aj, InvalidationJob(job)) + } + return aj +} + // ToServers converts a slice of the latest lib/go-tc and traffic_ops/vx-client type to the local alias. func ToServers(servers []tc.ServerV40) []Server { as := []Server{} @@ -669,8 +682,36 @@ type DeliveryServiceServer struct { DeliveryService int `json:"d"` } -func JobsToInvalidationJobs(oldJobs []tc.Job) ([]tc.InvalidationJob, error) { - jobs := make([]tc.InvalidationJob, len(oldJobs), len(oldJobs)) +// func JobsToInvalidationJobs(oldJobs []tc.Job) ([]tc.InvalidationJob, error) { +// jobs := make([]tc.InvalidationJob, len(oldJobs), len(oldJobs)) +// err := error(nil) +// for i, oldJob := range oldJobs { +// jobs[i], err = JobToInvalidationJob(oldJob) +// if err != nil { +// return nil, errors.New("converting old tc.Job to tc.InvalidationJob: " + err.Error()) +// } +// } +// return jobs, nil +// } + +// func JobToInvalidationJob(jb tc.Job) (tc.InvalidationJob, error) { +// startTime := tc.Time{} +// if err := json.Unmarshal([]byte(`"`+jb.StartTime+`"`), &startTime); err != nil { +// return tc.InvalidationJob{}, errors.New("unmarshalling time: " + err.Error()) +// } +// return tc.InvalidationJob{ +// AssetURL: util.StrPtr(jb.AssetURL), +// CreatedBy: util.StrPtr(jb.CreatedBy), +// DeliveryService: util.StrPtr(jb.DeliveryService), +// ID: util.Uint64Ptr(uint64(jb.ID)), +// Keyword: util.StrPtr(jb.Keyword), +// Parameters: util.StrPtr(jb.Parameters), +// StartTime: &startTime, +// }, nil +// } + +func JobsToInvalidationJobs(oldJobs []tc.Job) ([]InvalidationJob, error) { + jobs := make([]InvalidationJob, len(oldJobs), len(oldJobs)) err := error(nil) for i, oldJob := range oldJobs { jobs[i], err = JobToInvalidationJob(oldJob) @@ -681,44 +722,16 @@ func JobsToInvalidationJobs(oldJobs []tc.Job) ([]tc.InvalidationJob, error) { return jobs, nil } -func JobToInvalidationJob(jb tc.Job) (tc.InvalidationJob, error) { - startTime := tc.Time{} - if err := json.Unmarshal([]byte(`"`+jb.StartTime+`"`), &startTime); err != nil { - return tc.InvalidationJob{}, errors.New("unmarshalling time: " + err.Error()) - } - return tc.InvalidationJob{ - AssetURL: util.StrPtr(jb.AssetURL), - CreatedBy: util.StrPtr(jb.CreatedBy), - DeliveryService: util.StrPtr(jb.DeliveryService), - ID: util.Uint64Ptr(uint64(jb.ID)), - Keyword: util.StrPtr(jb.Keyword), - Parameters: util.StrPtr(jb.Parameters), - StartTime: &startTime, - }, nil -} - -func JobsToInvalidationJobsV4(oldJobs []tc.Job) ([]tc.InvalidationJobV4, error) { - jobs := make([]tc.InvalidationJobV4, len(oldJobs), len(oldJobs)) - err := error(nil) - for i, oldJob := range oldJobs { - jobs[i], err = JobToInvalidationJobV4(oldJob) - if err != nil { - return nil, errors.New("converting old tc.Job to tc.InvalidationJob: " + err.Error()) - } - } - return jobs, nil -} - -func JobToInvalidationJobV4(jb tc.Job) (tc.InvalidationJobV4, error) { +func JobToInvalidationJob(jb tc.Job) (InvalidationJob, error) { startTime := tc.Time{} if err := json.Unmarshal([]byte(`"`+jb.StartTime+`"`), &startTime); err != nil { - return tc.InvalidationJobV4{}, errors.New("unmarshalling time: " + err.Error()) + return InvalidationJob{}, errors.New("unmarshalling time: " + err.Error()) } ttl, err := strconv.Atoi(strings.TrimSuffix(strings.TrimPrefix(jb.Parameters, "TTL:"), "h")) if err != nil { - return tc.InvalidationJobV4{}, errors.New("unmarshalling ttl: " + err.Error()) + return InvalidationJob{}, errors.New("unmarshalling ttl: " + err.Error()) } - return tc.InvalidationJobV4{ + return InvalidationJob{ AssetURL: jb.AssetURL, CreatedBy: jb.CreatedBy, DeliveryService: jb.DeliveryService, diff --git a/lib/go-atscfg/regexrevalidatedotconfig.go b/lib/go-atscfg/regexrevalidatedotconfig.go index 5639e910a3..86674ad289 100644 --- a/lib/go-atscfg/regexrevalidatedotconfig.go +++ b/lib/go-atscfg/regexrevalidatedotconfig.go @@ -56,7 +56,7 @@ func MakeRegexRevalidateDotConfig( server *Server, deliveryServices []DeliveryService, globalParams []tc.Parameter, - jobs []tc.InvalidationJobV4, + jobs []InvalidationJob, opt *RegexRevalidateDotConfigOpts, ) (Cfg, error) { if opt == nil { @@ -79,7 +79,7 @@ func MakeRegexRevalidateDotConfig( dsNames[*ds.XMLID] = struct{}{} } - dsJobs := []tc.InvalidationJobV4{} + dsJobs := []InvalidationJob{} for _, job := range jobs { if job.DeliveryService == "" { warnings = append(warnings, "got job from Traffic Ops with an empty DeliveryService! Skipping!") @@ -146,7 +146,7 @@ func (jb jobsSort) Less(i, j int) bool { // - have a start time later than (now + maxReval days). That is, we don't query jobs older than maxReval in the past. // - have a start_time+ttl > now. That is, jobs that haven't expired yet. // Returns the filtered jobs. -func filterJobs(tcJobs []tc.InvalidationJobV4, maxReval time.Duration, minTTL time.Duration) []revalJob { +func filterJobs(tcJobs []InvalidationJob, maxReval time.Duration, minTTL time.Duration) []revalJob { jobMap := map[string]revalJob{} diff --git a/lib/go-atscfg/regexrevalidatedotconfig_test.go b/lib/go-atscfg/regexrevalidatedotconfig_test.go index 272a3c9249..1073a93468 100644 --- a/lib/go-atscfg/regexrevalidatedotconfig_test.go +++ b/lib/go-atscfg/regexrevalidatedotconfig_test.go @@ -45,7 +45,7 @@ func TestMakeRegexRevalidateDotConfig(t *testing.T) { "unrelated": {"unrelated0", "unrelated1"}, }) - jobs := []tc.InvalidationJobV4{ + jobs := []InvalidationJob{ { AssetURL: "assetURL0", StartTime: time.Now().Add(42*24*time.Hour + time.Hour), From 9ddb6c1b5bff4adae82b64b7ad36d8a6b7575756 Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Thu, 14 Oct 2021 15:39:49 -0600 Subject: [PATCH 33/35] Remove commented code --- lib/go-atscfg/atscfg.go | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/lib/go-atscfg/atscfg.go b/lib/go-atscfg/atscfg.go index a1c16b5132..b30b27538b 100644 --- a/lib/go-atscfg/atscfg.go +++ b/lib/go-atscfg/atscfg.go @@ -682,34 +682,6 @@ type DeliveryServiceServer struct { DeliveryService int `json:"d"` } -// func JobsToInvalidationJobs(oldJobs []tc.Job) ([]tc.InvalidationJob, error) { -// jobs := make([]tc.InvalidationJob, len(oldJobs), len(oldJobs)) -// err := error(nil) -// for i, oldJob := range oldJobs { -// jobs[i], err = JobToInvalidationJob(oldJob) -// if err != nil { -// return nil, errors.New("converting old tc.Job to tc.InvalidationJob: " + err.Error()) -// } -// } -// return jobs, nil -// } - -// func JobToInvalidationJob(jb tc.Job) (tc.InvalidationJob, error) { -// startTime := tc.Time{} -// if err := json.Unmarshal([]byte(`"`+jb.StartTime+`"`), &startTime); err != nil { -// return tc.InvalidationJob{}, errors.New("unmarshalling time: " + err.Error()) -// } -// return tc.InvalidationJob{ -// AssetURL: util.StrPtr(jb.AssetURL), -// CreatedBy: util.StrPtr(jb.CreatedBy), -// DeliveryService: util.StrPtr(jb.DeliveryService), -// ID: util.Uint64Ptr(uint64(jb.ID)), -// Keyword: util.StrPtr(jb.Keyword), -// Parameters: util.StrPtr(jb.Parameters), -// StartTime: &startTime, -// }, nil -// } - func JobsToInvalidationJobs(oldJobs []tc.Job) ([]InvalidationJob, error) { jobs := make([]InvalidationJob, len(oldJobs), len(oldJobs)) err := error(nil) From 3d606ad051599fee82601a1291399166d2ef0744 Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Fri, 22 Oct 2021 15:04:31 -0600 Subject: [PATCH 34/35] Uncomment API tests. Revert API error handling call --- traffic_ops/testing/api/v4/jobs_test.go | 38 +++++++++---------- .../invalidationjobs/invalidationjobs.go | 3 +- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/traffic_ops/testing/api/v4/jobs_test.go b/traffic_ops/testing/api/v4/jobs_test.go index ffd314daea..dcd62f5457 100644 --- a/traffic_ops/testing/api/v4/jobs_test.go +++ b/traffic_ops/testing/api/v4/jobs_test.go @@ -29,25 +29,25 @@ import ( func TestJobs(t *testing.T) { WithObjs(t, []TCObj{CDNs, Types, Tenants, Parameters, Profiles, Statuses, Divisions, Regions, PhysLocations, CacheGroups, Servers, Topologies, ServiceCategories, DeliveryServices}, func() { - // CreateTestJobs(t) - // CreateTestInvalidationJobs(t) - // CreateTestInvalidJob(t) - // GetTestJobsQueryParams(t) - // GetTestJobs(t) - // GetTestInvalidationJobs(t) - // JobCollisionWarningTest(t) - // GetTestJobsByValidData(t) - // GetTestJobsByInvalidData(t) - // CreateTestJobsInvalidDS(t) - // CreateTestJobsAlreadyExistTTL(t) - // CreateTestJobsWithPastDate(t) - // CreateTestJobsWithFutureDate(t) - // CreateJobsMissingDate(t) - // CreateJobsMissingRegex(t) - // CreateJobsMissingTtl(t) - // UpdateTestJobsInvalidDS(t) - // DeleteTestJobs(t) - // DeleteTestJobsByInvalidId(t) + CreateTestJobs(t) + CreateTestInvalidationJobs(t) + CreateTestInvalidJob(t) + GetTestJobsQueryParams(t) + GetTestJobs(t) + GetTestInvalidationJobs(t) + JobCollisionWarningTest(t) + GetTestJobsByValidData(t) + GetTestJobsByInvalidData(t) + CreateTestJobsInvalidDS(t) + CreateTestJobsAlreadyExistTTL(t) + CreateTestJobsWithPastDate(t) + CreateTestJobsWithFutureDate(t) + CreateJobsMissingDate(t) + CreateJobsMissingRegex(t) + CreateJobsMissingTtl(t) + UpdateTestJobsInvalidDS(t) + DeleteTestJobs(t) + DeleteTestJobsByInvalidId(t) CreateRefetchJobParameterFail(t) CreateRefetchJobParameterSuccess(t) }) diff --git a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go index 3b5761c874..c49b5ad68f 100644 --- a/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go +++ b/traffic_ops/traffic_ops_golang/invalidationjobs/invalidationjobs.go @@ -715,7 +715,8 @@ func Create(w http.ResponseWriter, r *http.Request) { return } - api.HandleErr(w, r, inf.Tx.Tx, http.StatusBadRequest, errors.New(string(append(resp, '\n'))), fmt.Errorf("error validating job: %v", err)) + w.WriteHeader(http.StatusBadRequest) + api.WriteAndLogErr(w, r, append(resp, '\n')) return } From 01d2fe6d2eaebb16f343c7649516a898b70e1866 Mon Sep 17 00:00:00 2001 From: Taylor Frey Date: Tue, 2 Nov 2021 12:01:02 -0600 Subject: [PATCH 35/35] Rebase and move DB migrations --- ...25500_refetch.down.sql => 2021110211525600_refetch.down.sql} | 0 ...321025500_refetch.up.sql => 2021110211525600_refetch.up.sql} | 0 traffic_ops/testing/api/README.md | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) rename traffic_ops/app/db/migrations/{2021081321025500_refetch.down.sql => 2021110211525600_refetch.down.sql} (100%) rename traffic_ops/app/db/migrations/{2021081321025500_refetch.up.sql => 2021110211525600_refetch.up.sql} (100%) diff --git a/traffic_ops/app/db/migrations/2021081321025500_refetch.down.sql b/traffic_ops/app/db/migrations/2021110211525600_refetch.down.sql similarity index 100% rename from traffic_ops/app/db/migrations/2021081321025500_refetch.down.sql rename to traffic_ops/app/db/migrations/2021110211525600_refetch.down.sql diff --git a/traffic_ops/app/db/migrations/2021081321025500_refetch.up.sql b/traffic_ops/app/db/migrations/2021110211525600_refetch.up.sql similarity index 100% rename from traffic_ops/app/db/migrations/2021081321025500_refetch.up.sql rename to traffic_ops/app/db/migrations/2021110211525600_refetch.up.sql diff --git a/traffic_ops/testing/api/README.md b/traffic_ops/testing/api/README.md index fdf57422f8..056d5517b0 100644 --- a/traffic_ops/testing/api/README.md +++ b/traffic_ops/testing/api/README.md @@ -95,7 +95,7 @@ $ TO_URL=https://localhost:8443 go test -v -cfg=traffic-ops-test.conf -run TestC or, since the cfg file location is inferred, the call could be shortened to test a specific API version with something like: ```shell -$ go test -v run TestJobs ./v4 +$ go test -v -run TestJobs ./v4 ```