diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f7e10bb74..94646cd1eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - [#7570](https://github.com/apache/trafficcontrol/pull/7570) *Traffic Ops* Fixes `deliveryservice_request_comments` v5 apis to respond with `RFC3339` date/time Format. - [#7312](https://github.com/apache/trafficcontrol/issues/7312) *Docs* Changing docs for CDN locks for DELETE response structure v4 and v5. - [#7572](https://github.com/apache/trafficcontrol/pull/7572) *Traffic Ops* Fixes Delivery Service Requests V5 apis docs with RFC3339 date/time Format +- [#7544](https://github.com/apache/trafficcontrol/issues/7544) *Traffic Ops* Fixes stats_summary v5 apis to respond with RFC3339 date/time Format. - [#7542](https://github.com/apache/trafficcontrol/pull/7542) *Traffic Ops* Fixed `CDN Locks` documentation to reflect the correct RFC3339 timestamps. - [#6340](https://github.com/apache/trafficcontrol/issues/6340) *Traffic Ops* Fixed alert messages for POST and PUT invalidation job APIs. - [#7519] (https://github.com/apache/trafficcontrol/issues/7519) *Traffic Ops* Fixed TO API /servers/{id}/deliveryservices endpoint to responding with all DS's on cache that are directly assigned and inherited through topology. diff --git a/docs/source/api/v5/stats_summary.rst b/docs/source/api/v5/stats_summary.rst index 60a45ee582..5628058685 100644 --- a/docs/source/api/v5/stats_summary.rst +++ b/docs/source/api/v5/stats_summary.rst @@ -110,7 +110,7 @@ Summary Stats :statName: Stat name summary stat represents :statValue: Summary stat value -:summaryTime: Timestamp of summary, in :ref:`non-rfc-datetime` +:summaryTime: Timestamp of summary, in :rfc:`3339` format :statDate: Date stat was taken, in ``YYYY-MM-DD`` format .. code-block:: http @@ -134,7 +134,7 @@ Summary Stats "deliveryServiceName": "all", "statName": "daily_maxgbps", "statValue": 5, - "summaryTime": "2019-11-19 00:04:06+00", + "summaryTime": "2019-11-19T03:37:33+05:30", "statDate": "2019-11-19" }, { @@ -142,7 +142,7 @@ Summary Stats "deliveryServiceName": "all", "statName": "daily_maxgbps", "statValue": 3, - "summaryTime": "2019-11-18 00:04:06+00", + "summaryTime": "2019-11-18T07:59:54+05:30", "statDate": "2019-11-18" }, { @@ -150,7 +150,7 @@ Summary Stats "deliveryServiceName": "all", "statName": "daily_bytesserved", "statValue": 1000, - "summaryTime": "2019-11-19 00:04:06+00", + "summaryTime": "2019-11-19T00:42:12+05:30", "statDate": "2019-11-19" } ]} @@ -158,7 +158,7 @@ Summary Stats Last Updated Summary Stat """"""""""""""""""""""""" -:summaryTime: Timestamp of the last updated summary, in :ref:`non-rfc-datetime` +:summaryTime: Timestamp of the last updated summary, in :rfc:`3339` format .. code-block:: http :caption: Response Example @@ -176,7 +176,7 @@ Last Updated Summary Stat Content-Length: 150 { "response": { - "summaryTime": "2019-11-19 00:04:06+00" + "summaryTime": "2019-11-19T06:12:34+05:30" }} ``POST`` @@ -201,7 +201,7 @@ Request Structure :statName: Stat name summary stat represents :statValue: Summary stat value -:summaryTime: Timestamp of summary, in :ref:`non-rfc-datetime` +:summaryTime: Timestamp of summary, in :rfc:`3339` format :statDate: Date stat was taken, in ``YYYY-MM-DD`` format .. note:: ``statName``, ``statValue`` and ``summaryTime`` are required. If ``cdnName`` and ``deliveryServiceName`` are not given they will default to ``all``. @@ -222,7 +222,7 @@ Request Structure "deliveryServiceName": "all", "statName": "daily_maxgbps", "statValue": 10, - "summaryTime": "2019-12-05 00:03:57+00", + "summaryTime": "2019-12-05T09:55:07+05:30", "statDate": "2019-12-05" } diff --git a/lib/go-tc/stats_summary.go b/lib/go-tc/stats_summary.go index a516ae5a55..bee02eba19 100644 --- a/lib/go-tc/stats_summary.go +++ b/lib/go-tc/stats_summary.go @@ -23,6 +23,7 @@ import ( "database/sql" "encoding/json" "errors" + "fmt" "time" "github.com/apache/trafficcontrol/lib/go-tc/tovalidate" @@ -187,3 +188,149 @@ type StatsSummaryLastUpdatedAPIResponse struct { Response StatsSummaryLastUpdated `json:"response"` Alerts } + +// StatsSummaryV5 is an alias for the latest minor version for the major version 5. +type StatsSummaryV5 StatsSummaryV50 + +// StatsSummaryV50 is a summary of some kind of statistic for a CDN and/or +// Delivery Service. +type StatsSummaryV50 struct { + CDNName *string `json:"cdnName" db:"cdn_name"` + DeliveryService *string `json:"deliveryServiceName" db:"deliveryservice_name"` + StatName *string `json:"statName" db:"stat_name"` + StatValue *float64 `json:"statValue" db:"stat_value"` + SummaryTime time.Time `json:"summaryTime" db:"summary_time"` + StatDate *time.Time `json:"statDate" db:"stat_date"` +} + +// Validate implements the +// github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/api.ParseValidator +// interface. +func (ss StatsSummaryV5) Validate(tx *sql.Tx) error { + errs := tovalidate.ToErrors(validation.Errors{ + "statName": validation.Validate(ss.StatName, validation.Required), + "statValue": validation.Validate(ss.StatValue, validation.Required), + }) + return util.JoinErrs(errs) +} + +// UnmarshalJSON implements the encoding/json.Unmarshaler interface with a +// customized decoding to force the date format on StatDate. +func (ss *StatsSummaryV5) UnmarshalJSON(data []byte) error { + type Alias StatsSummaryV5 + resp := struct { + SummaryTime string `json:"summaryTime"` + StatDate *string `json:"statDate"` + *Alias + }{ + Alias: (*Alias)(ss), + } + err := json.Unmarshal(data, &resp) + if err != nil { + return err + } + if resp.StatDate != nil { + statDate, err := parseTimeV5(*resp.StatDate) + if err != nil { + return fmt.Errorf("invalid timestamp given for statDate: %v", err) + } + ss.StatDate = &statDate + } + + ss.SummaryTime, err = parseTimeV5(resp.SummaryTime) + if err != nil { + return fmt.Errorf("invalid timestamp given for summaryTime: %v", err) + } + return nil +} + +func parseTimeV5(ts string) (time.Time, error) { + rt, err := time.Parse(time.RFC3339, ts) + if err == nil { + return rt, err + } + return time.Parse(dateFormat, ts) +} + +// MarshalJSON implements the encoding/json.Marshaler interface with a +// customized encoding to force the date format on StatDate. +func (ss StatsSummaryV5) MarshalJSON() ([]byte, error) { + type Alias StatsSummaryV5 + resp := struct { + StatDate *string `json:"statDate"` + SummaryTime string `json:"summaryTime"` + Alias + }{ + SummaryTime: ss.SummaryTime.Format(time.RFC3339), + Alias: (Alias)(ss), + } + if ss.StatDate != nil { + resp.StatDate = util.Ptr(ss.StatDate.Format(dateFormat)) + } + return json.Marshal(&resp) +} + +// StatsSummaryResponseV5 is an alias for the latest minor version for the major version 5. +type StatsSummaryResponseV5 StatsSummaryResponseV50 + +// StatsSummaryResponseV50 is the structure of a response from Traffic Ops to +// GET requests made to its /stats_summary V5 API endpoint. +type StatsSummaryResponseV50 struct { + Response []StatsSummaryV5 `json:"response"` + Alerts +} + +// StatsSummaryLastUpdatedV5 is an alias for the latest minor version for the major version 5. +type StatsSummaryLastUpdatedV5 StatsSummaryLastUpdatedV50 + +// StatsSummaryLastUpdatedV50 is the type of the `response` property of a response +// from Traffic Ops to a GET request made to its /stats_summary endpoint when +// the 'lastSummaryDate' query string parameter is passed as 'true'. +type StatsSummaryLastUpdatedV50 struct { + SummaryTime *time.Time `json:"summaryTime" db:"summary_time"` +} + +// MarshalJSON implements the encoding/json.Marshaler interface with a +// customized encoding to force the date format on SummaryTime. +func (ss StatsSummaryLastUpdatedV5) MarshalJSON() ([]byte, error) { + resp := struct { + SummaryTime *string `json:"summaryTime"` + }{} + if ss.SummaryTime != nil { + resp.SummaryTime = util.Ptr(ss.SummaryTime.Format(time.RFC3339)) + } + return json.Marshal(&resp) +} + +// UnmarshalJSON implements the encoding/json.Unmarshaler interface with a +// customized decoding to force the SummaryTime format. +func (ss *StatsSummaryLastUpdatedV5) UnmarshalJSON(data []byte) error { + resp := struct { + SummaryTime *string `json:"summaryTime"` + }{} + err := json.Unmarshal(data, &resp) + if err != nil { + return err + } + if resp.SummaryTime != nil { + var summaryTime time.Time + summaryTime, err = time.Parse(time.RFC3339, *resp.SummaryTime) + if err == nil { + ss.SummaryTime = &summaryTime + return nil + } + return err + } + return nil +} + +// StatsSummaryLastUpdatedAPIResponseV5 is an alias for the latest minor version for the major version 5. +type StatsSummaryLastUpdatedAPIResponseV5 StatsSummaryLastUpdatedAPIResponseV50 + +// StatsSummaryLastUpdatedAPIResponseV50 is the type of a response from Traffic +// Ops to a request to its /stats_summary endpoint with the 'lastSummaryDate' +// query string parameter set to 'true'. +type StatsSummaryLastUpdatedAPIResponseV50 struct { + Response StatsSummaryLastUpdatedV5 `json:"response"` + Alerts +} diff --git a/traffic_ops/testing/api/v5/stats_summary_test.go b/traffic_ops/testing/api/v5/stats_summary_test.go index d7525050ee..cd4a94d657 100644 --- a/traffic_ops/testing/api/v5/stats_summary_test.go +++ b/traffic_ops/testing/api/v5/stats_summary_test.go @@ -34,7 +34,7 @@ func TestStatsSummary(t *testing.T) { CreateTestStatsSummaries(t) - methodTests := utils.TestCase[client.Session, client.RequestOptions, tc.StatsSummary]{ + methodTests := utils.TestCase[client.Session, client.RequestOptions, tc.StatsSummaryV5]{ "GET": { "OK when VALID request": { ClientSession: TOSession, @@ -115,7 +115,7 @@ func TestStatsSummary(t *testing.T) { func validateStatsSummaryFields(expectedResp map[string]interface{}) utils.CkReqFunc { return func(t *testing.T, _ toclientlib.ReqInf, resp interface{}, _ tc.Alerts, _ error) { assert.RequireNotNil(t, resp, "Expected Stats Summary response to not be nil.") - statsSummaryResp := resp.([]tc.StatsSummary) + statsSummaryResp := resp.([]tc.StatsSummaryV5) for field, expected := range expectedResp { for _, statsSummary := range statsSummaryResp { switch field { @@ -141,9 +141,10 @@ func validateStatsSummaryFields(expectedResp map[string]interface{}) utils.CkReq func validateStatsSummaryLastUpdatedField(expectedTime time.Time) utils.CkReqFunc { return func(t *testing.T, _ toclientlib.ReqInf, resp interface{}, _ tc.Alerts, _ error) { assert.RequireNotNil(t, resp, "Expected StatsSummaryLastUpdated response to not be nil.") - statsSummaryLastUpdated := resp.(tc.StatsSummaryLastUpdated) + statsSummaryLastUpdated := resp.(tc.StatsSummaryLastUpdatedV5) assert.RequireNotNil(t, statsSummaryLastUpdated.SummaryTime, "Expected SummaryTime to not be nil.") - assert.Equal(t, expectedTime, *statsSummaryLastUpdated.SummaryTime, "Expected SummaryTime to be %v, but got %v", expectedTime, *statsSummaryLastUpdated.SummaryTime) + assert.Equal(t, true, expectedTime.Equal(*statsSummaryLastUpdated.SummaryTime), "Expected SummaryTime to be %v, but got %v", expectedTime, *statsSummaryLastUpdated.SummaryTime) + } } diff --git a/traffic_ops/testing/api/v5/traffic_control_test.go b/traffic_ops/testing/api/v5/traffic_control_test.go index 59048da291..409da97887 100644 --- a/traffic_ops/testing/api/v5/traffic_control_test.go +++ b/traffic_ops/testing/api/v5/traffic_control_test.go @@ -51,7 +51,7 @@ type TrafficControl struct { ServiceCategories []tc.ServiceCategoryV5 `json:"serviceCategories"` Statuses []tc.StatusNullable `json:"statuses"` StaticDNSEntries []tc.StaticDNSEntry `json:"staticdnsentries"` - StatsSummaries []tc.StatsSummary `json:"statsSummaries"` + StatsSummaries []tc.StatsSummaryV5 `json:"statsSummaries"` Tenants []tc.Tenant `json:"tenants"` ServerCheckExtensions []tc.ServerCheckExtensionNullable `json:"servercheck_extensions"` Topologies []tc.Topology `json:"topologies"` diff --git a/traffic_ops/traffic_ops_golang/trafficstats/stats_summary.go b/traffic_ops/traffic_ops_golang/trafficstats/stats_summary.go index 347e7861cc..deb4c7d346 100644 --- a/traffic_ops/traffic_ops_golang/trafficstats/stats_summary.go +++ b/traffic_ops/traffic_ops_golang/trafficstats/stats_summary.go @@ -25,10 +25,9 @@ import ( "github.com/apache/trafficcontrol/lib/go-tc" "github.com/apache/trafficcontrol/lib/go-util" - "github.com/jmoiron/sqlx" - "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/api" "github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang/dbhelpers" + "github.com/jmoiron/sqlx" ) // GetStatsSummary handler for getting stats summaries @@ -60,16 +59,31 @@ func getLastSummaryDate(w http.ResponseWriter, r *http.Request, inf *api.APIInfo return } query := selectQuery() + where + " ORDER BY summary_time DESC" - statsSummaries, err := queryStatsSummary(inf.Tx, query, queryValues) + statsSummaries, err := queryStatsSummary(inf.Tx, inf.Version.Major, query, queryValues) if err != nil { api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, err) return } - resp := tc.StatsSummaryLastUpdated{} - if len(statsSummaries) >= 1 { - resp.SummaryTime = &statsSummaries[0].SummaryTime + + if inf.Version.Major >= 5 { + resp := tc.StatsSummaryLastUpdatedV5{} + if len(statsSummaries) >= 1 { + if summary, ok := statsSummaries[0].(tc.StatsSummaryV5); ok { + resp.SummaryTime = &summary.SummaryTime + } + } + api.WriteResp(w, r, resp) + + } else { + resp := tc.StatsSummaryLastUpdated{} + if len(statsSummaries) >= 1 { + if summary, ok := statsSummaries[0].(tc.StatsSummary); ok { + resp.SummaryTime = &summary.SummaryTime + } + } + api.WriteResp(w, r, resp) } - api.WriteResp(w, r, resp) + } func getStatsSummary(w http.ResponseWriter, r *http.Request, inf *api.APIInfo) { @@ -84,31 +98,90 @@ func getStatsSummary(w http.ResponseWriter, r *http.Request, inf *api.APIInfo) { return } query := selectQuery() + where + orderBy + pagination - statsSummaries, err := queryStatsSummary(inf.Tx, query, queryValues) + queryStatsSummaries, err := queryStatsSummary(inf.Tx, inf.Version.Major, query, queryValues) if err != nil { api.HandleErr(w, r, inf.Tx.Tx, http.StatusInternalServerError, nil, err) return } - api.WriteResp(w, r, statsSummaries) + //api.WriteResp(w, r, statsSummaries) + + if inf.Version.Major >= 5 { + statsSummariesV5 := make([]tc.StatsSummaryV5, len(queryStatsSummaries)) + for i, oldStat := range queryStatsSummaries { + if summary, ok := oldStat.(tc.StatsSummaryV5); ok { + newStat := tc.StatsSummaryV5{ + CDNName: summary.CDNName, + DeliveryService: summary.DeliveryService, + StatName: summary.StatName, + StatValue: summary.StatValue, + SummaryTime: summary.SummaryTime, + StatDate: summary.StatDate, + } + statsSummariesV5[i] = newStat + } + } + api.WriteResp(w, r, statsSummariesV5) + } else { + statsSummaries := make([]tc.StatsSummary, len(queryStatsSummaries)) + for i, oldStat := range queryStatsSummaries { + if summary, ok := oldStat.(tc.StatsSummary); ok { + newStat := tc.StatsSummary{ + CDNName: summary.CDNName, + DeliveryService: summary.DeliveryService, + StatName: summary.StatName, + StatValue: summary.StatValue, + SummaryTime: summary.SummaryTime, + StatDate: summary.StatDate, + } + statsSummaries[i] = newStat + } + } + api.WriteResp(w, r, statsSummaries) + } + } -func queryStatsSummary(tx *sqlx.Tx, q string, queryValues map[string]interface{}) ([]tc.StatsSummary, error) { +func queryStatsSummary(tx *sqlx.Tx, version uint64, q string, queryValues map[string]interface{}) ([]interface{}, error) { rows, err := tx.NamedQuery(q, queryValues) if err != nil { return nil, fmt.Errorf("querying stats summary: %v", err) } defer rows.Close() - statsSummaries := []tc.StatsSummary{} - for rows.Next() { - s := tc.StatsSummary{} - if err = rows.StructScan(&s); err != nil { - return nil, fmt.Errorf("scanning stats summary: %v", err) + var returnStatsSummaries []interface{} + + if version >= 5 { + var statsSummariesV5 []tc.StatsSummaryV5 + for rows.Next() { + s := tc.StatsSummaryV5{} + if err = rows.StructScan(&s); err != nil { + return nil, fmt.Errorf("scanning stats summary: %v", err) + } + statsSummariesV5 = append(statsSummariesV5, s) + } + returnStatsSummaries = make([]interface{}, len(statsSummariesV5)) + for i, v := range statsSummariesV5 { + returnStatsSummaries[i] = v + } + + } else { + var statsSummaries []tc.StatsSummary + for rows.Next() { + s := tc.StatsSummary{} + if err = rows.StructScan(&s); err != nil { + return nil, fmt.Errorf("scanning stats summary: %v", err) + } + statsSummaries = append(statsSummaries, s) + } + + returnStatsSummaries = make([]interface{}, len(statsSummaries)) + for i, v := range statsSummaries { + returnStatsSummaries[i] = v } - statsSummaries = append(statsSummaries, s) } - return statsSummaries, nil + return returnStatsSummaries, nil + } // CreateStatsSummary handler for creating stats summaries diff --git a/traffic_ops/traffic_ops_golang/trafficstats/stats_summary_test.go b/traffic_ops/traffic_ops_golang/trafficstats/stats_summary_test.go new file mode 100644 index 0000000000..45dbfa7b79 --- /dev/null +++ b/traffic_ops/traffic_ops_golang/trafficstats/stats_summary_test.go @@ -0,0 +1,78 @@ +package trafficstats + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "testing" + "time" + + "github.com/apache/trafficcontrol/lib/go-util/assert" + + "github.com/jmoiron/sqlx" + "gopkg.in/DATA-DOG/go-sqlmock.v1" +) + +func TestQueryStatsSummary(t *testing.T) { + type testStruct struct { + version uint64 + } + + var testData = []testStruct{ + {4}, + {5}, + } + + query := "SELECT cdn_name, deliveryservice_name, stat_name, stat_value, summary_time, stat_date FROM stats_summary" + queryValues := map[string]interface{}{ + "lastSummaryDate": "true", + } + + for i, _ := range testData { + mockDB, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("an error '%v' was not expected when opening a stub database connection", err) + } + defer mockDB.Close() + + db := sqlx.NewDb(mockDB, "sqlmock") + defer db.Close() + + mock.ExpectBegin() + rows := sqlmock.NewRows([]string{ + "cdn_name", + "deliveryservice_name", + "stat_name", + "stat_value", + "summary_time", + "stat_date", + }) + + rows.AddRow("cdn1", "all", "daily_maxgbps", 5, time.Now().AddDate(0, 0, -5), time.Now().AddDate(0, 0, -5).Truncate(24*time.Hour)) + rows.AddRow("cdn2", "all", "daily_byteserved", 1000, time.Now().AddDate(0, 0, -10), time.Now().AddDate(0, 0, -10).Truncate(24*time.Hour)) + + mock.ExpectQuery("SELECT cdn_name, deliveryservice_name, stat_name, stat_value, summary_time, stat_date FROM stats_summary").WithArgs().WillReturnRows(rows) + + statsSummaries1, err1 := queryStatsSummary(db.MustBegin(), testData[i].version, query, queryValues) + + assert.NoError(t, err1) + assert.Equal(t, len(statsSummaries1), 2) + } + +} diff --git a/traffic_ops/v5-client/stats_summary.go b/traffic_ops/v5-client/stats_summary.go index 9b8e9a683d..03b7597cff 100644 --- a/traffic_ops/v5-client/stats_summary.go +++ b/traffic_ops/v5-client/stats_summary.go @@ -24,8 +24,8 @@ const apiStatsSummary = "/stats_summary" // GetSummaryStats gets a list of Summary Stats with the ability to filter on // CDN, Delivery Service, and/or stat name. -func (to *Session) GetSummaryStats(opts RequestOptions) (tc.StatsSummaryResponse, toclientlib.ReqInf, error) { - var resp tc.StatsSummaryResponse +func (to *Session) GetSummaryStats(opts RequestOptions) (tc.StatsSummaryResponseV5, toclientlib.ReqInf, error) { + var resp tc.StatsSummaryResponseV5 reqInf, err := to.get(apiStatsSummary, opts, &resp) return resp, reqInf, err } @@ -34,19 +34,19 @@ func (to *Session) GetSummaryStats(opts RequestOptions) (tc.StatsSummaryResponse // updated. // If 'statName' isn't nil, the response will be limited to the stat thereby // named. -func (to *Session) GetSummaryStatsLastUpdated(opts RequestOptions) (tc.StatsSummaryLastUpdatedAPIResponse, toclientlib.ReqInf, error) { +func (to *Session) GetSummaryStatsLastUpdated(opts RequestOptions) (tc.StatsSummaryLastUpdatedAPIResponseV5, toclientlib.ReqInf, error) { if opts.QueryParameters == nil { opts.QueryParameters = url.Values{} } opts.QueryParameters.Set("lastSummaryDate", "true") - var resp tc.StatsSummaryLastUpdatedAPIResponse + var resp tc.StatsSummaryLastUpdatedAPIResponseV5 reqInf, err := to.get(apiStatsSummary, opts, &resp) return resp, reqInf, err } // CreateSummaryStats creates the given Stats Summary. -func (to *Session) CreateSummaryStats(statsSummary tc.StatsSummary, opts RequestOptions) (tc.Alerts, toclientlib.ReqInf, error) { +func (to *Session) CreateSummaryStats(statsSummary tc.StatsSummaryV5, opts RequestOptions) (tc.Alerts, toclientlib.ReqInf, error) { var alerts tc.Alerts reqInf, err := to.post(apiStatsSummary, opts, statsSummary, &alerts) return alerts, reqInf, err