From 63fca4fe9c8e4227a23d585aae2f1f7bf66ead4a Mon Sep 17 00:00:00 2001 From: AbdelrahmanElawady Date: Fri, 1 Sep 2023 20:43:49 +0300 Subject: [PATCH] Configure uncacheable delivery services in Varnish --- lib/go-atscfg/cachedotconfig.go | 121 ++++++++++++++++----------- lib/varnishcfg/cache_control.go | 57 +++++++++++++ lib/varnishcfg/cache_control_test.go | 90 ++++++++++++++++++++ lib/varnishcfg/vclbuilder.go | 10 +++ 4 files changed, 227 insertions(+), 51 deletions(-) create mode 100644 lib/varnishcfg/cache_control.go create mode 100644 lib/varnishcfg/cache_control_test.go diff --git a/lib/go-atscfg/cachedotconfig.go b/lib/go-atscfg/cachedotconfig.go index 949bc4c27c..9418d462e3 100644 --- a/lib/go-atscfg/cachedotconfig.go +++ b/lib/go-atscfg/cachedotconfig.go @@ -20,6 +20,7 @@ package atscfg */ import ( + "fmt" "sort" "strings" @@ -71,6 +72,62 @@ func makeCacheDotConfigEdge( return Cfg{}, makeErr(warnings, "server missing profiles") } + profileDSes, dsWarnings := GetProfileDSes(server, servers, deliveryServices, deliveryServiceServers) + warnings = append(warnings, dsWarnings...) + + lines := map[string]struct{}{} // use a "set" for lines, to avoid duplicates, since we're looking up by profile + for _, ds := range profileDSes { + if ds.Type != tc.DSTypeHTTPNoCache { + continue + } + if ds.OriginFQDN == "" { + warnings = append(warnings, fmt.Sprintf("profileCacheDotConfig ds %s has no origin fqdn, skipping!", ds.Name)) + continue + } + originFQDN, originPort := GetHostPortFromURI(ds.OriginFQDN) + if originPort != "" { + l := "dest_domain=" + originFQDN + " port=" + originPort + " scheme=http action=never-cache\n" + lines[l] = struct{}{} + } else { + l := "dest_domain=" + originFQDN + " scheme=http action=never-cache\n" + lines[l] = struct{}{} + } + } + + linesArr := []string{} + for line, _ := range lines { + linesArr = append(linesArr, line) + } + sort.Strings(linesArr) + text := strings.Join(linesArr, "") + if text == "" { + text = "\n" // If no params exist, don't send "not found," but an empty file. We know the profile exists. + } + hdr := makeHdrComment(opt.HdrComment) + text = hdr + text + + return Cfg{ + Text: text, + ContentType: ContentTypeCacheDotConfig, + LineComment: LineCommentCacheDotConfig, + Warnings: warnings, + }, nil +} + +// ProfileDS struct for filtered delivery services. +type ProfileDS struct { + Name string + Type tc.DSType + OriginFQDN string +} + +// GetProfileDSes filters delivery services and return delivery services with valid type and non-empty FQDN. +func GetProfileDSes(server *Server, + servers []Server, + deliveryServices []DeliveryService, + deliveryServiceServers []DeliveryServiceServer, +) ([]ProfileDS, []string) { + warnings := make([]string, 0) profileServerIDsMap := map[int]struct{}{} for _, sv := range servers { if len(sv.ProfileNames) == 0 { @@ -97,7 +154,7 @@ func makeCacheDotConfigEdge( dsIDs[dss.DeliveryService] = struct{}{} } - profileDSes := []profileDS{} + profileDSes := []ProfileDS{} for _, ds := range deliveryServices { if ds.ID == nil { warnings = append(warnings, "deliveryservices had ds with nil id, skipping!") @@ -110,6 +167,10 @@ func makeCacheDotConfigEdge( if ds.OrgServerFQDN == nil { continue // this is normal for steering and anymap dses } + if ds.XMLID == nil || *ds.XMLID == "" { + warnings = append(warnings, "got ds with missing XMLID, skipping!") + continue + } if *ds.Type == tc.DSTypeInvalid { warnings = append(warnings, "deliveryservices had ds with invalid type, skipping!") continue @@ -121,73 +182,31 @@ func makeCacheDotConfigEdge( if _, ok := dsIDs[*ds.ID]; !ok && ds.Topology == nil { continue } - origin := *ds.OrgServerFQDN - profileDSes = append(profileDSes, profileDS{Type: *ds.Type, OriginFQDN: &origin}) + profileDSes = append(profileDSes, ProfileDS{Name: *ds.XMLID, Type: *ds.Type, OriginFQDN: *ds.OrgServerFQDN}) } - lines := map[string]struct{}{} // use a "set" for lines, to avoid duplicates, since we're looking up by profile - for _, ds := range profileDSes { - if ds.Type != tc.DSTypeHTTPNoCache { - continue - } - if ds.OriginFQDN == nil || *ds.OriginFQDN == "" { - warnings = append(warnings, "profileCacheDotConfig ds has no origin fqdn, skipping!") // TODO add ds name to data loaded, to put it in the error here? - continue - } - originFQDN, originPort := getHostPortFromURI(*ds.OriginFQDN) - if originPort != "" { - l := "dest_domain=" + originFQDN + " port=" + originPort + " scheme=http action=never-cache\n" - lines[l] = struct{}{} - } else { - l := "dest_domain=" + originFQDN + " scheme=http action=never-cache\n" - lines[l] = struct{}{} - } - } - - linesArr := []string{} - for line, _ := range lines { - linesArr = append(linesArr, line) - } - sort.Strings(linesArr) - text := strings.Join(linesArr, "") - if text == "" { - text = "\n" // If no params exist, don't send "not found," but an empty file. We know the profile exists. - } - hdr := makeHdrComment(opt.HdrComment) - text = hdr + text - - return Cfg{ - Text: text, - ContentType: ContentTypeCacheDotConfig, - LineComment: LineCommentCacheDotConfig, - Warnings: warnings, - }, nil -} - -type profileDS struct { - Type tc.DSType - OriginFQDN *string + return profileDSes, warnings } // dsesToProfileDSes is a helper function to convert a []tc.DeliveryServiceNullable to []ProfileDS. // Note this does not check for nil values. If any DeliveryService's Type or OrgServerFQDN may be nil, the returned ProfileDS should be checked for DSTypeInvalid and nil, respectively. -func dsesToProfileDSes(dses []tc.DeliveryServiceNullable) []profileDS { - pdses := []profileDS{} +func dsesToProfileDSes(dses []tc.DeliveryServiceNullable) []ProfileDS { + pdses := []ProfileDS{} for _, ds := range dses { - pds := profileDS{} + pds := ProfileDS{} if ds.Type != nil { pds.Type = *ds.Type } if ds.OrgServerFQDN != nil && *ds.OrgServerFQDN != "" { - org := *ds.OrgServerFQDN - pds.OriginFQDN = &org + pds.OriginFQDN = *ds.OrgServerFQDN } pdses = append(pdses, pds) } return pdses } -func getHostPortFromURI(uriStr string) (string, string) { +// GetHostPortFromURI strips HTTP(s) scheme and path and return host with port (if found). +func GetHostPortFromURI(uriStr string) (string, string) { originFQDN := uriStr originFQDN = strings.TrimPrefix(originFQDN, "http://") originFQDN = strings.TrimPrefix(originFQDN, "https://") diff --git a/lib/varnishcfg/cache_control.go b/lib/varnishcfg/cache_control.go new file mode 100644 index 0000000000..851c0ff919 --- /dev/null +++ b/lib/varnishcfg/cache_control.go @@ -0,0 +1,57 @@ +package varnishcfg + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "fmt" + "strings" + + "github.com/apache/trafficcontrol/lib/go-atscfg" + "github.com/apache/trafficcontrol/lib/go-tc" +) + +func (v VCLBuilder) configureUncacheableDSes(vclFile *vclFile, profileDSes []atscfg.ProfileDS) []string { + warnings := make([]string, 0) + + uncacheableHostsConditions := make([]string, 0) + for _, ds := range profileDSes { + if ds.Type != tc.DSTypeHTTPNoCache { + continue + } + + if ds.OriginFQDN == "" { + warnings = append(warnings, fmt.Sprintf("ds %s has no origin fqdn, skipping!", ds.Name)) + continue + } + + host, _ := atscfg.GetHostPortFromURI(ds.OriginFQDN) + uncacheableHostsConditions = append(uncacheableHostsConditions, fmt.Sprintf(`bereq.http.host == "%s"`, host)) + } + if len(uncacheableHostsConditions) == 0 { + return warnings + } + berespLines := make([]string, 0) + berespLines = append(berespLines, fmt.Sprintf(`if (%s) {`, strings.Join(uncacheableHostsConditions, " || "))) + berespLines = append(berespLines, fmt.Sprint(` set beresp.uncacheable = true;`)) + berespLines = append(berespLines, fmt.Sprint(`}`)) + vclFile.subroutines["vcl_backend_response"] = append(vclFile.subroutines["vcl_backend_response"], berespLines...) + + return warnings +} diff --git a/lib/varnishcfg/cache_control_test.go b/lib/varnishcfg/cache_control_test.go new file mode 100644 index 0000000000..beb57d6cb4 --- /dev/null +++ b/lib/varnishcfg/cache_control_test.go @@ -0,0 +1,90 @@ +package varnishcfg + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import ( + "reflect" + "testing" + + "github.com/apache/trafficcontrol/cache-config/t3cutil" + "github.com/apache/trafficcontrol/lib/go-atscfg" + "github.com/apache/trafficcontrol/lib/go-tc" +) + +func TestConfigureUncacheableDSes(t *testing.T) { + testCases := []struct { + name string + dses []atscfg.ProfileDS + expectedSubroutines map[string][]string + }{ + { + name: "no uncacheable DSes", + dses: []atscfg.ProfileDS{ + {Name: "ds1", Type: tc.DSTypeHTTP, OriginFQDN: "ds1.example.com"}, + {Name: "ds2", Type: tc.DSTypeHTTP, OriginFQDN: "ds2.example.com"}, + }, + expectedSubroutines: make(map[string][]string), + }, + { + name: "single uncacheable DS", + dses: []atscfg.ProfileDS{ + {Name: "ds1", Type: tc.DSTypeHTTP, OriginFQDN: "ds1.example.com"}, + {Name: "ds2", Type: tc.DSTypeHTTPNoCache, OriginFQDN: "ds2.example.com"}, + }, + expectedSubroutines: map[string][]string{ + "vcl_backend_response": { + `if (bereq.http.host == "ds2.example.com") {`, + ` set beresp.uncacheable = true;`, + `}`, + }, + }, + }, + { + name: "multiple uncacheable DS", + dses: []atscfg.ProfileDS{ + {Name: "ds1", Type: tc.DSTypeHTTP, OriginFQDN: "ds1.example.com"}, + {Name: "ds2", Type: tc.DSTypeHTTPNoCache, OriginFQDN: "ds2.example.com"}, + {Name: "ds3", Type: tc.DSTypeHTTPNoCache, OriginFQDN: "ds3.example.com"}, + {Name: "ds4", Type: tc.DSTypeHTTPNoCache, OriginFQDN: "ds4.example.com"}, + {Name: "ds5", Type: tc.DSTypeDNS, OriginFQDN: "ds5.example.com"}, + }, + expectedSubroutines: map[string][]string{ + "vcl_backend_response": { + `if (bereq.http.host == "ds2.example.com" || bereq.http.host == "ds3.example.com" || bereq.http.host == "ds4.example.com") {`, + ` set beresp.uncacheable = true;`, + `}`, + }, + }, + }, + } + for _, tC := range testCases { + t.Run(tC.name, func(t *testing.T) { + vb := NewVCLBuilder(&t3cutil.ConfigData{}) + vclFile := newVCLFile(defaultVCLVersion) + warnings := vb.configureUncacheableDSes(&vclFile, tC.dses) + if len(warnings) != 0 { + t.Errorf("got unexpected warnings %v", warnings) + } + if !reflect.DeepEqual(vclFile.subroutines, tC.expectedSubroutines) { + t.Errorf("got %v want %v", vclFile.subroutines, tC.expectedSubroutines) + } + }) + } +} diff --git a/lib/varnishcfg/vclbuilder.go b/lib/varnishcfg/vclbuilder.go index 70c89de9c7..480aaa686e 100644 --- a/lib/varnishcfg/vclbuilder.go +++ b/lib/varnishcfg/vclbuilder.go @@ -75,8 +75,18 @@ func (vb *VCLBuilder) BuildVCLFile() (string, []string, error) { if err != nil { return "", nil, fmt.Errorf("(warnings: %s) %w", strings.Join(warnings, ", "), err) } + profileDSes, dsWarnings := atscfg.GetProfileDSes( + vb.toData.Server, + vb.toData.Servers, + vb.toData.DeliveryServices, + vb.toData.DeliveryServiceServers, + ) + warnings = append(warnings, dsWarnings...) + cacheWarnings := vb.configureUncacheableDSes(&v, profileDSes) + warnings = append(warnings, cacheWarnings...) dirWarnings, err := vb.configureDirectors(&v, parents) warnings = append(warnings, dirWarnings...) + return fmt.Sprint(v), warnings, err }