From 2e26537a8e0e9a83d9139d556f7192b5fdad1fb0 Mon Sep 17 00:00:00 2001 From: Robert Butts Date: Tue, 24 Mar 2020 16:01:30 -0600 Subject: [PATCH 1/5] Make ORT gen deterministic --- lib/go-atscfg/atscfg.go | 7 ++++- lib/go-atscfg/cachedotconfig.go | 7 +++-- lib/go-atscfg/chkconfig.go | 15 +++++++++ lib/go-atscfg/hostingdotconfig.go | 10 ++++-- lib/go-atscfg/ipallowdotconfig.go | 18 +++++++++++ lib/go-atscfg/packages.go | 13 ++++++++ lib/go-atscfg/parentdotconfig.go | 5 +-- lib/go-atscfg/regexrevalidatedotconfig.go | 1 + lib/go-atscfg/servercachedotconfig.go | 10 ++++-- lib/go-atscfg/sslmulticertdotconfig.go | 9 ++++-- lib/go-atscfg/unknownconfig.go | 7 +++-- lib/go-atscfg/urlsigconfig.go | 13 ++++++-- traffic_ops/ort/atstccfg/atstccfg.go | 3 ++ .../ort/atstccfg/cfgfile/astatsdotconfig.go | 9 ++++++ .../ort/atstccfg/cfgfile/atsdotrules.go | 9 ++++++ traffic_ops/ort/atstccfg/cfgfile/cfgfile.go | 8 +++-- traffic_ops/ort/atstccfg/cfgfile/meta.go | 5 +-- .../ort/atstccfg/cfgfile/remapdotconfig.go | 31 ++++++++++--------- .../ort/atstccfg/cfgfile/sysctldotconf.go | 13 +------- .../ort/atstccfg/cfgfile/urlsigconfig.go | 12 +------ traffic_ops/ort/atstccfg/config/config.go | 12 +++++++ 21 files changed, 155 insertions(+), 62 deletions(-) diff --git a/lib/go-atscfg/atscfg.go b/lib/go-atscfg/atscfg.go index 2e38e4ea11..3e1eaa9d58 100644 --- a/lib/go-atscfg/atscfg.go +++ b/lib/go-atscfg/atscfg.go @@ -21,6 +21,7 @@ package atscfg import ( "errors" + "sort" "strconv" "strings" "time" @@ -103,10 +104,14 @@ func GenericProfileConfig( separator string, ) string { text := "" + + lines := []string{} for name, val := range paramData { name = trimParamUnderscoreNumSuffix(name) - text += name + separator + val + "\n" + lines = append(lines, name+separator+val+"\n") } + sort.Strings(lines) + text = strings.Join(lines, "") return text } diff --git a/lib/go-atscfg/cachedotconfig.go b/lib/go-atscfg/cachedotconfig.go index ba95e0e234..9de85e3ef7 100644 --- a/lib/go-atscfg/cachedotconfig.go +++ b/lib/go-atscfg/cachedotconfig.go @@ -20,6 +20,7 @@ package atscfg */ import ( + "sort" "strings" "github.com/apache/trafficcontrol/lib/go-log" @@ -60,10 +61,12 @@ func MakeCacheDotConfig( } } - text := "" + linesArr := []string{} for line, _ := range lines { - text += line + linesArr = append(linesArr, line) } + sort.Strings(linesArr) + text := strings.Join(linesArr, "") if text == "" { text = "\n" // If no params exist, don't send "not found," but an empty file. We know the profile exists. } diff --git a/lib/go-atscfg/chkconfig.go b/lib/go-atscfg/chkconfig.go index 581ca38000..6b4af5d5d1 100644 --- a/lib/go-atscfg/chkconfig.go +++ b/lib/go-atscfg/chkconfig.go @@ -21,6 +21,7 @@ package atscfg import ( "encoding/json" + "sort" "github.com/apache/trafficcontrol/lib/go-log" ) @@ -34,6 +35,17 @@ type ChkConfigEntry struct { Val string `json:"value"` } +type ChkConfigEntries []ChkConfigEntry + +func (e ChkConfigEntries) Len() int { return len(e) } +func (e ChkConfigEntries) Less(i, j int) bool { + if e[i].Name != e[j].Name { + return e[i].Name < e[j].Name + } + return e[i].Val < e[j].Val +} +func (e ChkConfigEntries) Swap(i, j int) { e[i], e[j] = e[j], e[i] } + // MakeChkconfig returns the 'chkconfig' ATS config file endpoint. // This is a JSON object, and should be served with an 'application/json' Content-Type. func MakeChkconfig( @@ -46,6 +58,9 @@ func MakeChkconfig( chkconfig = append(chkconfig, ChkConfigEntry{Name: name, Val: val}) } } + + sort.Sort(ChkConfigEntries(chkconfig)) + bts, err := json.Marshal(&chkconfig) if err != nil { // should never happen diff --git a/lib/go-atscfg/hostingdotconfig.go b/lib/go-atscfg/hostingdotconfig.go index f3f53bc79c..f74e9fa888 100644 --- a/lib/go-atscfg/hostingdotconfig.go +++ b/lib/go-atscfg/hostingdotconfig.go @@ -20,6 +20,7 @@ package atscfg */ import ( + "sort" "strconv" "strings" @@ -42,6 +43,7 @@ func MakeHostingDotConfig( ) string { text := GenericHeaderComment(string(serverName), toToolName, toURL) + lines := []string{} if _, ok := params[ParamRAMDrivePrefix]; ok { nextVolume := 1 if _, ok := params[ParamDrivePrefix]; ok { @@ -60,10 +62,14 @@ func MakeHostingDotConfig( seenOrigins[origin] = struct{}{} origin = strings.TrimPrefix(origin, `http://`) origin = strings.TrimPrefix(origin, `https://`) - text += `hostname=` + origin + ` volume=` + strconv.Itoa(ramVolume) + "\n" + lines = append(lines, `hostname=`+origin+` volume=`+strconv.Itoa(ramVolume)+"\n") } } diskVolume := 1 // note this will actually be the RAM (RAM_Drive_Prefix) volume if there is no Drive_Prefix parameter. - text += `hostname=* volume=` + strconv.Itoa(diskVolume) + "\n" + + lines = append(lines, `hostname=* volume=`+strconv.Itoa(diskVolume)+"\n") + + sort.Strings(lines) + text += strings.Join(lines, "") return text } diff --git a/lib/go-atscfg/ipallowdotconfig.go b/lib/go-atscfg/ipallowdotconfig.go index 74b27c13fa..f5cfb46f30 100644 --- a/lib/go-atscfg/ipallowdotconfig.go +++ b/lib/go-atscfg/ipallowdotconfig.go @@ -21,6 +21,7 @@ package atscfg import ( "net" + "sort" "strconv" "strings" @@ -38,6 +39,20 @@ type IPAllowData struct { Method string } +type IPAllowDatas []IPAllowData + +func (is IPAllowDatas) Len() int { return len(is) } +func (is IPAllowDatas) Swap(i, j int) { is[i], is[j] = is[j], is[i] } +func (is IPAllowDatas) Less(i, j int) bool { + if is[i].Src != is[j].Src { + return is[i].Src < is[j].Src + } + if is[i].Action != is[j].Action { + return is[i].Action < is[j].Action + } + return is[i].Method < is[j].Method +} + const ParamPurgeAllowIP = "purge_allow_ip" const ParamCoalesceMaskLenV4 = "coalesce_masklen_v4" const ParamCoalesceNumberV4 = "coalesce_number_v4" @@ -233,6 +248,9 @@ func MakeIPAllowDotConfig( Method: MethodAll, }) + // order matters, so sort before adding the denys + sort.Sort(IPAllowDatas(ipAllowData)) + // end with a deny ipAllowData = append(ipAllowData, IPAllowData{ Src: `0.0.0.0-255.255.255.255`, diff --git a/lib/go-atscfg/packages.go b/lib/go-atscfg/packages.go index b675072b9a..d6e8712861 100644 --- a/lib/go-atscfg/packages.go +++ b/lib/go-atscfg/packages.go @@ -21,6 +21,7 @@ package atscfg import ( "encoding/json" + "sort" "github.com/apache/trafficcontrol/lib/go-log" ) @@ -35,6 +36,17 @@ type Package struct { Version string `json:"version"` } +type Packages []Package + +func (ps Packages) Len() int { return len(ps) } +func (ps Packages) Less(i, j int) bool { + if ps[i].Name != ps[j].Name { + return ps[i].Name < ps[j].Name + } + return ps[i].Version < ps[j].Version +} +func (ps Packages) Swap(i, j int) { ps[i], ps[j] = ps[j], ps[i] } + // MakePackages returns the 'packages' ATS config file endpoint. // This is a JSON object, and should be served with an 'application/json' Content-Type. func MakePackages( @@ -46,6 +58,7 @@ func MakePackages( packages = append(packages, Package{Name: name, Version: version}) } } + sort.Sort(Packages(packages)) bts, err := json.Marshal(&packages) if err != nil { // should never happen diff --git a/lib/go-atscfg/parentdotconfig.go b/lib/go-atscfg/parentdotconfig.go index a80a7e20d6..c0cc3a9691 100644 --- a/lib/go-atscfg/parentdotconfig.go +++ b/lib/go-atscfg/parentdotconfig.go @@ -170,8 +170,7 @@ func MakeParentDotConfig( serverParams map[string]string, // getParentConfigServerProfileParams(serverID) parentInfos map[OriginHost][]ParentInfo, // getParentInfo(profileID, parentCachegroupID, secondaryParentCachegroupID) ) string { - - // parentInfos := makeParentInfo(serverInfo) + sort.Sort(ParentConfigDSTopLevelSortByName(parentConfigDSes)) nameVersionStr := GetNameVersionStringFromToolNameAndURL(toToolName, toURL) hdr := HeaderCommentWithTOVersionStr(serverInfo.HostName, nameVersionStr) @@ -257,8 +256,6 @@ func MakeParentDotConfig( roundRobin := `round_robin=consistent_hash` goDirect := `go_direct=false` - sort.Sort(ParentConfigDSTopLevelSortByName(parentConfigDSes)) - for _, ds := range parentConfigDSes { parents, secondaryParents := getParentStrs(ds, parentInfos[DeliveryServicesAllParentsKey], atsMajorVer) diff --git a/lib/go-atscfg/regexrevalidatedotconfig.go b/lib/go-atscfg/regexrevalidatedotconfig.go index ad614f937c..39096edb5c 100644 --- a/lib/go-atscfg/regexrevalidatedotconfig.go +++ b/lib/go-atscfg/regexrevalidatedotconfig.go @@ -71,6 +71,7 @@ func MakeRegexRevalidateDotConfig( maxDays := DefaultMaxRevalDurationDays if maxDaysStrs := params[RegexRevalidateMaxRevalDurationDaysParamName]; len(maxDaysStrs) > 0 { + sort.Strings(maxDaysStrs) if maxDays, err = strconv.Atoi(maxDaysStrs[0]); err != nil { // just use the first, if there were multiple params log.Warnln("making regex revalidate config: max days param '" + maxDaysStrs[0] + "' is not an integer, using default value!") maxDays = DefaultMaxRevalDurationDays diff --git a/lib/go-atscfg/servercachedotconfig.go b/lib/go-atscfg/servercachedotconfig.go index 9d1d29c5db..f436477b5e 100644 --- a/lib/go-atscfg/servercachedotconfig.go +++ b/lib/go-atscfg/servercachedotconfig.go @@ -20,6 +20,7 @@ package atscfg */ import ( + "sort" "strconv" "strings" @@ -39,6 +40,8 @@ func MakeServerCacheDotConfig( ) string { text := GenericHeaderComment(string(serverName), toToolName, toURL) + lines := []string{} + seenOrigins := map[string]struct{}{} for _, ds := range dses { if ds.Type != tc.DSTypeHTTPNoCache { @@ -51,12 +54,13 @@ func MakeServerCacheDotConfig( originFQDN, originPort := GetOriginFQDNAndPort(ds.OrgServerFQDN) if originPort != nil { - text += `dest_domain=` + originFQDN + ` port=` + strconv.Itoa(*originPort) + ` scheme=http action=never-cache` + "\n" + lines = append(lines, `dest_domain=`+originFQDN+` port=`+strconv.Itoa(*originPort)+` scheme=http action=never-cache`+"\n") } else { - text += `dest_domain=` + originFQDN + ` scheme=http action=never-cache` + "\n" + lines = append(lines, `dest_domain=`+originFQDN+` scheme=http action=never-cache`+"\n") } } - return text + sort.Strings(lines) + return text + strings.Join(lines, "") } // TODO unit test diff --git a/lib/go-atscfg/sslmulticertdotconfig.go b/lib/go-atscfg/sslmulticertdotconfig.go index 64cd48bbad..10813231e3 100644 --- a/lib/go-atscfg/sslmulticertdotconfig.go +++ b/lib/go-atscfg/sslmulticertdotconfig.go @@ -20,6 +20,7 @@ package atscfg */ import ( + "sort" "strings" "github.com/apache/trafficcontrol/lib/go-log" @@ -58,15 +59,17 @@ func MakeSSLMultiCertDotConfig( toURL string, // tm.url global parameter (TODO: cache itself?) dses map[tc.DeliveryServiceName]SSLMultiCertDS, ) string { - text := GenericHeaderComment(string(cdnName), toToolName, toURL) + hdr := GenericHeaderComment(string(cdnName), toToolName, toURL) dses = GetSSLMultiCertDotConfigDeliveryServices(dses) + lines := []string{} for dsName, ds := range dses { cerName, keyName := GetSSLMultiCertDotConfigCertAndKeyName(dsName, ds) - text += `ssl_cert_name=` + cerName + "\t" + ` ssl_key_name=` + keyName + "\n" + lines = append(lines, `ssl_cert_name=`+cerName+"\t"+` ssl_key_name=`+keyName+"\n") } - return text + sort.Strings(lines) + return hdr + strings.Join(lines, "") } // GetSSLMultiCertDotConfigCertAndKeyName returns the cert file name and key file name for the given delivery service. diff --git a/lib/go-atscfg/unknownconfig.go b/lib/go-atscfg/unknownconfig.go index 255d3b1a08..fc0b00399d 100644 --- a/lib/go-atscfg/unknownconfig.go +++ b/lib/go-atscfg/unknownconfig.go @@ -20,6 +20,7 @@ package atscfg */ import ( + "sort" "strings" ) @@ -33,7 +34,7 @@ func MakeUnknownConfig( ) string { hdr := GenericHeaderComment(profileName, toToolName, toURL) - text := "" + lines := []string{} for paramName, paramVal := range paramData { if paramName == "header" { if paramVal == "none" { @@ -42,9 +43,11 @@ func MakeUnknownConfig( hdr = paramVal + "\n" } } else { - text += paramVal + "\n" + lines = append(lines, paramVal+"\n") } } + sort.Strings(lines) + text := strings.Join(lines, "") text = strings.Replace(text, "__RETURN__", "\n", -1) return hdr + text } diff --git a/lib/go-atscfg/urlsigconfig.go b/lib/go-atscfg/urlsigconfig.go index 938414a911..7c46e0639f 100644 --- a/lib/go-atscfg/urlsigconfig.go +++ b/lib/go-atscfg/urlsigconfig.go @@ -20,6 +20,7 @@ package atscfg */ import ( + "sort" "strings" "github.com/apache/trafficcontrol/lib/go-tc" @@ -37,14 +38,22 @@ func MakeURLSigConfig( sep := " = " text := hdr + + paramLines := []string{} for paramName, paramVal := range paramData { if len(urlSigKeys) == 0 || !strings.HasPrefix(paramName, "key") { - text += paramName + sep + paramVal + "\n" + paramLines = append(paramLines, paramName+sep+paramVal+"\n") } } + sort.Strings(paramLines) + text += strings.Join(paramLines, "") + keyLines := []string{} for key, val := range urlSigKeys { - text += key + sep + val + "\n" + keyLines = append(keyLines, key+sep+val+"\n") } + sort.Strings(keyLines) + text += strings.Join(keyLines, "") + return text } diff --git a/traffic_ops/ort/atstccfg/atstccfg.go b/traffic_ops/ort/atstccfg/atstccfg.go index ee9ab8764e..a4942384dc 100644 --- a/traffic_ops/ort/atstccfg/atstccfg.go +++ b/traffic_ops/ort/atstccfg/atstccfg.go @@ -50,6 +50,7 @@ package main import ( "fmt" "os" + "sort" "strings" "github.com/apache/trafficcontrol/lib/go-log" @@ -117,6 +118,8 @@ func main() { modifyFilesData := plugin.ModifyFilesData{Cfg: tccfg, TOData: toData, Files: configs} configs = plugins.ModifyFiles(modifyFilesData) + sort.Sort(config.ATSConfigFiles(configs)) + if err := cfgfile.WriteConfigs(configs, os.Stdout); err != nil { log.Errorln("Writing configs for '" + cfg.CacheHostName + "': " + err.Error()) os.Exit(config.ExitCodeErrGeneric) diff --git a/traffic_ops/ort/atstccfg/cfgfile/astatsdotconfig.go b/traffic_ops/ort/atstccfg/cfgfile/astatsdotconfig.go index 32d09b7f7e..8fdbf3a59e 100644 --- a/traffic_ops/ort/atstccfg/cfgfile/astatsdotconfig.go +++ b/traffic_ops/ort/atstccfg/cfgfile/astatsdotconfig.go @@ -21,6 +21,7 @@ package cfgfile import ( "github.com/apache/trafficcontrol/lib/go-atscfg" + "github.com/apache/trafficcontrol/lib/go-log" "github.com/apache/trafficcontrol/traffic_ops/ort/atstccfg/config" ) @@ -34,6 +35,14 @@ func GetConfigFileProfileAstatsDotConfig(toData *config.TOData) (string, string, if param.Name == "location" { continue } + if val, ok := paramData[param.Name]; ok { + if val < param.Value { + log.Errorln("data error: making astats.config: parameter '" + param.Name + "' had multiple values, ignoring '" + param.Value + "'") + continue + } else { + log.Errorln("data error: making astats.config: parameter '" + param.Name + "' had multiple values, ignoring '" + val + "'") + } + } paramData[param.Name] = param.Value } diff --git a/traffic_ops/ort/atstccfg/cfgfile/atsdotrules.go b/traffic_ops/ort/atstccfg/cfgfile/atsdotrules.go index 40edddffff..eae92fcfd9 100644 --- a/traffic_ops/ort/atstccfg/cfgfile/atsdotrules.go +++ b/traffic_ops/ort/atstccfg/cfgfile/atsdotrules.go @@ -21,6 +21,7 @@ package cfgfile import ( "github.com/apache/trafficcontrol/lib/go-atscfg" + "github.com/apache/trafficcontrol/lib/go-log" "github.com/apache/trafficcontrol/traffic_ops/ort/atstccfg/config" ) @@ -36,6 +37,14 @@ func GetConfigFileProfileATSDotRules(toData *config.TOData) (string, string, err if param.Name == "location" { continue } + if val, ok := paramData[param.Name]; ok { + if val < param.Value { + log.Errorln("data error: making ats.rules: parameter '" + param.Name + "' had multiple values, ignoring '" + param.Value + "'") + continue + } else { + log.Errorln("data error: making ats.rules: parameter '" + param.Name + "' had multiple values, ignoring '" + val + "'") + } + } paramData[param.Name] = param.Value } return atscfg.MakeATSDotRules(toData.Server.Profile, paramData, toData.TOToolName, toData.TOURL), atscfg.ContentTypeATSDotRules, nil diff --git a/traffic_ops/ort/atstccfg/cfgfile/cfgfile.go b/traffic_ops/ort/atstccfg/cfgfile/cfgfile.go index 40b1aaa7e0..7279a108c2 100644 --- a/traffic_ops/ort/atstccfg/cfgfile/cfgfile.go +++ b/traffic_ops/ort/atstccfg/cfgfile/cfgfile.go @@ -396,8 +396,12 @@ func ParamsToMap(params []tc.Parameter) map[string]string { mp := map[string]string{} for _, param := range params { if val, ok := mp[param.Name]; ok { - log.Errorln("config generation got multiple parameters for name '" + param.Name + "' - using '" + val + "'") - continue + if val < param.Value { + log.Errorln("config generation got multiple parameters for name '" + param.Name + "' - ignoring '" + param.Value + "'") + continue + } else { + log.Errorln("config generation got multiple parameters for name '" + param.Name + "' - ignoring '" + val + "'") + } } mp[param.Name] = param.Value } diff --git a/traffic_ops/ort/atstccfg/cfgfile/meta.go b/traffic_ops/ort/atstccfg/cfgfile/meta.go index d4dfd00ca0..655e51307e 100644 --- a/traffic_ops/ort/atstccfg/cfgfile/meta.go +++ b/traffic_ops/ort/atstccfg/cfgfile/meta.go @@ -109,10 +109,7 @@ func GetMeta(toData *config.TOData) (*tc.ATSConfigMetaData, error) { } } - scopeParams := map[string]string{} - for _, param := range toData.ScopeParams { - scopeParams[param.ConfigFile] = param.Value - } + scopeParams := ParamsToMap(toData.ScopeParams) locationParams := map[string]atscfg.ConfigProfileParams{} for _, param := range toData.ServerParams { diff --git a/traffic_ops/ort/atstccfg/cfgfile/remapdotconfig.go b/traffic_ops/ort/atstccfg/cfgfile/remapdotconfig.go index f05124f649..bac5ac95bc 100644 --- a/traffic_ops/ort/atstccfg/cfgfile/remapdotconfig.go +++ b/traffic_ops/ort/atstccfg/cfgfile/remapdotconfig.go @@ -167,21 +167,20 @@ func GetConfigFileServerRemapDotConfig(toData *config.TOData) (string, string, e if paramValue == "STRING __HOSTNAME__" { paramValue = toData.Server.HostName + "." + toData.Server.DomainName // TODO strings.Replace to replace all anywhere, instead of just an exact match? } - serverPackageParamData[paramName] = paramValue - } - cacheURLParams := map[string]string{} - for _, param := range toData.ServerParams { - if param.ConfigFile != atscfg.CacheURLParameterConfigFile { - continue - } - if existingVal, ok := cacheURLParams[param.Name]; ok { - log.Warnln("generating remap.config: server profile '" + toData.Server.Profile + "' cacheurl.config has multiple parameters for '" + param.Name + "' - using '" + existingVal + "' and ignoring the rest!") - continue + if val, ok := serverPackageParamData[paramName]; ok { + if val < paramValue { + log.Errorln("remap config generation got multiple parameters for server package name '" + paramName + "' - ignoring '" + paramValue + "'") + continue + } else { + log.Errorln("config generation got multiple parameters for server package name '" + paramName + "' - ignoring '" + val + "'") + } } - cacheURLParams[param.Name] = param.Value + serverPackageParamData[paramName] = paramValue } + cacheURLParams := ParamsToMap(FilterParams(toData.ServerParams, atscfg.CacheURLParameterConfigFile, "", "", "")) + cacheKeyParamsWithProfiles, err := TCParamsToParamsWithProfiles(toData.CacheKeyParams) if err != nil { return "", "", errors.New("decoding cache key parameter profiles: " + err.Error()) @@ -204,9 +203,13 @@ func GetConfigFileServerRemapDotConfig(toData *config.TOData) (string, string, e if _, ok := dsProfilesCacheKeyConfigParams[dsProfileID]; !ok { dsProfilesCacheKeyConfigParams[dsProfileID] = map[string]string{} } - if _, ok := dsProfilesCacheKeyConfigParams[dsProfileID][param.Name]; ok { - // TODO warn - continue + if val, ok := dsProfilesCacheKeyConfigParams[dsProfileID][param.Name]; ok { + if val < param.Value { + log.Errorln("remap config generation got multiple parameters for name '" + param.Name + "' - ignoring '" + param.Value + "'") + continue + } else { + log.Errorln("remap config generation got multiple parameters for name '" + param.Name + "' - ignoring '" + val + "'") + } } dsProfilesCacheKeyConfigParams[dsProfileID][param.Name] = param.Value } diff --git a/traffic_ops/ort/atstccfg/cfgfile/sysctldotconf.go b/traffic_ops/ort/atstccfg/cfgfile/sysctldotconf.go index 44e9ae5fd8..8cd48754c9 100644 --- a/traffic_ops/ort/atstccfg/cfgfile/sysctldotconf.go +++ b/traffic_ops/ort/atstccfg/cfgfile/sysctldotconf.go @@ -25,17 +25,6 @@ import ( ) func GetConfigFileProfileSysCtlDotConf(toData *config.TOData) (string, string, error) { - paramData := map[string]string{} - // TODO add configFile query param to profile/parameters endpoint, to only get needed data - for _, param := range toData.ServerParams { - if param.ConfigFile != atscfg.SysctlFileName { - continue - } - if param.Name == "location" { - continue - } - paramData[param.Name] = param.Value - } - + paramData := ParamsToMap(FilterParams(toData.ServerParams, atscfg.SysctlFileName, "", "", "location")) return atscfg.MakeSysCtlDotConf(toData.Server.Profile, paramData, toData.TOToolName, toData.TOURL), atscfg.ContentTypeSysctlDotConf, nil } diff --git a/traffic_ops/ort/atstccfg/cfgfile/urlsigconfig.go b/traffic_ops/ort/atstccfg/cfgfile/urlsigconfig.go index f19a32d994..5e3ff2f068 100644 --- a/traffic_ops/ort/atstccfg/cfgfile/urlsigconfig.go +++ b/traffic_ops/ort/atstccfg/cfgfile/urlsigconfig.go @@ -29,17 +29,7 @@ import ( ) func GetConfigFileProfileURLSigConfig(toData *config.TOData, fileName string) (string, string, error) { - paramData := map[string]string{} - // TODO add configFile query param to profile/parameters endpoint, to only get needed data - for _, param := range toData.ServerParams { - if param.ConfigFile != fileName { - continue - } - if param.Name == "location" { - continue - } - paramData[param.Name] = param.Value - } + paramData := ParamsToMap(FilterParams(toData.ServerParams, fileName, "", "", "location")) dsName := GetDSFromURLSigConfigFileName(fileName) if dsName == "" { diff --git a/traffic_ops/ort/atstccfg/config/config.go b/traffic_ops/ort/atstccfg/config/config.go index 1265714745..27a49d8fb1 100644 --- a/traffic_ops/ort/atstccfg/config/config.go +++ b/traffic_ops/ort/atstccfg/config/config.go @@ -201,6 +201,18 @@ type ATSConfigFile struct { ContentType string } +// ATSConfigFiles implements sort.Interface and sorts by the Location and then FileNameOnDisk, i.e. the full file path. +type ATSConfigFiles []ATSConfigFile + +func (fs ATSConfigFiles) Len() int { return len(fs) } +func (fs ATSConfigFiles) Less(i, j int) bool { + if fs[i].Location != fs[j].Location { + return fs[i].Location < fs[j].Location + } + return fs[i].FileNameOnDisk < fs[j].FileNameOnDisk +} +func (fs ATSConfigFiles) Swap(i, j int) { fs[i], fs[j] = fs[j], fs[i] } + // TOData is the Traffic Ops data needed to generate configs. // See each field for details on the data required. // - If a field says 'must', the creation of TOData is guaranteed to do so, and users of the struct may rely on that. From dd8d44bf370bcfcd649db2c041416986f99b994d Mon Sep 17 00:00:00 2001 From: Robert Butts Date: Fri, 27 Mar 2020 08:40:14 -0600 Subject: [PATCH 2/5] Add ORT/atstccfg Line-Comment header Adds a header to the multipart file, with the line comment syntax of the file (if it has one). This lets ORT safely strip line comments with times for diffing. --- lib/go-atscfg/astatsdotconfig.go | 1 + lib/go-atscfg/atscfg.go | 2 + lib/go-atscfg/atsdotrules.go | 1 + lib/go-atscfg/bgfetchdotconfig.go | 1 + lib/go-atscfg/cachedotconfig.go | 1 + lib/go-atscfg/cacheurldotconfig.go | 1 + lib/go-atscfg/chkconfig.go | 1 + lib/go-atscfg/dropqstringdotconfig.go | 1 + lib/go-atscfg/facts.go | 1 + lib/go-atscfg/headerrewritedotconfig.go | 1 + lib/go-atscfg/hostingdotconfig.go | 1 + lib/go-atscfg/ipallowdotconfig.go | 1 + lib/go-atscfg/loggingdotconfig.go | 1 + lib/go-atscfg/loggingdotyaml.go | 1 + lib/go-atscfg/logsdotxml.go | 6 ++ lib/go-atscfg/packages.go | 1 + lib/go-atscfg/parentdotconfig.go | 1 + lib/go-atscfg/plugindotconfig.go | 1 + lib/go-atscfg/recordsdotconfig.go | 1 + lib/go-atscfg/regexremapdotconfig.go | 1 + lib/go-atscfg/regexrevalidatedotconfig.go | 1 + lib/go-atscfg/remapdotconfig.go | 1 + lib/go-atscfg/serverunknown.go | 18 ++++++ lib/go-atscfg/setdscpdotconfig.go | 1 + lib/go-atscfg/sslmulticertdotconfig.go | 1 + lib/go-atscfg/storagedotconfig.go | 1 + lib/go-atscfg/sysctldotconf.go | 1 + lib/go-atscfg/unknownconfig.go | 17 +++++ lib/go-atscfg/urisigningconfig.go | 1 + lib/go-atscfg/urlsigconfig.go | 3 + lib/go-atscfg/volumedotconfig.go | 1 + traffic_ops/ort/atstccfg/cfgfile/all.go | 8 ++- .../ort/atstccfg/cfgfile/astatsdotconfig.go | 4 +- .../ort/atstccfg/cfgfile/atsdotrules.go | 4 +- .../ort/atstccfg/cfgfile/bgfetchdotconfig.go | 4 +- .../ort/atstccfg/cfgfile/cachedotconfig.go | 4 +- .../ort/atstccfg/cfgfile/cacheurldotconfig.go | 6 +- .../ort/atstccfg/cfgfile/cfgfile_test.go | 6 +- traffic_ops/ort/atstccfg/cfgfile/chkconfig.go | 4 +- .../atstccfg/cfgfile/dropqstringdotconfig.go | 4 +- traffic_ops/ort/atstccfg/cfgfile/facts.go | 4 +- .../cfgfile/headerrewritedotconfig.go | 10 +-- .../cfgfile/headerrewritemiddotconfig.go | 10 +-- .../ort/atstccfg/cfgfile/hostingdotconfig.go | 6 +- .../ort/atstccfg/cfgfile/ipallowdotconfig.go | 8 +-- .../ort/atstccfg/cfgfile/loggingdotconfig.go | 4 +- .../ort/atstccfg/cfgfile/loggingdotyaml.go | 4 +- .../ort/atstccfg/cfgfile/logsxmldotconfig.go | 4 +- traffic_ops/ort/atstccfg/cfgfile/packages.go | 4 +- .../ort/atstccfg/cfgfile/parentdotconfig.go | 36 +++++------ .../ort/atstccfg/cfgfile/plugindotconfig.go | 4 +- .../ort/atstccfg/cfgfile/recordsdotconfig.go | 4 +- .../atstccfg/cfgfile/regexremapdotconfig.go | 10 +-- .../cfgfile/regexrevalidatedotconfig.go | 4 +- .../ort/atstccfg/cfgfile/remapdotconfig.go | 24 +++---- traffic_ops/ort/atstccfg/cfgfile/routing.go | 63 ++++++++++--------- .../atstccfg/cfgfile/servercachedotconfig.go | 6 +- .../atstccfg/cfgfile/serverunknownconfig.go | 5 +- .../ort/atstccfg/cfgfile/setdscpdotconfig.go | 4 +- .../atstccfg/cfgfile/sslmulticertdotconfig.go | 4 +- .../ort/atstccfg/cfgfile/storagedotconfig.go | 4 +- .../ort/atstccfg/cfgfile/sysctldotconf.go | 4 +- .../ort/atstccfg/cfgfile/unknownconfig.go | 9 ++- .../ort/atstccfg/cfgfile/urisigningconfig.go | 8 +-- .../ort/atstccfg/cfgfile/urlsigconfig.go | 8 +-- .../ort/atstccfg/cfgfile/volumedotconfig.go | 4 +- traffic_ops/ort/atstccfg/config/config.go | 1 + .../ort/atstccfg/plugin/hello_world.go | 1 + .../ort/atstccfg/plugin/plugin_test.go | 1 + 69 files changed, 229 insertions(+), 145 deletions(-) diff --git a/lib/go-atscfg/astatsdotconfig.go b/lib/go-atscfg/astatsdotconfig.go index 0fbc0bff96..8ea62d5755 100644 --- a/lib/go-atscfg/astatsdotconfig.go +++ b/lib/go-atscfg/astatsdotconfig.go @@ -23,6 +23,7 @@ const AstatsSeparator = "=" const AstatsFileName = "astats.config" const ContentTypeAstatsDotConfig = ContentTypeTextASCII +const LineCommentAstatsDotConfig = LineCommentHash func MakeAStatsDotConfig( profileName string, diff --git a/lib/go-atscfg/atscfg.go b/lib/go-atscfg/atscfg.go index 3e1eaa9d58..8798ed37e4 100644 --- a/lib/go-atscfg/atscfg.go +++ b/lib/go-atscfg/atscfg.go @@ -37,6 +37,8 @@ const HeaderCommentDateFormat = "Mon Jan 2 15:04:05 MST 2006" const ContentTypeTextASCII = `text/plain; charset=us-ascii` +const LineCommentHash = "#" + type ServerCapability string type ServerInfo struct { diff --git a/lib/go-atscfg/atsdotrules.go b/lib/go-atscfg/atsdotrules.go index 5ea8f437f8..286d44edd2 100644 --- a/lib/go-atscfg/atsdotrules.go +++ b/lib/go-atscfg/atsdotrules.go @@ -24,6 +24,7 @@ import ( ) const ContentTypeATSDotRules = ContentTypeTextASCII +const LineCommentATSDotRules = LineCommentHash func MakeATSDotRules( profileName string, diff --git a/lib/go-atscfg/bgfetchdotconfig.go b/lib/go-atscfg/bgfetchdotconfig.go index 348061353a..7629e40a8a 100644 --- a/lib/go-atscfg/bgfetchdotconfig.go +++ b/lib/go-atscfg/bgfetchdotconfig.go @@ -24,6 +24,7 @@ import ( ) const ContentTypeBGFetchDotConfig = ContentTypeTextASCII +const LineCommentBGFetchDotConfig = LineCommentHash func MakeBGFetchDotConfig( cdnName tc.CDNName, diff --git a/lib/go-atscfg/cachedotconfig.go b/lib/go-atscfg/cachedotconfig.go index 9de85e3ef7..7b23107df7 100644 --- a/lib/go-atscfg/cachedotconfig.go +++ b/lib/go-atscfg/cachedotconfig.go @@ -28,6 +28,7 @@ import ( ) const ContentTypeCacheDotConfig = ContentTypeTextASCII +const LineCommentCacheDotConfig = LineCommentHash type ProfileDS struct { Type tc.DSType diff --git a/lib/go-atscfg/cacheurldotconfig.go b/lib/go-atscfg/cacheurldotconfig.go index f6a077c53d..1f48110f21 100644 --- a/lib/go-atscfg/cacheurldotconfig.go +++ b/lib/go-atscfg/cacheurldotconfig.go @@ -27,6 +27,7 @@ import ( ) const ContentTypeCacheURLDotConfig = ContentTypeTextASCII +const LineCommentCacheURLDotConfig = LineCommentHash type CacheURLDS struct { OrgServerFQDN string diff --git a/lib/go-atscfg/chkconfig.go b/lib/go-atscfg/chkconfig.go index 6b4af5d5d1..9d870c90b2 100644 --- a/lib/go-atscfg/chkconfig.go +++ b/lib/go-atscfg/chkconfig.go @@ -29,6 +29,7 @@ import ( const ChkconfigFileName = `chkconfig` const ChkconfigParamConfigFile = `chkconfig` const ContentTypeChkconfig = ContentTypeTextASCII +const LineCommentChkconfig = LineCommentHash type ChkConfigEntry struct { Name string `json:"name"` diff --git a/lib/go-atscfg/dropqstringdotconfig.go b/lib/go-atscfg/dropqstringdotconfig.go index 3c7c59bd5e..68d91897f5 100644 --- a/lib/go-atscfg/dropqstringdotconfig.go +++ b/lib/go-atscfg/dropqstringdotconfig.go @@ -22,6 +22,7 @@ package atscfg const DropQStringDotConfigFileName = "drop_qstring.config" const DropQStringDotConfigParamName = "content" const ContentTypeDropQStringDotConfig = ContentTypeTextASCII +const LineCommentDropQStringDotConfig = LineCommentHash func MakeDropQStringDotConfig( profileName string, diff --git a/lib/go-atscfg/facts.go b/lib/go-atscfg/facts.go index 70102ce2ad..c7e6eb61ba 100644 --- a/lib/go-atscfg/facts.go +++ b/lib/go-atscfg/facts.go @@ -20,6 +20,7 @@ package atscfg */ const ContentType12MFacts = ContentTypeTextASCII +const LineComment12MFacts = LineCommentHash func Make12MFacts( profileName string, diff --git a/lib/go-atscfg/headerrewritedotconfig.go b/lib/go-atscfg/headerrewritedotconfig.go index 083ee91080..34a96aeb14 100644 --- a/lib/go-atscfg/headerrewritedotconfig.go +++ b/lib/go-atscfg/headerrewritedotconfig.go @@ -31,6 +31,7 @@ import ( const HeaderRewritePrefix = "hdr_rw_" const ContentTypeHeaderRewriteDotConfig = ContentTypeTextASCII +const LineCommentHeaderRewriteDotConfig = LineCommentHash const MaxOriginConnectionsNoMax = 0 // 0 indicates no limit on origin connections diff --git a/lib/go-atscfg/hostingdotconfig.go b/lib/go-atscfg/hostingdotconfig.go index f74e9fa888..5df86e9acb 100644 --- a/lib/go-atscfg/hostingdotconfig.go +++ b/lib/go-atscfg/hostingdotconfig.go @@ -30,6 +30,7 @@ import ( const HostingConfigFileName = `hosting.config` const HostingConfigParamConfigFile = `storage.config` const ContentTypeHostingDotConfig = ContentTypeTextASCII +const LineCommentHostingDotConfig = LineCommentHash const ParamDrivePrefix = "Drive_Prefix" const ParamRAMDrivePrefix = "RAM_Drive_Prefix" diff --git a/lib/go-atscfg/ipallowdotconfig.go b/lib/go-atscfg/ipallowdotconfig.go index f5cfb46f30..d2a1744f5c 100644 --- a/lib/go-atscfg/ipallowdotconfig.go +++ b/lib/go-atscfg/ipallowdotconfig.go @@ -32,6 +32,7 @@ import ( const IPAllowConfigFileName = `ip_allow.config` const ContentTypeIPAllowDotConfig = ContentTypeTextASCII +const LineCommentIPAllowDotConfig = LineCommentHash type IPAllowData struct { Src string diff --git a/lib/go-atscfg/loggingdotconfig.go b/lib/go-atscfg/loggingdotconfig.go index fdb24fcc62..0d9f375275 100644 --- a/lib/go-atscfg/loggingdotconfig.go +++ b/lib/go-atscfg/loggingdotconfig.go @@ -30,6 +30,7 @@ const MaxLogObjects = 10 const LoggingFileName = "logging.config" const ContentTypeLoggingDotConfig = ContentTypeTextASCII +const LineCommentLoggingDotConfig = LineCommentHash // MakeStorageDotConfig creates storage.config for a given ATS Profile. // The paramData is the map of parameter names to values, for all parameters assigned to the given profile, with the config_file "storage.config". diff --git a/lib/go-atscfg/loggingdotyaml.go b/lib/go-atscfg/loggingdotyaml.go index 02e39683b7..72ccd66dba 100644 --- a/lib/go-atscfg/loggingdotyaml.go +++ b/lib/go-atscfg/loggingdotyaml.go @@ -28,6 +28,7 @@ import ( const LoggingYAMLFileName = "logging.yaml" const ContentTypeLoggingDotYAML = "application/yaml; charset=us-ascii" // Note YAML has no IANA standard mime type. This is one of several common usages, and is likely to be the standardized value. If you're reading this, please check IANA to see if YAML has been added, and change this to the IANA definition if so. Also note we include 'charset=us-ascii' because YAML is commonly UTF-8, but ATS is likely to be unable to handle UTF. +const LineCommentLoggingDotYAML = LineCommentHash func MakeLoggingDotYAML( profileName string, diff --git a/lib/go-atscfg/logsdotxml.go b/lib/go-atscfg/logsdotxml.go index 991b41186d..72e6525ef9 100644 --- a/lib/go-atscfg/logsdotxml.go +++ b/lib/go-atscfg/logsdotxml.go @@ -27,12 +27,18 @@ import ( const LogsXMLFileName = "logs_xml.config" const ContentTypeLogsDotXML = `text/xml` +const LineCommentLogsDotXML = `