4.1.x Backport: Add ORT/atstccfg Deterministic Config Generation (#4557)

lib/go-atscfg/astatsdotconfig.go

@@ -23,6 +23,7 @@ const AstatsSeparator = "="
 const AstatsFileName = "astats.config"
 
 const ContentTypeAstatsDotConfig = ContentTypeTextASCII
+const LineCommentAstatsDotConfig = LineCommentHash
 
 func MakeAStatsDotConfig(
 	profileName string,

lib/go-atscfg/atscfg.go

@@ -21,6 +21,7 @@ package atscfg
 
 import (
 	"errors"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -36,6 +37,8 @@ const HeaderCommentDateFormat = "Mon Jan 2 15:04:05 MST 2006"
 
 const ContentTypeTextASCII = `text/plain; charset=us-ascii`
 
+const LineCommentHash = "#"
+
 type ServerCapability string
 
 type ServerInfo struct {
@@ -103,10 +106,14 @@ func GenericProfileConfig(
 	separator string,
 ) string {
 	text := ""
+
+	lines := []string{}
 	for name, val := range paramData {
 		name = trimParamUnderscoreNumSuffix(name)
-		text += name + separator + val + "\n"
+		lines = append(lines, name+separator+val+"\n")
 	}
+	sort.Strings(lines)
+	text = strings.Join(lines, "")
 	return text
 }
 

lib/go-atscfg/atsdotrules.go

@@ -24,6 +24,7 @@ import (
 )
 
 const ContentTypeATSDotRules = ContentTypeTextASCII
+const LineCommentATSDotRules = LineCommentHash
 
 func MakeATSDotRules(
 	profileName string,

lib/go-atscfg/bgfetchdotconfig.go

@@ -24,6 +24,7 @@ import (
 )
 
 const ContentTypeBGFetchDotConfig = ContentTypeTextASCII
+const LineCommentBGFetchDotConfig = LineCommentHash
 
 func MakeBGFetchDotConfig(
 	cdnName tc.CDNName,

lib/go-atscfg/cachedotconfig.go

@@ -20,13 +20,15 @@ package atscfg
  */
 
 import (
+	"sort"
 	"strings"
 
 	"github.com/apache/trafficcontrol/lib/go-log"
 	"github.com/apache/trafficcontrol/lib/go-tc"
 )
 
 const ContentTypeCacheDotConfig = ContentTypeTextASCII
+const LineCommentCacheDotConfig = LineCommentHash
 
 type ProfileDS struct {
 	Type       tc.DSType
@@ -60,10 +62,12 @@ func MakeCacheDotConfig(
 		}
 	}
 
-	text := ""
+	linesArr := []string{}
 	for line, _ := range lines {
-		text += line
+		linesArr = append(linesArr, line)
 	}
+	sort.Strings(linesArr)
+	text := strings.Join(linesArr, "")
 	if text == "" {
 		text = "\n" // If no params exist, don't send "not found," but an empty file. We know the profile exists.
 	}

lib/go-atscfg/cacheurldotconfig.go

@@ -27,6 +27,7 @@ import (
 )
 
 const ContentTypeCacheURLDotConfig = ContentTypeTextASCII
+const LineCommentCacheURLDotConfig = LineCommentHash
 
 type CacheURLDS struct {
 	OrgServerFQDN string

lib/go-atscfg/chkconfig.go

@@ -21,19 +21,32 @@ package atscfg
 
 import (
 	"encoding/json"
+	"sort"
 
 	"github.com/apache/trafficcontrol/lib/go-log"
 )
 
 const ChkconfigFileName = `chkconfig`
 const ChkconfigParamConfigFile = `chkconfig`
 const ContentTypeChkconfig = ContentTypeTextASCII
+const LineCommentChkconfig = LineCommentHash
 
 type ChkConfigEntry struct {
 	Name string `json:"name"`
 	Val  string `json:"value"`
 }
 
+type ChkConfigEntries []ChkConfigEntry
+
+func (e ChkConfigEntries) Len() int { return len(e) }
+func (e ChkConfigEntries) Less(i, j int) bool {
+	if e[i].Name != e[j].Name {
+		return e[i].Name < e[j].Name
+	}
+	return e[i].Val < e[j].Val
+}
+func (e ChkConfigEntries) Swap(i, j int) { e[i], e[j] = e[j], e[i] }
+
 // MakeChkconfig returns the 'chkconfig' ATS config file endpoint.
 // This is a JSON object, and should be served with an 'application/json' Content-Type.
 func MakeChkconfig(
@@ -46,6 +59,9 @@ func MakeChkconfig(
 			chkconfig = append(chkconfig, ChkConfigEntry{Name: name, Val: val})
 		}
 	}
+
+	sort.Sort(ChkConfigEntries(chkconfig))
+
 	bts, err := json.Marshal(&chkconfig)
 	if err != nil {
 		// should never happen

lib/go-atscfg/dropqstringdotconfig.go

@@ -22,6 +22,7 @@ package atscfg
 const DropQStringDotConfigFileName = "drop_qstring.config"
 const DropQStringDotConfigParamName = "content"
 const ContentTypeDropQStringDotConfig = ContentTypeTextASCII
+const LineCommentDropQStringDotConfig = LineCommentHash
 
 func MakeDropQStringDotConfig(
 	profileName string,

lib/go-atscfg/facts.go

@@ -20,6 +20,7 @@ package atscfg
  */
 
 const ContentType12MFacts = ContentTypeTextASCII
+const LineComment12MFacts = LineCommentHash
 
 func Make12MFacts(
 	profileName string,

lib/go-atscfg/headerrewritedotconfig.go

@@ -31,6 +31,7 @@ import (
 
 const HeaderRewritePrefix = "hdr_rw_"
 const ContentTypeHeaderRewriteDotConfig = ContentTypeTextASCII
+const LineCommentHeaderRewriteDotConfig = LineCommentHash
 
 const MaxOriginConnectionsNoMax = 0 // 0 indicates no limit on origin connections
 

lib/go-atscfg/hostingdotconfig.go

@@ -20,6 +20,7 @@ package atscfg
  */
 
 import (
+	"sort"
 	"strconv"
 	"strings"
 
@@ -29,6 +30,7 @@ import (
 const HostingConfigFileName = `hosting.config`
 const HostingConfigParamConfigFile = `storage.config`
 const ContentTypeHostingDotConfig = ContentTypeTextASCII
+const LineCommentHostingDotConfig = LineCommentHash
 
 const ParamDrivePrefix = "Drive_Prefix"
 const ParamRAMDrivePrefix = "RAM_Drive_Prefix"
@@ -42,6 +44,7 @@ func MakeHostingDotConfig(
 ) string {
 	text := GenericHeaderComment(string(serverName), toToolName, toURL)
 
+	lines := []string{}
 	if _, ok := params[ParamRAMDrivePrefix]; ok {
 		nextVolume := 1
 		if _, ok := params[ParamDrivePrefix]; ok {
@@ -60,10 +63,14 @@ func MakeHostingDotConfig(
 			seenOrigins[origin] = struct{}{}
 			origin = strings.TrimPrefix(origin, `http://`)
 			origin = strings.TrimPrefix(origin, `https://`)
-			text += `hostname=` + origin + ` volume=` + strconv.Itoa(ramVolume) + "\n"
+			lines = append(lines, `hostname=`+origin+` volume=`+strconv.Itoa(ramVolume)+"\n")
 		}
 	}
 	diskVolume := 1 // note this will actually be the RAM (RAM_Drive_Prefix) volume if there is no Drive_Prefix parameter.
-	text += `hostname=*   volume=` + strconv.Itoa(diskVolume) + "\n"
+
+	lines = append(lines, `hostname=*   volume=`+strconv.Itoa(diskVolume)+"\n")
+
+	sort.Strings(lines)
+	text += strings.Join(lines, "")
 	return text
 }

lib/go-atscfg/ipallowdotconfig.go

@@ -21,6 +21,7 @@ package atscfg
 
 import (
 	"net"
+	"sort"
 	"strconv"
 	"strings"
 
@@ -31,13 +32,28 @@ import (
 
 const IPAllowConfigFileName = `ip_allow.config`
 const ContentTypeIPAllowDotConfig = ContentTypeTextASCII
+const LineCommentIPAllowDotConfig = LineCommentHash
 
 type IPAllowData struct {
 	Src    string
 	Action string
 	Method string
 }
 
+type IPAllowDatas []IPAllowData
+
+func (is IPAllowDatas) Len() int      { return len(is) }
+func (is IPAllowDatas) Swap(i, j int) { is[i], is[j] = is[j], is[i] }
+func (is IPAllowDatas) Less(i, j int) bool {
+	if is[i].Src != is[j].Src {
+		return is[i].Src < is[j].Src
+	}
+	if is[i].Action != is[j].Action {
+		return is[i].Action < is[j].Action
+	}
+	return is[i].Method < is[j].Method
+}
+
 const ParamPurgeAllowIP = "purge_allow_ip"
 const ParamCoalesceMaskLenV4 = "coalesce_masklen_v4"
 const ParamCoalesceNumberV4 = "coalesce_number_v4"
@@ -233,6 +249,9 @@ func MakeIPAllowDotConfig(
 			Method: MethodAll,
 		})
 
+		// order matters, so sort before adding the denys
+		sort.Sort(IPAllowDatas(ipAllowData))
+
 		// end with a deny
 		ipAllowData = append(ipAllowData, IPAllowData{
 			Src:    `0.0.0.0-255.255.255.255`,

lib/go-atscfg/loggingdotconfig.go

@@ -30,6 +30,7 @@ const MaxLogObjects = 10
 
 const LoggingFileName = "logging.config"
 const ContentTypeLoggingDotConfig = ContentTypeTextASCII
+const LineCommentLoggingDotConfig = LineCommentHash
 
 // MakeStorageDotConfig creates storage.config for a given ATS Profile.
 // The paramData is the map of parameter names to values, for all parameters assigned to the given profile, with the config_file "storage.config".

lib/go-atscfg/loggingdotyaml.go

@@ -28,6 +28,7 @@ import (
 
 const LoggingYAMLFileName = "logging.yaml"
 const ContentTypeLoggingDotYAML = "application/yaml; charset=us-ascii" // Note YAML has no IANA standard mime type. This is one of several common usages, and is likely to be the standardized value. If you're reading this, please check IANA to see if YAML has been added, and change this to the IANA definition if so. Also note we include 'charset=us-ascii' because YAML is commonly UTF-8, but ATS is likely to be unable to handle UTF.
+const LineCommentLoggingDotYAML = LineCommentHash
 
 func MakeLoggingDotYAML(
 	profileName string,

lib/go-atscfg/logsdotxml.go

@@ -27,12 +27,18 @@ import (
 const LogsXMLFileName = "logs_xml.config"
 const ContentTypeLogsDotXML = `text/xml`
 
+const LineCommentLogsDotXML = `<!--`
+
 func MakeLogsXMLDotConfig(
 	profileName string,
 	paramData map[string]string, // GetProfileParamData(tx, profile.ID, LoggingYAMLFileName)
 	toToolName string, // tm.toolname global parameter (TODO: cache itself?)
 	toURL string, // tm.url global parameter (TODO: cache itself?)
 ) string {
+
+	// Note LineCommentLogsDotXML must be a single-line comment!
+	// But this file doesn't have a single-line format, so we use <!-- for the header and promise it's on a single line
+	// Note! if this file is ever changed to have multi-line comments, LineCommentLogsDotXML will have to be changed to the empty string.
 	hdrComment := GenericHeaderComment(profileName, toToolName, toURL)
 	hdrComment = strings.Replace(hdrComment, `# `, ``, -1)
 	hdrComment = strings.Replace(hdrComment, "\n", ``, -1)

lib/go-atscfg/packages.go

@@ -21,6 +21,7 @@ package atscfg
 
 import (
 	"encoding/json"
+	"sort"
 
 	"github.com/apache/trafficcontrol/lib/go-log"
 )
@@ -29,12 +30,24 @@ const PackagesFileName = `packages`
 const PackagesParamConfigFile = `package`
 
 const ContentTypePackages = ContentTypeTextASCII
+const LineCommentPackages = ""
 
 type Package struct {
 	Name    string `json:"name"`
 	Version string `json:"version"`
 }
 
+type Packages []Package
+
+func (ps Packages) Len() int { return len(ps) }
+func (ps Packages) Less(i, j int) bool {
+	if ps[i].Name != ps[j].Name {
+		return ps[i].Name < ps[j].Name
+	}
+	return ps[i].Version < ps[j].Version
+}
+func (ps Packages) Swap(i, j int) { ps[i], ps[j] = ps[j], ps[i] }
+
 // MakePackages returns the 'packages' ATS config file endpoint.
 // This is a JSON object, and should be served with an 'application/json' Content-Type.
 func MakePackages(
@@ -46,6 +59,7 @@ func MakePackages(
 			packages = append(packages, Package{Name: name, Version: version})
 		}
 	}
+	sort.Sort(Packages(packages))
 	bts, err := json.Marshal(&packages)
 	if err != nil {
 		// should never happen

lib/go-atscfg/parentdotconfig.go

@@ -32,6 +32,7 @@ import (
 )
 
 const ContentTypeParentDotConfig = ContentTypeTextASCII
+const LineCommentParentDotConfig = LineCommentHash
 
 const ParentConfigParamQStringHandling = "psel.qstring_handling"
 const ParentConfigParamMSOAlgorithm = "mso.algorithm"
@@ -170,8 +171,7 @@ func MakeParentDotConfig(
 	serverParams map[string]string, // getParentConfigServerProfileParams(serverID)
 	parentInfos map[OriginHost][]ParentInfo, // getParentInfo(profileID, parentCachegroupID, secondaryParentCachegroupID)
 ) string {
-
-	// parentInfos := makeParentInfo(serverInfo)
+	sort.Sort(ParentConfigDSTopLevelSortByName(parentConfigDSes))
 
 	nameVersionStr := GetNameVersionStringFromToolNameAndURL(toToolName, toURL)
 	hdr := HeaderCommentWithTOVersionStr(serverInfo.HostName, nameVersionStr)
@@ -257,8 +257,6 @@ func MakeParentDotConfig(
 		roundRobin := `round_robin=consistent_hash`
 		goDirect := `go_direct=false`
 
-		sort.Sort(ParentConfigDSTopLevelSortByName(parentConfigDSes))
-
 		for _, ds := range parentConfigDSes {
 			parents, secondaryParents := getParentStrs(ds, parentInfos[DeliveryServicesAllParentsKey], atsMajorVer)
 

lib/go-atscfg/plugindotconfig.go

@@ -22,6 +22,7 @@ package atscfg
 const PluginSeparator = " "
 const PluginFileName = "plugin.config"
 const ContentTypePluginDotConfig = ContentTypeTextASCII
+const LineCommentPluginDotConfig = LineCommentHash
 
 func MakePluginDotConfig(
 	profileName string,

lib/go-atscfg/recordsdotconfig.go

@@ -26,6 +26,7 @@ import (
 const RecordsSeparator = " "
 const RecordsFileName = "records.config"
 const ContentTypeRecordsDotConfig = ContentTypeTextASCII
+const LineCommentRecordsDotConfig = LineCommentHash
 
 func MakeRecordsDotConfig(
 	profileName string,

lib/go-atscfg/regexremapdotconfig.go

@@ -27,6 +27,7 @@ import (
 )
 
 const ContentTypeRegexRemapDotConfig = ContentTypeTextASCII
+const LineCommentRegexRemapDotConfig = LineCommentHash
 
 type CDNDS struct {
 	OrgServerFQDN string

lib/go-atscfg/regexrevalidatedotconfig.go

@@ -41,6 +41,7 @@ const JobKeywordPurge = "PURGE"
 const RegexRevalidateMinTTL = time.Hour
 
 const ContentTypeRegexRevalidateDotConfig = ContentTypeTextASCII
+const LineCommentRegexRevalidateDotConfig = LineCommentHash
 
 type Job struct {
 	AssetURL string
@@ -71,6 +72,7 @@ func MakeRegexRevalidateDotConfig(
 
 	maxDays := DefaultMaxRevalDurationDays
 	if maxDaysStrs := params[RegexRevalidateMaxRevalDurationDaysParamName]; len(maxDaysStrs) > 0 {
+		sort.Strings(maxDaysStrs)
 		if maxDays, err = strconv.Atoi(maxDaysStrs[0]); err != nil { // just use the first, if there were multiple params
 			log.Warnln("making regex revalidate config: max days param '" + maxDaysStrs[0] + "' is not an integer, using default value!")
 			maxDays = DefaultMaxRevalDurationDays