Add Topology ATS Config Generation

CHANGELOG.md

@@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
     - Traffic Ops: Added new topology-based delivery service fields for header rewrites: `firstHeaderRewrite`, `innerHeaderRewrite`, `lastHeaderRewrite`
     - Traffic Ops: Added validation to prohibit assigning caches to topology-based delivery services
     - Traffic Ops: Consider Topologies parentage when queueing or checking server updates
+    - ORT: Added Topologies to Config Generation.
     - Traffic Portal: Added the ability to create, read, update and delete flexible topologies.
     - Traffic Portal: Added the ability to assign topologies to delivery services.
     - Traffic Portal: Added the ability to view all delivery services and cache groups associated with a topology.
@@ -53,6 +54,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - Changed some Traffic Ops Go Client methods to use `DeliveryServiceNullable` inputs and outputs.
 - Changed Traffic Portal to use Traffic Ops API v3
 - Changed ORT Config Generation to be deterministic, which will prevent spurious diffs when nothing actually changed.
+- Changed ORT to find the local ATS config directory and use it when location Parameters don't exist for many required configs, including all Delivery Service files (Header Rewrites, Regex Remap, URL Sig, URI Signing).
 - Changed the access logs in Traffic Ops to now show the route ID with every API endpoint call. The Route ID is appended to the end of the access log line.
 - [Multiple Interface Servers](https://github.com/apache/trafficcontrol/blob/master/blueprints/multi-interface-servers.md)
     - Interface data is constructed from IP Address/Gateway/Netmask (and their IPv6 counterparts) and Interface Name and Interface MTU fields on services. These **MUST** have proper, valid data before attempting to upgrade or the upgrade **WILL** fail. In particular IP fields need to be valid IP addresses/netmasks, and MTU must only be positive integers of at least 1280.

lib/go-atscfg/atscfg.go

@@ -20,12 +20,14 @@ package atscfg
  */
 
 import (
+	"encoding/json"
 	"errors"
 	"sort"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/apache/trafficcontrol/lib/go-log"
 	"github.com/apache/trafficcontrol/lib/go-tc"
 )
 
@@ -39,10 +41,13 @@ const ContentTypeTextASCII = `text/plain; charset=us-ascii`
 
 const LineCommentHash = "#"
 
+type TopologyName string
+
 type ServerCapability string
 
 type ServerInfo struct {
 	CacheGroupID                  int
+	CacheGroupName                string
 	CDN                           tc.CDNName
 	CDNID                         int
 	DomainName                    string
@@ -134,3 +139,158 @@ const ConfigSuffix = ".config"
 func GetConfigFile(prefix string, xmlId string) string {
 	return prefix + xmlId + ConfigSuffix
 }
+
+// topologyIncludesServer returns whether the given topology includes the given server
+func topologyIncludesServer(topology tc.Topology, server tc.Server) bool {
+	for _, node := range topology.Nodes {
+		if node.Cachegroup == server.Cachegroup {
+			return true
+		}
+	}
+	return false
+}
+
+// TopologyCacheTier is the position of a cache in the topology.
+// Note this is the cache tier itself, notwithstanding MSO. So for an MSO service,
+// Caches immediately before the origin are the TopologyCacheTierLast, even for MSO.
+type TopologyCacheTier string
+
+const (
+	TopologyCacheTierFirst   = TopologyCacheTier("first")
+	TopologyCacheTierInner   = TopologyCacheTier("inner")
+	TopologyCacheTierLast    = TopologyCacheTier("last")
+	TopologyCacheTierInvalid = TopologyCacheTier("")
+)
+
+// TopologyPlacement contains data about the placement of a server in a topology.
+type TopologyPlacement struct {
+	// InTopology is whether the server is in the topology at all.
+	InTopology bool
+	// IsLastTier is whether the server is the last tier in the topology.
+	// Note this is different for MSO vs non-MSO. For MSO, the last tier is the Origin. For non-MSO, the last tier is the last cache tier.
+	IsLastTier bool
+	// CacheTier is the position of the cache in the topology.
+	// Note this is whether the cache is the last cache, even if it has parents in the topology who are origins (MSO).
+	// Thus, it's possible for a server to be CacheTierLast and not IsLastTier.
+	CacheTier TopologyCacheTier
+}
+
+// getTopologyPlacement returns information about the cachegroup's placement in the topology.
+// - Whether the cachegroup is the last tier in the topology.
+// - Whether the cachegroup is in the topology at all.
+// - Whether it's the first, inner, or last cache tier before the Origin.
+func getTopologyPlacement(cacheGroup tc.CacheGroupName, topology tc.Topology, cacheGroups map[tc.CacheGroupName]tc.CacheGroupNullable) TopologyPlacement {
+	serverNode := tc.TopologyNode{}
+	serverNodeIndex := -1
+	for nodeI, node := range topology.Nodes {
+		if node.Cachegroup == string(cacheGroup) {
+			serverNode = node
+			serverNodeIndex = nodeI
+			break
+		}
+	}
+	if serverNode.Cachegroup == "" {
+		return TopologyPlacement{InTopology: false}
+	}
+
+	topologyNodeHasChildren := false
+nodeFor:
+	for _, node := range topology.Nodes {
+		for _, parent := range node.Parents {
+			if parent == serverNodeIndex {
+				topologyNodeHasChildren = true
+				break nodeFor
+			}
+		}
+	}
+
+	cacheTier := TopologyCacheTierFirst
+	if topologyNodeHasChildren {
+		cacheTier = TopologyCacheTierInner
+	}
+
+	isLastTier := len(serverNode.Parents) == 0
+
+	if isLastTier {
+		cacheTier = TopologyCacheTierLast
+	}
+	// Check if the parent is an Origin, and if so, set to Last
+	if cacheTier == TopologyCacheTierInner {
+		// TODO extra safety: check other parents, and warn if parents have different types?
+		parentI := serverNode.Parents[0]
+		if parentI >= len(topology.Nodes) {
+			log.Errorln("ATS config generation: topology '" + topology.Name + "' has node with parent larger than nodes size! Config Generation will be malformed!")
+		} else {
+			parentNode := topology.Nodes[parentI]
+			cg, ok := cacheGroups[tc.CacheGroupName(parentNode.Cachegroup)]
+			if !ok {
+				log.Errorln("ATS config generation: topology '" + topology.Name + "' has node with cachegroup '" + parentNode.Cachegroup + "' that wasn't found in cachegroups! Config Generation will be malformed!")
+			} else if cg.Type == nil {
+				log.Errorln("ATS config generation: cachegroup '" + parentNode.Cachegroup + "' with nil type! Config Generation will be malformed!")
+			} else if *cg.Type == tc.CacheGroupOriginTypeName {
+				// this server's parent in the topology is an Origin, so this server is the last cache tier.
+				cacheTier = TopologyCacheTierLast
+			}
+		}
+	}
+	return TopologyPlacement{InTopology: true, IsLastTier: isLastTier, CacheTier: cacheTier}
+}
+
+func MakeTopologyNameMap(topologies []tc.Topology) map[TopologyName]tc.Topology {
+	topoNames := map[TopologyName]tc.Topology{}
+	for _, to := range topologies {
+		topoNames[TopologyName(to.Name)] = to
+	}
+	return topoNames
+}
+
+func MakeCGMap(cgs []tc.CacheGroupNullable) map[tc.CacheGroupName]tc.CacheGroupNullable {
+	cgMap := map[tc.CacheGroupName]tc.CacheGroupNullable{}
+	for _, cg := range cgs {
+		if cg.Name == nil {
+			log.Errorln("ATS config generation: got cachegroup with nil name, skipping!")
+			continue
+		}
+		cgMap[tc.CacheGroupName(*cg.Name)] = cg
+	}
+	return cgMap
+}
+
+type ParameterWithProfiles struct {
+	tc.Parameter
+	ProfileNames []string
+}
+
+type ParameterWithProfilesMap struct {
+	tc.Parameter
+	ProfileNames map[string]struct{}
+}
+
+// TCParamsToParamsWithProfiles unmarshals the Profiles that the tc struct doesn't.
+func TCParamsToParamsWithProfiles(tcParams []tc.Parameter) ([]ParameterWithProfiles, error) {
+	params := make([]ParameterWithProfiles, 0, len(tcParams))
+	for _, tcParam := range tcParams {
+		param := ParameterWithProfiles{Parameter: tcParam}
+
+		profiles := []string{}
+		if err := json.Unmarshal(tcParam.Profiles, &profiles); err != nil {
+			return nil, errors.New("unmarshalling JSON from parameter '" + strconv.Itoa(param.ID) + "': " + err.Error())
+		}
+		param.ProfileNames = profiles
+		param.Profiles = nil
+		params = append(params, param)
+	}
+	return params, nil
+}
+
+func ParameterWithProfilesToMap(tcParams []ParameterWithProfiles) []ParameterWithProfilesMap {
+	params := []ParameterWithProfilesMap{}
+	for _, tcParam := range tcParams {
+		param := ParameterWithProfilesMap{Parameter: tcParam.Parameter, ProfileNames: map[string]struct{}{}}
+		for _, profile := range tcParam.ProfileNames {
+			param.ProfileNames[profile] = struct{}{}
+		}
+		params = append(params, param)
+	}
+	return params
+}

lib/go-atscfg/hostingdotconfig.go

@@ -35,14 +35,20 @@ const LineCommentHostingDotConfig = LineCommentHash
 const ParamDrivePrefix = "Drive_Prefix"
 const ParamRAMDrivePrefix = "RAM_Drive_Prefix"
 
+const ServerHostingDotConfigMidIncludeInactive = false
+const ServerHostingDotConfigEdgeIncludeInactive = true
+
 func MakeHostingDotConfig(
-	serverName tc.CacheName,
+	server tc.Server,
 	toToolName string, // tm.toolname global parameter (TODO: cache itself?)
 	toURL string, // tm.url global parameter (TODO: cache itself?)
 	params map[string]string, // map[name]value - config file should always be storage.config
-	origins []string, // origins of delivery services assigned to this server. Should only include LIVE and LIVE_NATNL for Edges, and only LIVE_NATNL for Mids.
+	dses []tc.DeliveryServiceNullable,
+	topologies []tc.Topology,
 ) string {
-	text := GenericHeaderComment(string(serverName), toToolName, toURL)
+	text := GenericHeaderComment(server.HostName, toToolName, toURL)
+
+	nameTopologies := MakeTopologyNameMap(topologies)
 
 	lines := []string{}
 	if _, ok := params[ParamRAMDrivePrefix]; ok {
@@ -56,10 +62,23 @@ func MakeHostingDotConfig(
 		text += `# TRAFFIC OPS NOTE: volume ` + strconv.Itoa(ramVolume) + ` is the RAM volume` + "\n"
 
 		seenOrigins := map[string]struct{}{}
-		for _, origin := range origins {
+		for _, ds := range dses {
+			if ds.OrgServerFQDN == nil || ds.XMLID == nil || ds.Active == nil {
+				continue // TODO warn?
+			}
+
+			origin := *ds.OrgServerFQDN
 			if _, ok := seenOrigins[origin]; ok {
 				continue
 			}
+
+			if ds.Topology != nil && *ds.Topology != "" {
+				topology, hasTopology := nameTopologies[TopologyName(*ds.Topology)]
+				if hasTopology && !topologyIncludesServer(topology, server) {
+					continue
+				}
+			}
+
 			seenOrigins[origin] = struct{}{}
 			origin = strings.TrimPrefix(origin, `http://`)
 			origin = strings.TrimPrefix(origin, `https://`)

lib/go-atscfg/hostingdotconfig_test.go

@@ -24,10 +24,11 @@ import (
 	"testing"
 
 	"github.com/apache/trafficcontrol/lib/go-tc"
+	"github.com/apache/trafficcontrol/lib/go-util"
 )
 
 func TestMakeHostingDotConfig(t *testing.T) {
-	serverName := tc.CacheName("server0")
+	server := tc.Server{HostName: "server0"}
 	toToolName := "to0"
 	toURL := "trafficops.example.net"
 	params := map[string]string{
@@ -43,8 +44,14 @@ func TestMakeHostingDotConfig(t *testing.T) {
 		"https://origin4.example.net/",
 		"http://origin5.example.net/",
 	}
+	dses := []tc.DeliveryServiceNullable{}
+	for _, origin := range origins {
+		ds := tc.DeliveryServiceNullable{}
+		ds.OrgServerFQDN = util.StrPtr(origin)
+		dses = append(dses, ds)
+	}
 
-	txt := MakeHostingDotConfig(serverName, toToolName, toURL, params, origins)
+	txt := MakeHostingDotConfig(server, toToolName, toURL, params, dses, nil)
 
 	lines := strings.Split(txt, "\n")
 
@@ -84,6 +91,67 @@ func TestMakeHostingDotConfig(t *testing.T) {
 	}
 }
 
+func TestMakeHostingDotConfigTopologiesIgnoreDSS(t *testing.T) {
+	server := tc.Server{HostName: "server0", Cachegroup: "edgeCG"}
+	toToolName := "to0"
+	toURL := "trafficops.example.net"
+	params := map[string]string{
+		ParamRAMDrivePrefix: "ParamRAMDrivePrefix-shouldnotappearinconfig",
+		ParamDrivePrefix:    "ParamDrivePrefix-shouldnotappearinconfig",
+		"somethingelse":     "somethingelse-shouldnotappearinconfig",
+	}
+
+	dsTopology := tc.DeliveryServiceNullable{}
+	dsTopology.OrgServerFQDN = util.StrPtr("https://origin0.example.net")
+	dsTopology.XMLID = util.StrPtr("ds-topology")
+	dsTopology.Topology = util.StrPtr("t0")
+	dsTopology.Active = util.BoolPtr(true)
+
+	dsTopologyWithoutServer := tc.DeliveryServiceNullable{}
+	dsTopologyWithoutServer.OrgServerFQDN = util.StrPtr("https://origin1.example.net")
+	dsTopologyWithoutServer.XMLID = util.StrPtr("ds-topology-without-server")
+	dsTopologyWithoutServer.Topology = util.StrPtr("t1")
+	dsTopologyWithoutServer.Active = util.BoolPtr(true)
+
+	dses := []tc.DeliveryServiceNullable{dsTopology, dsTopologyWithoutServer}
+
+	topologies := []tc.Topology{
+		tc.Topology{
+			Name: "t0",
+			Nodes: []tc.TopologyNode{
+				tc.TopologyNode{
+					Cachegroup: "edgeCG",
+					Parents:    []int{1},
+				},
+				tc.TopologyNode{
+					Cachegroup: "midCG",
+				},
+			},
+		},
+		tc.Topology{
+			Name: "t1",
+			Nodes: []tc.TopologyNode{
+				tc.TopologyNode{
+					Cachegroup: "otherEdgeCG",
+					Parents:    []int{1},
+				},
+				tc.TopologyNode{
+					Cachegroup: "midCG",
+				},
+			},
+		},
+	}
+
+	txt := MakeHostingDotConfig(server, toToolName, toURL, params, dses, topologies)
+
+	if !strings.Contains(txt, "origin0") {
+		t.Errorf("expected origin0 in topology, actual %v\n", txt)
+	}
+	if strings.Contains(txt, "origin1") {
+		t.Errorf("expected no origin1 not in topology, actual %v\n", txt)
+	}
+}
+
 func strArrContainsSubstr(arr []string, substr string) bool {
 	for _, as := range arr {
 		if strings.Contains(as, substr) {