Add knobs on LB sandbox

cmd/proxy/network_proxy_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"bytes"
-	"flag"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -12,10 +11,10 @@ import (
 	"time"
 
 	"github.com/ishidawataru/sctp"
+	// this takes care of the incontainer flag
+	_ "github.com/docker/libnetwork/testutils"
 )
 
-var _ = flag.Bool("incontainer", false, "Indicates if the test is running in a container")
-
 var testBuf = []byte("Buffalo buffalo Buffalo buffalo buffalo buffalo Buffalo buffalo")
 var testBufSize = len(testBuf)
 

controller.go

@@ -1144,6 +1144,11 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (S
 		}
 	}
 
+	if sb.osSbox != nil {
+		// Apply operating specific knobs on the load balancer sandbox
+		sb.osSbox.ApplyOSTweaks(sb.oslTypes)
+	}
+
 	c.Lock()
 	c.sandboxes[sb.id] = sb
 	c.Unlock()

drivers/overlay/ostweaks_linux.go

@@ -1,72 +1,23 @@
 package overlay
 
 import (
-	"io/ioutil"
-	"path"
 	"strconv"
-	"strings"
 
-	"github.com/sirupsen/logrus"
+	"github.com/docker/libnetwork/osl/kernel"
 )
 
-type conditionalCheck func(val1, val2 string) bool
-
-type osValue struct {
-	value   string
-	checkFn conditionalCheck
-}
-
-var osConfig = map[string]osValue{
+var ovConfig = map[string]*kernel.OSValue{
 	"net.ipv4.neigh.default.gc_thresh1": {"8192", checkHigher},
 	"net.ipv4.neigh.default.gc_thresh2": {"49152", checkHigher},
 	"net.ipv4.neigh.default.gc_thresh3": {"65536", checkHigher},
 }
 
-func propertyIsValid(val1, val2 string, check conditionalCheck) bool {
-	if check == nil || check(val1, val2) {
-		return true
-	}
-	return false
-}
-
 func checkHigher(val1, val2 string) bool {
 	val1Int, _ := strconv.ParseInt(val1, 10, 32)
 	val2Int, _ := strconv.ParseInt(val2, 10, 32)
 	return val1Int < val2Int
 }
 
-// writeSystemProperty writes the value to a path under /proc/sys as determined from the key.
-// For e.g. net.ipv4.ip_forward translated to /proc/sys/net/ipv4/ip_forward.
-func writeSystemProperty(key, value string) error {
-	keyPath := strings.Replace(key, ".", "/", -1)
-	return ioutil.WriteFile(path.Join("/proc/sys", keyPath), []byte(value), 0644)
-}
-
-func readSystemProperty(key string) (string, error) {
-	keyPath := strings.Replace(key, ".", "/", -1)
-	value, err := ioutil.ReadFile(path.Join("/proc/sys", keyPath))
-	if err != nil {
-		return "", err
-	}
-	return string(value), nil
-}
-
 func applyOStweaks() {
-	for k, v := range osConfig {
-		// read the existing property from disk
-		oldv, err := readSystemProperty(k)
-		if err != nil {
-			logrus.Errorf("error reading the kernel parameter %s, error: %s", k, err)
-			continue
-		}
-
-		if propertyIsValid(oldv, v.value, v.checkFn) {
-			// write new prop value to disk
-			if err := writeSystemProperty(k, v.value); err != nil {
-				logrus.Errorf("error setting the kernel parameter %s = %s, (leaving as %s) error: %s", k, v.value, oldv, err)
-				continue
-			}
-			logrus.Debugf("updated kernel parameter %s = %s (was %s)", k, v.value, oldv)
-		}
-	}
+	kernel.ApplyOSTweaks(ovConfig)
 }

drvregistry/drvregistry_test.go

@@ -1,7 +1,6 @@
 package drvregistry
 
 import (
-	"flag"
 	"sort"
 	"testing"
 
@@ -13,10 +12,10 @@ import (
 	nullIpam "github.com/docker/libnetwork/ipams/null"
 	remoteIpam "github.com/docker/libnetwork/ipams/remote"
 	"github.com/stretchr/testify/assert"
-)
 
-var runningInContainer = flag.Bool("incontainer", false,
-	"Indicates if the test is running in a container")
+	// this takes care of the incontainer flag
+	_ "github.com/docker/libnetwork/testutils"
+)
 
 const mockDriverName = "mock-driver"
 

network.go

@@ -2125,7 +2125,8 @@ func (n *network) lbEndpointName() string {
 
 func (n *network) createLoadBalancerSandbox() (retErr error) {
 	sandboxName := n.lbSandboxName()
-	sbOptions := []SandboxOption{}
+	// Mark the sandbox to be a load balancer
+	sbOptions := []SandboxOption{OptionLoadBalancer()}
 	if n.ingress {
 		sbOptions = append(sbOptions, OptionIngress())
 	}

networkdb/networkdb_test.go

@@ -1,7 +1,6 @@
 package networkdb
 
 import (
-	"flag"
 	"fmt"
 	"io/ioutil"
 	"log"
@@ -18,13 +17,13 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-)
 
-var (
-	dbPort             int32 = 10000
-	runningInContainer       = flag.Bool("incontainer", false, "Indicates if the test is running in a container")
+	// this takes care of the incontainer flag
+	_ "github.com/docker/libnetwork/testutils"
 )
 
+var dbPort int32 = 10000
+
 func TestMain(m *testing.M) {
 	ioutil.WriteFile("/proc/sys/net/ipv6/conf/lo/disable_ipv6", []byte{'0', '\n'}, 0644)
 	logrus.SetLevel(logrus.ErrorLevel)

osl/kernel/knobs.go

@@ -0,0 +1,16 @@
+package kernel
+
+type conditionalCheck func(val1, val2 string) bool
+
+// OSValue represents a tuple, value defired, check function when to apply the value
+type OSValue struct {
+	Value   string
+	CheckFn conditionalCheck
+}
+
+func propertyIsValid(val1, val2 string, check conditionalCheck) bool {
+	if check == nil || check(val1, val2) {
+		return true
+	}
+	return false
+}

osl/kernel/knobs_linux.go

@@ -0,0 +1,47 @@
+package kernel
+
+import (
+	"io/ioutil"
+	"path"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+)
+
+// writeSystemProperty writes the value to a path under /proc/sys as determined from the key.
+// For e.g. net.ipv4.ip_forward translated to /proc/sys/net/ipv4/ip_forward.
+func writeSystemProperty(key, value string) error {
+	keyPath := strings.Replace(key, ".", "/", -1)
+	return ioutil.WriteFile(path.Join("/proc/sys", keyPath), []byte(value), 0644)
+}
+
+// readSystemProperty reads the value from the path under /proc/sys and returns it
+func readSystemProperty(key string) (string, error) {
+	keyPath := strings.Replace(key, ".", "/", -1)
+	value, err := ioutil.ReadFile(path.Join("/proc/sys", keyPath))
+	if err != nil {
+		return "", err
+	}
+	return strings.TrimSpace(string(value)), nil
+}
+
+// ApplyOSTweaks applies the configuration values passed as arguments
+func ApplyOSTweaks(osConfig map[string]*OSValue) {
+	for k, v := range osConfig {
+		// read the existing property from disk
+		oldv, err := readSystemProperty(k)
+		if err != nil {
+			logrus.WithError(err).Errorf("error reading the kernel parameter %s", k)
+			continue
+		}
+
+		if propertyIsValid(oldv, v.Value, v.CheckFn) {
+			// write new prop value to disk
+			if err := writeSystemProperty(k, v.Value); err != nil {
+				logrus.WithError(err).Errorf("error setting the kernel parameter %s = %s, (leaving as %s)", k, v.Value, oldv)
+				continue
+			}
+			logrus.Debugf("updated kernel parameter %s = %s (was %s)", k, v.Value, oldv)
+		}
+	}
+}

osl/kernel/knobs_linux_test.go

@@ -0,0 +1,33 @@
+package kernel
+
+import (
+	"testing"
+
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+
+	_ "github.com/docker/libnetwork/testutils"
+)
+
+func TestReadWriteKnobs(t *testing.T) {
+	for _, k := range []string{
+		"net.ipv4.neigh.default.gc_thresh1",
+		"net.ipv4.neigh.default.gc_thresh2",
+		"net.ipv4.neigh.default.gc_thresh3",
+	} {
+		// Check if the test is able to read the value
+		v, err := readSystemProperty(k)
+		if err != nil {
+			logrus.WithError(err).Warnf("Path %v not readable", k)
+			// the path is not there, skip this key
+			continue
+		}
+		// Test the write
+		assert.NoError(t, writeSystemProperty(k, "10000"))
+		newV, err := readSystemProperty(k)
+		assert.NoError(t, err)
+		assert.Equal(t, newV, "10000")
+		// Restore value
+		assert.NoError(t, writeSystemProperty(k, v))
+	}
+}

osl/kernel/knobs_unsupported.go

@@ -0,0 +1,7 @@
+// +build !linux
+
+package kernel
+
+// ApplyOSTweaks applies the configuration values passed as arguments
+func ApplyOSTweaks(osConfig map[string]*OSValue) {
+}

osl/namespace_linux.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/docker/docker/pkg/reexec"
 	"github.com/docker/libnetwork/ns"
+	"github.com/docker/libnetwork/osl/kernel"
 	"github.com/docker/libnetwork/types"
 	"github.com/sirupsen/logrus"
 	"github.com/vishvananda/netlink"
@@ -29,13 +30,18 @@ func init() {
 }
 
 var (
-	once             sync.Once
-	garbagePathMap   = make(map[string]bool)
-	gpmLock          sync.Mutex
-	gpmWg            sync.WaitGroup
-	gpmCleanupPeriod = 60 * time.Second
-	gpmChan          = make(chan chan struct{})
-	prefix           = defaultPrefix
+	once               sync.Once
+	garbagePathMap     = make(map[string]bool)
+	gpmLock            sync.Mutex
+	gpmWg              sync.WaitGroup
+	gpmCleanupPeriod   = 60 * time.Second
+	gpmChan            = make(chan chan struct{})
+	prefix             = defaultPrefix
+	loadBalancerConfig = map[string]*kernel.OSValue{
+		// expires connection from the IPVS connection table when the backend is not available
+		// more info: https://github.com/torvalds/linux/blob/master/Documentation/networking/ipvs-sysctl.txt#L126:1
+		"net.ipv4.vs.expire_nodest_conn": {"1", nil},
+	}
 )
 
 // The networkNamespace type is the linux implementation of the Sandbox
@@ -630,3 +636,13 @@ func setIPv6(path, iface string, enable bool) error {
 	}
 	return nil
 }
+
+// ApplyOSTweaks applies linux configs on the sandbox
+func (n *networkNamespace) ApplyOSTweaks(types []SandboxType) {
+	for _, t := range types {
+		switch t {
+		case SandboxTypeLoadBalancer:
+			kernel.ApplyOSTweaks(loadBalancerConfig)
+		}
+	}
+}

osl/sandbox.go

@@ -7,6 +7,16 @@ import (
 	"github.com/docker/libnetwork/types"
 )
 
+// SandboxType specify the time of the sandbox, this can be used to apply special configs
+type SandboxType int
+
+const (
+	// SandboxTypeIngress indicates that the sandbox is for the ingress
+	SandboxTypeIngress = iota
+	// SandboxTypeLoadBalancer indicates that the sandbox is a load balancer
+	SandboxTypeLoadBalancer = iota
+)
+
 // Sandbox represents a network sandbox, identified by a specific key.  It
 // holds a list of Interfaces, routes etc, and more can be added dynamically.
 type Sandbox interface {
@@ -70,6 +80,9 @@ type Sandbox interface {
 
 	// restore sandbox
 	Restore(ifsopt map[string][]IfaceOption, routes []*types.StaticRoute, gw net.IP, gw6 net.IP) error
+
+	// ApplyOSTweaks applies operating system specific knobs on the sandbox
+	ApplyOSTweaks([]SandboxType)
 }
 
 // NeighborOptionSetter interface defines the option setter methods for interface options

sandbox.go

@@ -83,6 +83,7 @@ type sandbox struct {
 	inDelete           bool
 	ingress            bool
 	ndotsSet           bool
+	oslTypes           []osl.SandboxType // slice of properties of this sandbox
 	sync.Mutex
 	// This mutex is used to serialize service related operation for an endpoint
 	// The lock is here because the endpoint is saved into the store so is not unique
@@ -1162,6 +1163,15 @@ func OptionPortMapping(portBindings []types.PortBinding) SandboxOption {
 func OptionIngress() SandboxOption {
 	return func(sb *sandbox) {
 		sb.ingress = true
+		sb.oslTypes = append(sb.oslTypes, osl.SandboxTypeIngress)
+	}
+}
+
+// OptionLoadBalancer function returns an option setter for marking a
+// sandbox as a load balancer sandbox.
+func OptionLoadBalancer() SandboxOption {
+	return func(sb *sandbox) {
+		sb.oslTypes = append(sb.oslTypes, osl.SandboxTypeLoadBalancer)
 	}
 }
 

types/types_test.go

@@ -1,13 +1,13 @@
 package types
 
 import (
-	"flag"
-	"github.com/stretchr/testify/require"
 	"net"
 	"testing"
-)
 
-var runningInContainer = flag.Bool("incontainer", false, "Indicates if the test is running in a container")
+	"github.com/stretchr/testify/require"
+
+	_ "github.com/docker/libnetwork/testutils"
+)
 
 func TestTransportPortConv(t *testing.T) {
 	sform := "tcp/23"