[Backport 17.06] service connectivity issue

agent.go

@@ -738,11 +738,12 @@ func (n *network) addDriverWatches() {
 			return
 		}
 
-		agent.networkDB.WalkTable(table.name, func(nid, key string, value []byte) bool {
-			if nid == n.ID() {
+		agent.networkDB.WalkTable(table.name, func(nid, key string, value []byte, deleted bool) bool {
+			// skip the entries that are mark for deletion, this is safe because this function is
+			// called at initialization time so there is no state to delete
+			if nid == n.ID() && !deleted {
 				d.EventNotify(driverapi.Create, nid, table.name, key, value)
 			}
-
 			return false
 		})
 	}

common/caller.go

@@ -0,0 +1,29 @@
+package common
+
+import (
+	"runtime"
+	"strings"
+)
+
+func callerInfo(i int) string {
+	ptr, _, _, ok := runtime.Caller(i)
+	fName := "unknown"
+	if ok {
+		f := runtime.FuncForPC(ptr)
+		if f != nil {
+			// f.Name() is like: github.com/docker/libnetwork/common.MethodName
+			tmp := strings.Split(f.Name(), ".")
+			if len(tmp) > 0 {
+				fName = tmp[len(tmp)-1]
+			}
+		}
+	}
+
+	return fName
+}
+
+// CallerName returns the name of the function at the specified level
+// level == 0 means current method name
+func CallerName(level int) string {
+	return callerInfo(2 + level)
+}

common/caller_test.go

@@ -0,0 +1,49 @@
+package common
+
+import "testing"
+
+func fun1() string {
+	return CallerName(0)
+}
+
+func fun2() string {
+	return CallerName(1)
+}
+
+func fun3() string {
+	return fun4()
+}
+
+func fun4() string {
+	return CallerName(0)
+}
+
+func fun5() string {
+	return fun6()
+}
+
+func fun6() string {
+	return CallerName(1)
+}
+
+func TestCaller(t *testing.T) {
+	funName := fun1()
+	if funName != "fun1" {
+		t.Fatalf("error on fun1 caller %s", funName)
+	}
+
+	funName = fun2()
+	if funName != "TestCaller" {
+		t.Fatalf("error on fun2 caller %s", funName)
+	}
+
+	funName = fun3()
+	if funName != "fun4" {
+		t.Fatalf("error on fun2 caller %s", funName)
+	}
+
+	funName = fun5()
+	if funName != "fun5" {
+		t.Fatalf("error on fun5 caller %s", funName)
+	}
+}

drivers/overlay/joinleave.go

@@ -120,8 +120,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
 		}
 	}
 
-	d.peerDbAdd(nid, eid, ep.addr.IP, ep.addr.Mask, ep.mac,
-		net.ParseIP(d.advertiseAddress), true)
+	d.peerAdd(nid, eid, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.advertiseAddress), true, false, false, true)
 
 	if err := d.checkEncryption(nid, nil, n.vxlanID(s), true, true); err != nil {
 		logrus.Warn(err)
@@ -205,7 +204,7 @@ func (d *driver) EventNotify(etype driverapi.EventType, nid, tableName, key stri
 		return
 	}
 
-	d.peerAdd(nid, eid, addr.IP, addr.Mask, mac, vtep, true, false, false)
+	d.peerAdd(nid, eid, addr.IP, addr.Mask, mac, vtep, true, false, false, false)
 }
 
 // Leave method is invoked when a Sandbox detaches from an endpoint.

drivers/overlay/ov_network.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 	"sync"
 	"syscall"
+	"time"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/pkg/reexec"
@@ -682,10 +683,12 @@ func (n *network) initSandbox(restore bool) error {
 		return fmt.Errorf("could not get network sandbox (oper %t): %v", restore, err)
 	}
 
+	// this is needed to let the peerAdd configure the sandbox
 	n.setSandbox(sbox)
 
 	if !restore {
-		n.driver.peerDbUpdateSandbox(n.id)
+		// Initialize the sandbox with all the peers previously received from networkdb
+		n.driver.initSandboxPeerDB(n.id)
 	}
 
 	var nlSock *nl.NetlinkSocket
@@ -705,6 +708,7 @@ func (n *network) initSandbox(restore bool) error {
 }
 
 func (n *network) watchMiss(nlSock *nl.NetlinkSocket) {
+	t := time.Now()
 	for {
 		msgs, err := nlSock.Receive()
 		if err != nil {
@@ -757,23 +761,52 @@ func (n *network) watchMiss(nlSock *nl.NetlinkSocket) {
 				continue
 			}
 
-			if !n.driver.isSerfAlive() {
-				continue
-			}
-
-			mac, IPmask, vtep, err := n.driver.resolvePeer(n.id, ip)
-			if err != nil {
-				logrus.Errorf("could not resolve peer %q: %v", ip, err)
-				continue
-			}
-
-			if err := n.driver.peerAdd(n.id, "dummy", ip, IPmask, mac, vtep, true, l2Miss, l3Miss); err != nil {
-				logrus.Errorf("could not add neighbor entry for missed peer %q: %v", ip, err)
+			if n.driver.isSerfAlive() {
+				mac, IPmask, vtep, err := n.driver.resolvePeer(n.id, ip)
+				if err != nil {
+					logrus.Errorf("could not resolve peer %q: %v", ip, err)
+					continue
+				}
+				n.driver.peerAdd(n.id, "dummy", ip, IPmask, mac, vtep, true, l2Miss, l3Miss, false)
+			} else {
+				// If the gc_thresh values are lower kernel might knock off the neighor entries.
+				// When we get a L3 miss check if its a valid peer and reprogram the neighbor
+				// entry again. Rate limit it to once attempt every 500ms, just in case a faulty
+				// container sends a flood of packets to invalid peers
+				if !l3Miss {
+					continue
+				}
+				if time.Since(t) > 500*time.Millisecond {
+					t = time.Now()
+					n.programNeighbor(ip)
+				}
 			}
 		}
 	}
 }
 
+func (n *network) programNeighbor(ip net.IP) {
+	peerMac, _, _, err := n.driver.peerDbSearch(n.id, ip)
+	if err != nil {
+		logrus.Errorf("Reprogramming on L3 miss failed for %s, no peer entry", ip)
+		return
+	}
+	s := n.getSubnetforIPAddr(ip)
+	if s == nil {
+		logrus.Errorf("Reprogramming on L3 miss failed for %s, not a valid subnet", ip)
+		return
+	}
+	sbox := n.sandbox()
+	if sbox == nil {
+		logrus.Errorf("Reprogramming on L3 miss failed for %s, overlay sandbox missing", ip)
+		return
+	}
+	if err := sbox.AddNeighbor(ip, peerMac, true, sbox.NeighborOptions().LinkName(s.vxlanName)); err != nil {
+		logrus.Errorf("Reprogramming on L3 miss failed for %s: %v", ip, err)
+		return
+	}
+}
+
 func (d *driver) addNetwork(n *network) {
 	d.Lock()
 	d.networks[n.id] = n
@@ -1057,6 +1090,15 @@ func (n *network) contains(ip net.IP) bool {
 	return false
 }
 
+func (n *network) getSubnetforIPAddr(ip net.IP) *subnet {
+	for _, s := range n.subnets {
+		if s.subnetIP.Contains(ip) {
+			return s
+		}
+	}
+	return nil
+}
+
 // getSubnetforIP returns the subnet to which the given IP belongs
 func (n *network) getSubnetforIP(ip *net.IPNet) *subnet {
 	for _, s := range n.subnets {

drivers/overlay/ov_serf.go

@@ -120,15 +120,10 @@ func (d *driver) processEvent(u serf.UserEvent) {
 
 	switch action {
 	case "join":
-		if err := d.peerAdd(nid, eid, net.ParseIP(ipStr), net.IPMask(net.ParseIP(maskStr).To4()), mac,
-			net.ParseIP(vtepStr), true, false, false); err != nil {
-			logrus.Errorf("Peer add failed in the driver: %v\n", err)
-		}
+		d.peerAdd(nid, eid, net.ParseIP(ipStr), net.IPMask(net.ParseIP(maskStr).To4()), mac, net.ParseIP(vtepStr),
+			true, false, false, false)
 	case "leave":
-		if err := d.peerDelete(nid, eid, net.ParseIP(ipStr), net.IPMask(net.ParseIP(maskStr).To4()), mac,
-			net.ParseIP(vtepStr), true); err != nil {
-			logrus.Errorf("Peer delete failed in the driver: %v\n", err)
-		}
+		d.peerDelete(nid, eid, net.ParseIP(ipStr), net.IPMask(net.ParseIP(maskStr).To4()), mac, net.ParseIP(vtepStr), true)
 	}
 }
 

drivers/overlay/overlay.go

@@ -3,6 +3,7 @@ package overlay
 //go:generate protoc -I.:../../Godeps/_workspace/src/github.com/gogo/protobuf  --gogo_out=import_path=github.com/docker/libnetwork/drivers/overlay,Mgogoproto/gogo.proto=github.com/gogo/protobuf/gogoproto:. overlay.proto
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"sync"
@@ -50,6 +51,8 @@ type driver struct {
 	joinOnce         sync.Once
 	localJoinOnce    sync.Once
 	keys             []*key
+	peerOpCh         chan *peerOperation
+	peerOpCancel     context.CancelFunc
 	sync.Mutex
 }
 
@@ -64,10 +67,16 @@ func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
 		peerDb: peerNetworkMap{
 			mp: map[string]*peerMap{},
 		},
-		secMap: &encrMap{nodes: map[string][]*spi{}},
-		config: config,
+		secMap:   &encrMap{nodes: map[string][]*spi{}},
+		config:   config,
+		peerOpCh: make(chan *peerOperation),
 	}
 
+	// Launch the go routine for processing peer operations
+	ctx, cancel := context.WithCancel(context.Background())
+	d.peerOpCancel = cancel
+	go d.peerOpRoutine(ctx, d.peerOpCh)
+
 	if data, ok := config[netlabel.GlobalKVClient]; ok {
 		var err error
 		dsc, ok := data.(discoverapi.DatastoreConfigData)
@@ -161,7 +170,7 @@ func (d *driver) restoreEndpoints() error {
 		}
 
 		n.incEndpointCount()
-		d.peerDbAdd(ep.nid, ep.id, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.advertiseAddress), true)
+		d.peerAdd(ep.nid, ep.id, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.advertiseAddress), true, false, false, true)
 	}
 	return nil
 }
@@ -170,6 +179,11 @@ func (d *driver) restoreEndpoints() error {
 func Fini(drv driverapi.Driver) {
 	d := drv.(*driver)
 
+	// Notify the peer go routine to return
+	if d.peerOpCancel != nil {
+		d.peerOpCancel()
+	}
+
 	if d.exitCh != nil {
 		waitCh := make(chan struct{})