Fix integration test data race and slowness

ca/config.go

@@ -51,6 +51,13 @@ const (
 	base36DigestLen = 50
 )
 
+// RenewTLSExponentialBackoff sets the exponential backoff when trying to renew TLS certificates that have expired
+var RenewTLSExponentialBackoff = events.ExponentialBackoffConfig{
+	Base:   time.Second * 5,
+	Factor: time.Minute,
+	Max:    1 * time.Hour,
+}
+
 // SecurityConfig is used to represent a node's security configuration. It includes information about
 // the RootCA and ServerTLSCreds/ClientTLSCreds transport authenticators to be used for MTLS
 type SecurityConfig struct {
@@ -446,15 +453,9 @@ func RenewTLSConfigNow(ctx context.Context, s *SecurityConfig, connBroker *conne
 func RenewTLSConfig(ctx context.Context, s *SecurityConfig, connBroker *connectionbroker.Broker, renew <-chan struct{}) <-chan CertificateUpdate {
 	updates := make(chan CertificateUpdate)
 
-	backoffConfig := events.ExponentialBackoffConfig{
-		Base:   time.Second * 5,
-		Factor: time.Minute,
-		Max:    1 * time.Hour,
-	}
-
 	go func() {
 		var retry time.Duration
-		expBackoff := events.NewExponentialBackoff(backoffConfig)
+		expBackoff := events.NewExponentialBackoff(RenewTLSExponentialBackoff)
 		defer close(updates)
 		for {
 			ctx = log.WithModule(ctx, "tls")
@@ -494,7 +495,7 @@ func RenewTLSConfig(ctx context.Context, s *SecurityConfig, connBroker *connecti
 
 			log.WithFields(logrus.Fields{
 				"time": time.Now().Add(retry),
-			}).Debugf("next certificate renewal scheduled")
+			}).Debugf("next certificate renewal scheduled for %v from now", retry)
 
 			select {
 			case <-time.After(retry):
@@ -513,7 +514,7 @@ func RenewTLSConfig(ctx context.Context, s *SecurityConfig, connBroker *connecti
 				expBackoff.Failure(nil, nil)
 			} else {
 				certUpdate.Role = s.ClientTLSCreds.Role()
-				expBackoff = events.NewExponentialBackoff(backoffConfig)
+				expBackoff = events.NewExponentialBackoff(RenewTLSExponentialBackoff)
 			}
 
 			select {

integration/cluster.go

@@ -12,6 +12,7 @@ import (
 	"github.com/docker/swarmkit/ca"
 	"github.com/docker/swarmkit/log"
 	raftutils "github.com/docker/swarmkit/manager/state/raft/testutils"
+	"github.com/docker/swarmkit/node"
 	"golang.org/x/net/context"
 )
 
@@ -330,15 +331,30 @@ func (c *testCluster) StartNode(id string) error {
 	}
 
 	ctx := log.WithLogger(c.ctx, log.L.WithField("testnode", c.nodesOrder[id]))
+	errCtx, cancel := context.WithCancel(context.Background())
+	done := make(chan error)
+	defer cancel()
+	defer close(done)
 
-	c.wg.Add(1)
+	c.wg.Add(2)
 	go func() {
 		c.errs <- n.node.Start(ctx)
 		c.wg.Done()
 	}()
+	go func(n *node.Node) {
+		err := n.Err(errCtx)
+		select {
+		case <-errCtx.Done():
+		default:
+			done <- err
+		}
+		c.wg.Done()
+	}(n.node)
 
 	select {
 	case <-n.node.Ready():
+	case err := <-done:
+		return err
 	case <-time.After(opsTimeout):
 		return fmt.Errorf("node did not ready in time")
 	}

integration/integration_test.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/Sirupsen/logrus"
 	"github.com/cloudflare/cfssl/helpers"
+	events "github.com/docker/go-events"
 	"github.com/docker/swarmkit/api"
 	"github.com/docker/swarmkit/ca"
 	raftutils "github.com/docker/swarmkit/manager/state/raft/testutils"
@@ -44,6 +45,10 @@ func printTrace() {
 }
 
 func TestMain(m *testing.M) {
+	ca.RenewTLSExponentialBackoff = events.ExponentialBackoffConfig{
+		Factor: time.Millisecond * 500,
+		Max:    time.Minute,
+	}
 	flag.Parse()
 	res := m.Run()
 	if *showTrace {
@@ -463,7 +468,6 @@ func TestRestartLeader(t *testing.T) {
 
 func TestForceNewCluster(t *testing.T) {
 	t.Parallel()
-	logrus.SetLevel(logrus.DebugLevel)
 
 	// create an external CA so that we can use it to generate expired certificates
 	tempDir, err := ioutil.TempDir("", "external-ca")