From 05462c27b2d9ea9574549bbc8ddcfd83b06ebd0e Mon Sep 17 00:00:00 2001 From: Jana Radhakrishnan Date: Tue, 26 May 2015 21:14:01 +0000 Subject: [PATCH] Workaround kernel bugs s related to namespaces This PR attempts to work around bugs present in kernel version 3.18-4.0.1 relating to namespace creation and destruction. This fix attempts to avoid certain systemmcalls to not get in the kkernel bug path as well as lazily garbage collecting the name paths when they are removed. Signed-off-by: Jana Radhakrishnan --- sandbox/namespace_linux.go | 133 ++++++++++++++++++++++++++++------ sandbox/sandbox_linux_test.go | 11 +++ sandbox/sandbox_test.go | 11 +++ 3 files changed, 133 insertions(+), 22 deletions(-) diff --git a/sandbox/namespace_linux.go b/sandbox/namespace_linux.go index 17881f1404..9da67d7abe 100644 --- a/sandbox/namespace_linux.go +++ b/sandbox/namespace_linux.go @@ -4,17 +4,28 @@ import ( "fmt" "net" "os" + "os/exec" + "path/filepath" "runtime" "sync" "syscall" + "time" + log "github.com/Sirupsen/logrus" + "github.com/docker/docker/pkg/reexec" "github.com/vishvananda/netlink" "github.com/vishvananda/netns" ) const prefix = "/var/run/docker/netns" -var once sync.Once +var ( + once sync.Once + garbagePathMap = make(map[string]bool) + gpmLock sync.Mutex + gpmWg sync.WaitGroup + gpmCleanupPeriod = 60 +) // The networkNamespace type is the linux implementation of the Sandbox // interface. It represents a linux network namespace, and moves an interface @@ -26,11 +37,56 @@ type networkNamespace struct { sync.Mutex } +func init() { + reexec.Register("netns-create", reexecCreateNamespace) +} + func createBasePath() { err := os.MkdirAll(prefix, 0644) if err != nil && !os.IsExist(err) { panic("Could not create net namespace path directory") } + + // cleanup any stale namespace files if any + cleanupNamespaceFiles() + + // Start the garbage collection go routine + go removeUnusedPaths() +} + +func removeUnusedPaths() { + for { + time.Sleep(time.Duration(gpmCleanupPeriod) * time.Second) + + gpmLock.Lock() + pathList := make([]string, 0, len(garbagePathMap)) + for path := range garbagePathMap { + pathList = append(pathList, path) + } + garbagePathMap = make(map[string]bool) + gpmWg.Add(1) + gpmLock.Unlock() + + for _, path := range pathList { + os.Remove(path) + } + + gpmWg.Done() + } +} + +func addToGarbagePaths(path string) { + gpmLock.Lock() + defer gpmLock.Unlock() + + garbagePathMap[path] = true +} + +func removeFromGarbagePaths(path string) { + gpmLock.Lock() + defer gpmLock.Unlock() + + delete(garbagePathMap, path) } // GenerateKey generates a sandbox key based on the passed @@ -55,6 +111,16 @@ func NewSandbox(key string, osCreate bool) (Sandbox, error) { return &networkNamespace{path: key, sinfo: info}, nil } +func reexecCreateNamespace() { + if len(os.Args) < 2 { + log.Fatal("no namespace path provided") + } + + if err := syscall.Mount("/proc/self/ns/net", os.Args[1], "bind", syscall.MS_BIND, ""); err != nil { + log.Fatal(err) + } +} + func createNetworkNamespace(path string, osCreate bool) (*Info, error) { runtime.LockOSThread() defer runtime.UnlockOSThread() @@ -69,23 +135,18 @@ func createNetworkNamespace(path string, osCreate bool) (*Info, error) { return nil, err } + cmd := &exec.Cmd{ + Path: reexec.Self(), + Args: append([]string{"netns-create"}, path), + Stdout: os.Stdout, + Stderr: os.Stderr, + } if osCreate { - defer netns.Set(origns) - newns, err := netns.New() - if err != nil { - return nil, err - } - defer newns.Close() - - if err := loopbackUp(); err != nil { - return nil, err - } + cmd.SysProcAttr = &syscall.SysProcAttr{} + cmd.SysProcAttr.Cloneflags = syscall.CLONE_NEWNET } - - procNet := fmt.Sprintf("/proc/%d/task/%d/ns/net", os.Getpid(), syscall.Gettid()) - - if err := syscall.Mount(procNet, path, "bind", syscall.MS_BIND, ""); err != nil { - return nil, err + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("namespace creation reexec command failed: %v", err) } interfaces := []*Interface{} @@ -93,10 +154,27 @@ func createNetworkNamespace(path string, osCreate bool) (*Info, error) { return info, nil } -func cleanupNamespaceFile(path string) { +func cleanupNamespaceFiles() { + filepath.Walk(prefix, func(path string, info os.FileInfo, err error) error { + stat, err := os.Stat(path) + if err != nil { + return err + } + + if stat.IsDir() { + return filepath.SkipDir + } + + syscall.Unmount(path, syscall.MNT_DETACH) + os.Remove(path) + + return nil + }) +} + +func unmountNamespaceFile(path string) { if _, err := os.Stat(path); err == nil { - n := &networkNamespace{path: path} - n.Destroy() + syscall.Unmount(path, syscall.MNT_DETACH) } } @@ -104,11 +182,20 @@ func createNamespaceFile(path string) (err error) { var f *os.File once.Do(createBasePath) - // cleanup namespace file if it already exists because of a previous ungraceful exit. - cleanupNamespaceFile(path) + // Remove it from garbage collection list if present + removeFromGarbagePaths(path) + + // If the path is there unmount it first + unmountNamespaceFile(path) + + // wait for garbage collection to complete if it is in progress + // before trying to create the file. + gpmWg.Wait() + if f, err = os.Create(path); err == nil { f.Close() } + return err } @@ -269,5 +356,7 @@ func (n *networkNamespace) Destroy() error { return err } - return os.Remove(n.path) + // Stash it into the garbage collection list + addToGarbagePaths(n.path) + return nil } diff --git a/sandbox/sandbox_linux_test.go b/sandbox/sandbox_linux_test.go index d4af061f91..d8dded741f 100644 --- a/sandbox/sandbox_linux_test.go +++ b/sandbox/sandbox_linux_test.go @@ -6,6 +6,7 @@ import ( "path/filepath" "runtime" "testing" + "time" "github.com/docker/libnetwork/netutils" "github.com/vishvananda/netlink" @@ -31,6 +32,9 @@ func newKey(t *testing.T) (string, error) { return "", err } + // Set the rpmCleanupPeriod to be low to make the test run quicker + gpmCleanupPeriod = 2 + return name, nil } @@ -137,3 +141,10 @@ func verifySandbox(t *testing.T, s Sandbox) { err) } } + +func verifyCleanup(t *testing.T, s Sandbox) { + time.Sleep(time.Duration(gpmCleanupPeriod*2) * time.Second) + if _, err := os.Stat(s.Key()); err == nil { + t.Fatalf("The sandbox path %s is not getting cleanup event after twice the cleanup period", s.Key()) + } +} diff --git a/sandbox/sandbox_test.go b/sandbox/sandbox_test.go index 811af6d916..258616ada4 100644 --- a/sandbox/sandbox_test.go +++ b/sandbox/sandbox_test.go @@ -2,9 +2,19 @@ package sandbox import ( "net" + "os" "testing" + + "github.com/docker/docker/pkg/reexec" ) +func TestMain(m *testing.M) { + if reexec.Init() { + return + } + os.Exit(m.Run()) +} + func TestSandboxCreate(t *testing.T) { key, err := newKey(t) if err != nil { @@ -44,6 +54,7 @@ func TestSandboxCreate(t *testing.T) { verifySandbox(t, s) s.Destroy() + verifyCleanup(t, s) } func TestSandboxCreateTwice(t *testing.T) {