Merge pull request #203 from andrewhsu/rc4

[17.07] bump version to 17.07.0-ce-rc4

CHANGELOG.md

@@ -5,7 +5,7 @@ information on the list of deprecated flags and APIs please have a look at
 https://docs.docker.com/engine/deprecated/ where target removal dates can also
 be found.
 
-## 17.07.0-ce (2017-07-XX)
+## 17.07.0-ce (2017-08-XX)
 
 ### API & Client
 
@@ -57,15 +57,16 @@ be found.
   * LCOW: Change directory from lcow to "Linux Containers" [moby/moby#33835](https://github.com/moby/moby/pull/33835)
   * LCOW: pass command arguments without extra quoting [moby/moby#33815](https://github.com/moby/moby/pull/33815)
   * LCOW: Updates necessary due to platform schema change [moby/moby#33785](https://github.com/moby/moby/pull/33785)
+- Aufs: ensure diff layers are correctly removed to prevent leftover files from using up storage [moby/moby#34587](https://github.com/moby/moby/pull/34587)
 
 ### Swarm Mode
 
-* Add support for plugins on swarm [moby/moby#33575](https://github.com/moby/moby/pull/33575)
 * Initial support for plugable secret backends [moby/moby#34157](https://github.com/moby/moby/pull/34157) [moby/moby#34123](https://github.com/moby/moby/pull/34123)
 * Sort swarm stacks and nodes using natural sorting [docker/cli#315](https://github.com/docker/cli/pull/315)
 * Make engine support cluster config event [moby/moby#34032](https://github.com/moby/moby/pull/34032)
 * Only pass a join address when in the process of joining a cluster [moby/moby#33361](https://github.com/moby/moby/pull/33361)
 * Fix error during service creation if a network with the same name exists both as "local" and "swarm" scoped network [docker/cli#184](https://github.com/docker/cli/pull/184)
+* (experimental) Add support for plugins on swarm [moby/moby#33575](https://github.com/moby/moby/pull/33575)
 
 ## 17.06.0-ce (2017-06-07)
 

VERSION

@@ -1 +1 @@
-17.07.0-ce-rc2
+17.07.0-ce-rc4

components/cli/VERSION

@@ -1 +1 @@
-17.07.0-ce-rc2
+17.07.0-ce-rc4

components/cli/vendor/github.com/docker/docker/client/client.go

@@ -51,6 +51,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 
@@ -219,9 +220,9 @@ func (cli *Client) getAPIPath(p string, query url.Values) string {
 	var apiPath string
 	if cli.version != "" {
 		v := strings.TrimPrefix(cli.version, "v")
-		apiPath = cli.basePath + "/v" + v + p
+		apiPath = path.Join(cli.basePath, "/v"+v+p)
 	} else {
-		apiPath = cli.basePath + p
+		apiPath = path.Join(cli.basePath, p)
 	}
 
 	u := &url.URL{

components/cli/vendor/github.com/docker/docker/client/ping.go

@@ -1,6 +1,8 @@
 package client
 
 import (
+	"path"
+
 	"github.com/docker/docker/api/types"
 	"golang.org/x/net/context"
 )
@@ -8,7 +10,7 @@ import (
 // Ping pings the server and returns the value of the "Docker-Experimental", "OS-Type" & "API-Version" headers
 func (cli *Client) Ping(ctx context.Context) (types.Ping, error) {
 	var ping types.Ping
-	req, err := cli.buildRequest("GET", cli.basePath+"/_ping", nil, nil)
+	req, err := cli.buildRequest("GET", path.Join(cli.basePath, "/_ping"), nil, nil)
 	if err != nil {
 		return ping, err
 	}

components/engine/VERSION

@@ -1 +1 @@
-17.07.0-ce-rc2
+17.07.0-ce-rc4

components/engine/api/swagger.yaml

@@ -1981,7 +1981,7 @@ definitions:
     properties:
       PluginSpec:
         type: "object"
-        description: "Invalid when specified with `ContainerSpec`."
+        description: "Invalid when specified with `ContainerSpec`. *(Experimental release only.)*"
         properties:
           Name:
             description: "The name or 'alias' to use for the plugin."

components/engine/client/client.go

@@ -51,6 +51,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 
@@ -219,9 +220,9 @@ func (cli *Client) getAPIPath(p string, query url.Values) string {
 	var apiPath string
 	if cli.version != "" {
 		v := strings.TrimPrefix(cli.version, "v")
-		apiPath = cli.basePath + "/v" + v + p
+		apiPath = path.Join(cli.basePath, "/v"+v+p)
 	} else {
-		apiPath = cli.basePath + p
+		apiPath = path.Join(cli.basePath, p)
 	}
 
 	u := &url.URL{

components/engine/client/ping.go

@@ -1,6 +1,8 @@
 package client
 
 import (
+	"path"
+
 	"github.com/docker/docker/api/types"
 	"golang.org/x/net/context"
 )
@@ -8,7 +10,7 @@ import (
 // Ping pings the server and returns the value of the "Docker-Experimental", "OS-Type" & "API-Version" headers
 func (cli *Client) Ping(ctx context.Context) (types.Ping, error) {
 	var ping types.Ping
-	req, err := cli.buildRequest("GET", cli.basePath+"/_ping", nil, nil)
+	req, err := cli.buildRequest("GET", path.Join(cli.basePath, "/_ping"), nil, nil)
 	if err != nil {
 		return ping, err
 	}

components/engine/daemon/cluster/executor/container/executor.go

@@ -183,13 +183,17 @@ func (e *executor) Controller(t *api.Task) (exec.Controller, error) {
 		}
 		switch runtimeKind {
 		case string(swarmtypes.RuntimePlugin):
+			info, _ := e.backend.SystemInfo()
+			if !info.ExperimentalBuild {
+				return ctlr, fmt.Errorf("runtime type %q only supported in experimental", swarmtypes.RuntimePlugin)
+			}
 			c, err := plugin.NewController(e.pluginBackend, t)
 			if err != nil {
 				return ctlr, err
 			}
 			ctlr = c
 		default:
-			return ctlr, fmt.Errorf("unsupported runtime type: %q", r.Generic.Kind)
+			return ctlr, fmt.Errorf("unsupported runtime type: %q", runtimeKind)
 		}
 	case *api.TaskSpec_Container:
 		c, err := newController(e.backend, t, dependencyGetter)

components/engine/daemon/cluster/services.go

@@ -139,9 +139,16 @@ func (c *Cluster) CreateService(s types.ServiceSpec, encodedAuth string, queryRe
 		case *swarmapi.TaskSpec_Generic:
 			switch serviceSpec.Task.GetGeneric().Kind {
 			case string(types.RuntimePlugin):
+				info, _ := c.config.Backend.SystemInfo()
+				if !info.ExperimentalBuild {
+					return fmt.Errorf("runtime type %q only supported in experimental", types.RuntimePlugin)
+				}
 				if s.TaskTemplate.PluginSpec == nil {
 					return errors.New("plugin spec must be set")
 				}
+
+			default:
+				return fmt.Errorf("unsupported runtime type: %q", serviceSpec.Task.GetGeneric().Kind)
 			}
 
 			r, err := state.controlClient.CreateService(ctx, &swarmapi.CreateServiceRequest{Spec: &serviceSpec})

components/engine/daemon/graphdriver/aufs/aufs.go

@@ -142,6 +142,23 @@ func Init(root string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		}
 	}
 
+	for _, path := range []string{"mnt", "diff"} {
+		p := filepath.Join(root, path)
+		dirs, err := ioutil.ReadDir(p)
+		if err != nil {
+			logrus.WithError(err).WithField("dir", p).Error("error reading dir entries")
+			continue
+		}
+		for _, dir := range dirs {
+			if strings.HasSuffix(dir.Name(), "-removing") {
+				logrus.WithField("dir", dir.Name()).Debug("Cleaning up stale layer dir")
+				if err := system.EnsureRemoveAll(filepath.Join(p, dir.Name())); err != nil {
+					logrus.WithField("dir", dir.Name()).WithError(err).Error("Error removing stale layer dir")
+				}
+			}
+		}
+	}
+
 	a.naiveDiff = graphdriver.NewNaiveDiffDriver(a, uidMaps, gidMaps)
 	return a, nil
 }
@@ -317,26 +334,6 @@ func (a *Driver) Remove(id string) error {
 		retries++
 		logger.Warnf("unmount failed due to EBUSY: retry count: %d", retries)
 		time.Sleep(100 * time.Millisecond)
-		continue
-	}
-
-	// Atomically remove each directory in turn by first moving it out of the
-	// way (so that docker doesn't find it anymore) before doing removal of
-	// the whole tree.
-	tmpMntPath := path.Join(a.mntPath(), fmt.Sprintf("%s-removing", id))
-	if err := os.Rename(mountpoint, tmpMntPath); err != nil && !os.IsNotExist(err) {
-		if err == unix.EBUSY {
-			logger.WithField("dir", mountpoint).WithError(err).Warn("os.Rename err due to EBUSY")
-		}
-		return errors.Wrapf(err, "error preparing atomic delete of aufs mountpoint for id: %s", id)
-	}
-	if err := system.EnsureRemoveAll(tmpMntPath); err != nil {
-		return errors.Wrapf(err, "error removing aufs layer %s", id)
-	}
-
-	tmpDiffpath := path.Join(a.diffPath(), fmt.Sprintf("%s-removing", id))
-	if err := os.Rename(a.getDiffPath(id), tmpDiffpath); err != nil && !os.IsNotExist(err) {
-		return errors.Wrapf(err, "error preparing atomic delete of aufs diff dir for id: %s", id)
 	}
 
 	// Remove the layers file for the id
@@ -344,12 +341,44 @@ func (a *Driver) Remove(id string) error {
 		return errors.Wrapf(err, "error removing layers dir for %s", id)
 	}
 
+	if err := atomicRemove(a.getDiffPath(id)); err != nil {
+		return errors.Wrapf(err, "could not remove diff path for id %s", id)
+	}
+
+	// Atomically remove each directory in turn by first moving it out of the
+	// way (so that docker doesn't find it anymore) before doing removal of
+	// the whole tree.
+	if err := atomicRemove(mountpoint); err != nil {
+		if errors.Cause(err) == unix.EBUSY {
+			logger.WithField("dir", mountpoint).WithError(err).Warn("error performing atomic remove due to EBUSY")
+		}
+		return errors.Wrapf(err, "could not remove mountpoint for id %s", id)
+	}
+
 	a.pathCacheLock.Lock()
 	delete(a.pathCache, id)
 	a.pathCacheLock.Unlock()
 	return nil
 }
 
+func atomicRemove(source string) error {
+	target := source + "-removing"
+
+	err := os.Rename(source, target)
+	switch {
+	case err == nil, os.IsNotExist(err):
+	case os.IsExist(err):
+		// Got error saying the target dir already exists, maybe the source doesn't exist due to a previous (failed) remove
+		if _, e := os.Stat(source); !os.IsNotExist(e) {
+			return errors.Wrapf(err, "target rename dir '%s' exists but should not, this needs to be manually cleaned up")
+		}
+	default:
+		return errors.Wrapf(err, "error preparing atomic delete")
+	}
+
+	return system.EnsureRemoveAll(target)
+}
+
 // Get returns the rootfs path for the id.
 // This will mount the dir at its given path
 func (a *Driver) Get(id, mountLabel string) (string, error) {

components/engine/daemon/graphdriver/aufs/aufs_test.go

@@ -12,6 +12,8 @@ import (
 	"sync"
 	"testing"
 
+	"path/filepath"
+
 	"github.com/docker/docker/daemon/graphdriver"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/reexec"
@@ -147,7 +149,10 @@ func TestRemoveImage(t *testing.T) {
 
 	for _, p := range paths {
 		if _, err := os.Stat(path.Join(tmp, p, "1")); err == nil {
-			t.Fatalf("Error should not be nil because dirs with id 1 should be delted: %s", p)
+			t.Fatalf("Error should not be nil because dirs with id 1 should be deleted: %s", p)
+		}
+		if _, err := os.Stat(path.Join(tmp, p, "1-removing")); err == nil {
+			t.Fatalf("Error should not be nil because dirs with id 1-removing should be deleted: %s", p)
 		}
 	}
 }
@@ -800,3 +805,23 @@ func BenchmarkConcurrentAccess(b *testing.B) {
 		}
 	}
 }
+
+func TestInitStaleCleanup(t *testing.T) {
+	if err := os.MkdirAll(tmp, 0755); err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmp)
+
+	for _, d := range []string{"diff", "mnt"} {
+		if err := os.MkdirAll(filepath.Join(tmp, d, "123-removing"), 0755); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	testInit(tmp, t)
+	for _, d := range []string{"diff", "mnt"} {
+		if _, err := os.Stat(filepath.Join(tmp, d, "123-removing")); err == nil {
+			t.Fatal("cleanup failed")
+		}
+	}
+}

components/engine/integration-cli/docker_api_swarm_service_test.go

@@ -603,7 +603,8 @@ func (s *DockerSwarmSuite) TestAPISwarmServicesStateReporting(c *check.C) {
 
 // Test plugins deployed via swarm services
 func (s *DockerSwarmSuite) TestAPISwarmServicesPlugin(c *check.C) {
-	testRequires(c, DaemonIsLinux, IsAmd64)
+	testRequires(c, ExperimentalDaemon, DaemonIsLinux, IsAmd64)
+
 	reg := setupRegistry(c, false, "", "")
 	defer reg.Close()
 

components/engine/vendor.conf

@@ -27,7 +27,7 @@ github.com/imdario/mergo 0.2.1
 golang.org/x/sync de49d9dcd27d4f764488181bea099dfe6179bcf0
 
 #get libnetwork packages
-github.com/docker/libnetwork 9647f993a81e404639592e8ed73693b48ec09c2e
+github.com/docker/libnetwork 4a242dba7739bb7d4351b260863b5e8789ea5abe
 github.com/docker/go-events 18b43f1bc85d9cdd42c05a6cd2d444c7a200a894
 github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80
 github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec

components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/ov_network.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"runtime"
 	"strconv"
 	"strings"
 	"sync"
@@ -81,6 +82,10 @@ func setDefaultVlan() {
 		logrus.Error("insufficient number of arguments")
 		os.Exit(1)
 	}
+
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
 	nsPath := os.Args[1]
 	ns, err := netns.GetFromPath(nsPath)
 	if err != nil {

components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/peerdb.go

@@ -254,7 +254,7 @@ func (d *driver) peerOpRoutine(ctx context.Context, ch chan *peerOperation) {
 			case peerOperationADD:
 				err = d.peerAddOp(op.networkID, op.endpointID, op.peerIP, op.peerIPMask, op.peerMac, op.vtepIP, op.updateDB, op.l2Miss, op.l3Miss, op.localPeer)
 			case peerOperationDELETE:
-				err = d.peerDeleteOp(op.networkID, op.endpointID, op.peerIP, op.peerIPMask, op.peerMac, op.vtepIP, op.localPeer)
+				err = d.peerDeleteOp(op.networkID, op.endpointID, op.peerIP, op.peerIPMask, op.peerMac, op.vtepIP, op.updateDB)
 			}
 			if err != nil {
 				logrus.Warnf("Peer operation failed:%s op:%v", err, op)