From fd3779deb1e88561ebcb33ba361e36ebd220290d Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 8 Dec 2017 14:02:32 -0800 Subject: [PATCH 01/19] Update go-swagger installation steps in Dockerfile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The installation steps for go-swagger was a bit noisy, and not consistent with other installation steps. This patch makes it similar to other steps, which makes it less noisy, and makes the image slightly smaller. Before: b53d7aac3200 14 minutes ago |1 APT_MIRROR=deb.debian.org /bin/sh -c git … 107MB fa74acf32f99 2 hours ago /bin/sh -c #(nop) ENV GO_SWAGGER_COMMIT=c28… 0B After: 6b2454f1a9a5 10 minutes ago |1 APT_MIRROR=deb.debian.org /bin/sh -c set … 35.2MB fa74acf32f99 2 hours ago /bin/sh -c #(nop) ENV GO_SWAGGER_COMMIT=c28… 0B Signed-off-by: Sebastiaan van Stijn Upstream-commit: 29d77acaf8bfb234ee4f0b3db9e28d7410b99d4e Component: engine --- components/engine/Dockerfile | 9 ++++++--- components/engine/Dockerfile.aarch64 | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile index 5f78eda682..b6d32481b0 100644 --- a/components/engine/Dockerfile +++ b/components/engine/Dockerfile @@ -145,9 +145,12 @@ RUN pip install yamllint==1.5.0 # Install go-swagger for validating swagger.yaml ENV GO_SWAGGER_COMMIT c28258affb0b6251755d92489ef685af8d4ff3eb -RUN git clone https://github.com/go-swagger/go-swagger.git /go/src/github.com/go-swagger/go-swagger \ - && (cd /go/src/github.com/go-swagger/go-swagger && git checkout -q $GO_SWAGGER_COMMIT) \ - && go install -v github.com/go-swagger/go-swagger/cmd/swagger +RUN set -x \ + && export GOPATH="$(mktemp -d)" \ + && git clone https://github.com/go-swagger/go-swagger.git "$GOPATH/src/github.com/go-swagger/go-swagger" \ + && (cd "$GOPATH/src/github.com/go-swagger/go-swagger" && git checkout -q "$GO_SWAGGER_COMMIT") \ + && go build -o /usr/local/bin/swagger github.com/go-swagger/go-swagger/cmd/swagger \ + && rm -rf "$GOPATH" # Set user.email so crosbymichael's in-container merge commits go smoothly RUN git config --global user.email 'docker-dummy@example.com' diff --git a/components/engine/Dockerfile.aarch64 b/components/engine/Dockerfile.aarch64 index 58ca40d878..4a54cd391f 100644 --- a/components/engine/Dockerfile.aarch64 +++ b/components/engine/Dockerfile.aarch64 @@ -118,9 +118,12 @@ RUN pip install yamllint==1.5.0 # Install go-swagger for validating swagger.yaml ENV GO_SWAGGER_COMMIT c28258affb0b6251755d92489ef685af8d4ff3eb -RUN git clone https://github.com/go-swagger/go-swagger.git /go/src/github.com/go-swagger/go-swagger \ - && (cd /go/src/github.com/go-swagger/go-swagger && git checkout -q $GO_SWAGGER_COMMIT) \ - && go install -v github.com/go-swagger/go-swagger/cmd/swagger +RUN set -x \ + && export GOPATH="$(mktemp -d)" \ + && git clone https://github.com/go-swagger/go-swagger.git "$GOPATH/src/github.com/go-swagger/go-swagger" \ + && (cd "$GOPATH/src/github.com/go-swagger/go-swagger" && git checkout -q "$GO_SWAGGER_COMMIT") \ + && go build -o /usr/local/bin/swagger github.com/go-swagger/go-swagger/cmd/swagger \ + && rm -rf "$GOPATH" # Set user.email so crosbymichael's in-container merge commits go smoothly RUN git config --global user.email 'docker-dummy@example.com' From 49c61840e214e3ebdb70425348634b45576f7af6 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Thu, 18 Jan 2018 16:55:27 -0500 Subject: [PATCH 02/19] Use rslave propagation for mounts from daemon root By default, if a user requests a bind mount it uses private propagation. When the source path is a path within the daemon root this, along with some other propagation values that the user can use, causes issues when the daemon tries to remove a mountpoint because a container will then have a private reference to that mount which prevents removal. Unmouting with MNT_DETATCH can help this scenario on newer kernels, but ultimately this is just covering up the problem and doesn't actually free up the underlying resources until all references are destroyed. This change does essentially 2 things: 1. Change the default propagation when unspecified to `rslave` when the source path is within the daemon root path or a parent of the daemon root (because everything is using rbinds). 2. Creates a validation error on create when the user tries to specify an unacceptable propagation mode for these paths... basically the only two acceptable modes are `rslave` and `rshared`. In cases where we have used the new default propagation but the underlying filesystem is not setup to handle it (fs must hvae at least rshared propagation) instead of erroring out like we normally would, this falls back to the old default mode of `private`, which preserves backwards compatibility. Signed-off-by: Brian Goff Upstream-commit: 589a0afa8cbe39b6512662fd1705873e2d236dd0 Component: engine --- components/engine/daemon/oci_linux.go | 34 ++++- components/engine/daemon/volumes.go | 15 ++ components/engine/daemon/volumes_linux.go | 36 +++++ .../engine/daemon/volumes_linux_test.go | 56 ++++++++ components/engine/daemon/volumes_windows.go | 5 + .../container/mounts_linux_test.go | 136 +++++++++++++++++- 6 files changed, 272 insertions(+), 10 deletions(-) create mode 100644 components/engine/daemon/volumes_linux.go create mode 100644 components/engine/daemon/volumes_linux_test.go diff --git a/components/engine/daemon/oci_linux.go b/components/engine/daemon/oci_linux.go index dbc26e8efe..87a22d50eb 100644 --- a/components/engine/daemon/oci_linux.go +++ b/components/engine/daemon/oci_linux.go @@ -604,7 +604,8 @@ func setMounts(daemon *Daemon, s *specs.Spec, c *container.Container, mounts []c // // For private volumes any root propagation value should work. pFlag := mountPropagationMap[m.Propagation] - if pFlag == mount.SHARED || pFlag == mount.RSHARED { + switch pFlag { + case mount.SHARED, mount.RSHARED: if err := ensureShared(m.Source); err != nil { return err } @@ -612,13 +613,34 @@ func setMounts(daemon *Daemon, s *specs.Spec, c *container.Container, mounts []c if rootpg != mount.SHARED && rootpg != mount.RSHARED { s.Linux.RootfsPropagation = mountPropagationReverseMap[mount.SHARED] } - } else if pFlag == mount.SLAVE || pFlag == mount.RSLAVE { + case mount.SLAVE, mount.RSLAVE: + var fallback bool if err := ensureSharedOrSlave(m.Source); err != nil { - return err + // For backwards compatability purposes, treat mounts from the daemon root + // as special since we automatically add rslave propagation to these mounts + // when the user did not set anything, so we should fallback to the old + // behavior which is to use private propagation which is normally the + // default. + if !strings.HasPrefix(m.Source, daemon.root) && !strings.HasPrefix(daemon.root, m.Source) { + return err + } + + cm, ok := c.MountPoints[m.Destination] + if !ok { + return err + } + if cm.Spec.BindOptions != nil && cm.Spec.BindOptions.Propagation != "" { + // This means the user explicitly set a propagation, do not fallback in that case. + return err + } + fallback = true + logrus.WithField("container", c.ID).WithField("source", m.Source).Warn("Falling back to default propagation for bind source in daemon root") } - rootpg := mountPropagationMap[s.Linux.RootfsPropagation] - if rootpg != mount.SHARED && rootpg != mount.RSHARED && rootpg != mount.SLAVE && rootpg != mount.RSLAVE { - s.Linux.RootfsPropagation = mountPropagationReverseMap[mount.RSLAVE] + if !fallback { + rootpg := mountPropagationMap[s.Linux.RootfsPropagation] + if rootpg != mount.SHARED && rootpg != mount.RSHARED && rootpg != mount.SLAVE && rootpg != mount.RSLAVE { + s.Linux.RootfsPropagation = mountPropagationReverseMap[mount.RSLAVE] + } } } diff --git a/components/engine/daemon/volumes.go b/components/engine/daemon/volumes.go index 2e75feebda..7833ea2200 100644 --- a/components/engine/daemon/volumes.go +++ b/components/engine/daemon/volumes.go @@ -10,6 +10,7 @@ import ( "github.com/docker/docker/api/types" containertypes "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/mount" mounttypes "github.com/docker/docker/api/types/mount" "github.com/docker/docker/container" "github.com/docker/docker/errdefs" @@ -146,6 +147,13 @@ func (daemon *Daemon) registerMountPoints(container *container.Container, hostCo if err != nil { return err } + needsSlavePropagation, err := daemon.validateBindDaemonRoot(bind.Spec) + if err != nil { + return err + } + if needsSlavePropagation { + bind.Propagation = mount.PropagationRSlave + } // #10618 _, tmpfsExists := hostConfig.Tmpfs[bind.Destination] @@ -178,6 +186,13 @@ func (daemon *Daemon) registerMountPoints(container *container.Container, hostCo if err != nil { return errdefs.InvalidParameter(err) } + needsSlavePropagation, err := daemon.validateBindDaemonRoot(mp.Spec) + if err != nil { + return err + } + if needsSlavePropagation { + mp.Propagation = mount.PropagationRSlave + } if binds[mp.Destination] { return duplicateMountPointError(cfg.Target) diff --git a/components/engine/daemon/volumes_linux.go b/components/engine/daemon/volumes_linux.go new file mode 100644 index 0000000000..cf3d9ed159 --- /dev/null +++ b/components/engine/daemon/volumes_linux.go @@ -0,0 +1,36 @@ +package daemon + +import ( + "strings" + + "github.com/docker/docker/api/types/mount" + "github.com/docker/docker/errdefs" + "github.com/pkg/errors" +) + +// validateBindDaemonRoot ensures that if a given mountpoint's source is within +// the daemon root path, that the propagation is setup to prevent a container +// from holding private refereneces to a mount within the daemon root, which +// can cause issues when the daemon attempts to remove the mountpoint. +func (daemon *Daemon) validateBindDaemonRoot(m mount.Mount) (bool, error) { + if m.Type != mount.TypeBind { + return false, nil + } + + // check if the source is within the daemon root, or if the daemon root is within the source + if !strings.HasPrefix(m.Source, daemon.root) && !strings.HasPrefix(daemon.root, m.Source) { + return false, nil + } + + if m.BindOptions == nil { + return true, nil + } + + switch m.BindOptions.Propagation { + case mount.PropagationRSlave, mount.PropagationRShared, "": + return m.BindOptions.Propagation == "", nil + default: + } + + return false, errdefs.InvalidParameter(errors.Errorf(`invalid mount config: must use either propagation mode "rslave" or "rshared" when mount source is within the daemon root, daemon root: %q, bind mount source: %q, propagation: %q`, daemon.root, m.Source, m.BindOptions.Propagation)) +} diff --git a/components/engine/daemon/volumes_linux_test.go b/components/engine/daemon/volumes_linux_test.go new file mode 100644 index 0000000000..72830c3e81 --- /dev/null +++ b/components/engine/daemon/volumes_linux_test.go @@ -0,0 +1,56 @@ +package daemon + +import ( + "path/filepath" + "testing" + + "github.com/docker/docker/api/types/mount" +) + +func TestBindDaemonRoot(t *testing.T) { + t.Parallel() + d := &Daemon{root: "/a/b/c/daemon"} + for _, test := range []struct { + desc string + opts *mount.BindOptions + needsProp bool + err bool + }{ + {desc: "nil propagation settings", opts: nil, needsProp: true, err: false}, + {desc: "empty propagation settings", opts: &mount.BindOptions{}, needsProp: true, err: false}, + {desc: "private propagation", opts: &mount.BindOptions{Propagation: mount.PropagationPrivate}, err: true}, + {desc: "rprivate propagation", opts: &mount.BindOptions{Propagation: mount.PropagationRPrivate}, err: true}, + {desc: "slave propagation", opts: &mount.BindOptions{Propagation: mount.PropagationSlave}, err: true}, + {desc: "rslave propagation", opts: &mount.BindOptions{Propagation: mount.PropagationRSlave}, err: false, needsProp: false}, + {desc: "shared propagation", opts: &mount.BindOptions{Propagation: mount.PropagationShared}, err: true}, + {desc: "rshared propagation", opts: &mount.BindOptions{Propagation: mount.PropagationRSlave}, err: false, needsProp: false}, + } { + t.Run(test.desc, func(t *testing.T) { + test := test + for desc, source := range map[string]string{ + "source is root": d.root, + "source is subpath": filepath.Join(d.root, "a", "b"), + "source is parent": filepath.Dir(d.root), + "source is /": "/", + } { + t.Run(desc, func(t *testing.T) { + mount := mount.Mount{ + Type: mount.TypeBind, + Source: source, + BindOptions: test.opts, + } + needsProp, err := d.validateBindDaemonRoot(mount) + if (err != nil) != test.err { + t.Fatalf("expected err=%v, got: %v", test.err, err) + } + if test.err { + return + } + if test.needsProp != needsProp { + t.Fatalf("expected needsProp=%v, got: %v", test.needsProp, needsProp) + } + }) + } + }) + } +} diff --git a/components/engine/daemon/volumes_windows.go b/components/engine/daemon/volumes_windows.go index bfb5133d3d..aced2665e0 100644 --- a/components/engine/daemon/volumes_windows.go +++ b/components/engine/daemon/volumes_windows.go @@ -3,6 +3,7 @@ package daemon import ( "sort" + "github.com/docker/docker/api/types/mount" "github.com/docker/docker/container" "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/volume" @@ -44,3 +45,7 @@ func (daemon *Daemon) setupMounts(c *container.Container) ([]container.Mount, er func setBindModeIfNull(bind *volume.MountPoint) { return } + +func (daemon *Daemon) validateBindDaemonRoot(m mount.Mount) (bool, error) { + return false, nil +} diff --git a/components/engine/integration/container/mounts_linux_test.go b/components/engine/integration/container/mounts_linux_test.go index eab0fd5d74..368234f708 100644 --- a/components/engine/integration/container/mounts_linux_test.go +++ b/components/engine/integration/container/mounts_linux_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "path/filepath" "testing" "github.com/docker/docker/api/types" @@ -12,6 +13,7 @@ import ( "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" "github.com/docker/docker/integration-cli/daemon" + "github.com/docker/docker/integration/util/request" "github.com/docker/docker/pkg/stdcopy" "github.com/docker/docker/pkg/system" "github.com/gotestyourself/gotestyourself/fs" @@ -51,10 +53,10 @@ func TestContainerShmNoLeak(t *testing.T) { hc := container.HostConfig{ Mounts: []mount.Mount{ { - Type: mount.TypeBind, - Source: d.Root, - Target: "/testdaemonroot", - BindOptions: &mount.BindOptions{Propagation: mount.PropagationRPrivate}}, + Type: mount.TypeBind, + Source: d.Root, + Target: "/testdaemonroot", + }, }, } cfg.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("mount | grep testdaemonroot | grep containers | grep %s", ctr.ID)} @@ -141,3 +143,129 @@ func TestContainerNetworkMountsNoChown(t *testing.T) { require.NoError(t, err) assert.Equal(t, uint32(0), statT.UID(), "bind mounted network file should not change ownership from root") } + +func TestMountDaemonRoot(t *testing.T) { + t.Parallel() + + client := request.NewAPIClient(t) + ctx := context.Background() + info, err := client.Info(ctx) + if err != nil { + t.Fatal(err) + } + + for _, test := range []struct { + desc string + propagation mount.Propagation + expected mount.Propagation + }{ + { + desc: "default", + propagation: "", + expected: mount.PropagationRSlave, + }, + { + desc: "private", + propagation: mount.PropagationPrivate, + }, + { + desc: "rprivate", + propagation: mount.PropagationRPrivate, + }, + { + desc: "slave", + propagation: mount.PropagationSlave, + }, + { + desc: "rslave", + propagation: mount.PropagationRSlave, + expected: mount.PropagationRSlave, + }, + { + desc: "shared", + propagation: mount.PropagationShared, + }, + { + desc: "rshared", + propagation: mount.PropagationRShared, + expected: mount.PropagationRShared, + }, + } { + t.Run(test.desc, func(t *testing.T) { + test := test + t.Parallel() + + propagationSpec := fmt.Sprintf(":%s", test.propagation) + if test.propagation == "" { + propagationSpec = "" + } + bindSpecRoot := info.DockerRootDir + ":" + "/foo" + propagationSpec + bindSpecSub := filepath.Join(info.DockerRootDir, "containers") + ":/foo" + propagationSpec + + for name, hc := range map[string]*container.HostConfig{ + "bind root": {Binds: []string{bindSpecRoot}}, + "bind subpath": {Binds: []string{bindSpecSub}}, + "mount root": { + Mounts: []mount.Mount{ + { + Type: mount.TypeBind, + Source: info.DockerRootDir, + Target: "/foo", + BindOptions: &mount.BindOptions{Propagation: test.propagation}, + }, + }, + }, + "mount subpath": { + Mounts: []mount.Mount{ + { + Type: mount.TypeBind, + Source: filepath.Join(info.DockerRootDir, "containers"), + Target: "/foo", + BindOptions: &mount.BindOptions{Propagation: test.propagation}, + }, + }, + }, + } { + t.Run(name, func(t *testing.T) { + hc := hc + t.Parallel() + + c, err := client.ContainerCreate(ctx, &container.Config{ + Image: "busybox", + Cmd: []string{"true"}, + }, hc, nil, "") + + if err != nil { + if test.expected != "" { + t.Fatal(err) + } + // expected an error, so this is ok and should not continue + return + } + if test.expected == "" { + t.Fatal("expected create to fail") + } + + defer func() { + if err := client.ContainerRemove(ctx, c.ID, types.ContainerRemoveOptions{Force: true}); err != nil { + panic(err) + } + }() + + inspect, err := client.ContainerInspect(ctx, c.ID) + if err != nil { + t.Fatal(err) + } + if len(inspect.Mounts) != 1 { + t.Fatalf("unexpected number of mounts: %+v", inspect.Mounts) + } + + m := inspect.Mounts[0] + if m.Propagation != test.expected { + t.Fatalf("got unexpected propagation mode, expected %q, got: %v", test.expected, m.Propagation) + } + }) + } + }) + } +} From 33ddc6d17220d3dc0640bcf1e898c0adf7e077ea Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Wed, 7 Feb 2018 14:49:20 -0500 Subject: [PATCH 03/19] Do not recursive unmount on cleanup of zfs/btrfs This was added in #36047 just as a way to make sure the tree is fully unmounted on shutdown. For ZFS this could be a breaking change since there was no unmount before. Someone could have setup the zfs tree themselves. It would be better, if we really do want the cleanup to actually the unpacked layers checking for mounts rather than a blind recursive unmount of the root. BTRFS does not use mounts and does not need to unmount anyway. These was only an unmount to begin with because for some reason the btrfs tree was being moutned with `private` propagation. For the other graphdrivers that still have a recursive unmount here... these were already being unmounted and performing the recursive unmount shouldn't break anything. If anyone had anything mounted at the graphdriver location it would have been unmounted on shutdown anyway. Signed-off-by: Brian Goff Upstream-commit: 2fe4f888bee52b1f256d6fa5e20f9b061d30221c Component: engine --- components/engine/daemon/graphdriver/btrfs/btrfs.go | 3 +-- components/engine/daemon/graphdriver/zfs/zfs.go | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/engine/daemon/graphdriver/btrfs/btrfs.go b/components/engine/daemon/graphdriver/btrfs/btrfs.go index 54bb9b7902..fbaa96f246 100644 --- a/components/engine/daemon/graphdriver/btrfs/btrfs.go +++ b/components/engine/daemon/graphdriver/btrfs/btrfs.go @@ -29,7 +29,6 @@ import ( "github.com/docker/docker/daemon/graphdriver" "github.com/docker/docker/pkg/containerfs" "github.com/docker/docker/pkg/idtools" - "github.com/docker/docker/pkg/mount" "github.com/docker/docker/pkg/parsers" "github.com/docker/docker/pkg/system" "github.com/docker/go-units" @@ -163,7 +162,7 @@ func (d *Driver) Cleanup() error { return err } - return mount.RecursiveUnmount(d.home) + return nil } func free(p *C.char) { diff --git a/components/engine/daemon/graphdriver/zfs/zfs.go b/components/engine/daemon/graphdriver/zfs/zfs.go index 9ab5d87ad0..743d6daf5b 100644 --- a/components/engine/daemon/graphdriver/zfs/zfs.go +++ b/components/engine/daemon/graphdriver/zfs/zfs.go @@ -178,9 +178,10 @@ func (d *Driver) String() string { return "zfs" } -// Cleanup is called on daemon shutdown, it is used to clean up any remaining mounts +// Cleanup is called on daemon shutdown, it is a no-op for ZFS. +// TODO(@cpuguy83): Walk layer tree and check mounts? func (d *Driver) Cleanup() error { - return mount.RecursiveUnmount(d.options.mountPath) + return nil } // Status returns information about the ZFS filesystem. It returns a two dimensional array of information From d3c6f2e0efb00fc69742447774688c4e86698d06 Mon Sep 17 00:00:00 2001 From: John Howard Date: Fri, 9 Feb 2018 12:03:22 -0800 Subject: [PATCH 04/19] Remove interim env var LCOW_API_PLATFORM_IF_OMITTED Signed-off-by: John Howard Upstream-commit: c111fec758770a37e8674cac312e528e85d89428 Component: engine --- .../engine/api/server/router/build/build_routes.go | 11 ----------- .../engine/api/server/router/image/image_routes.go | 11 ----------- 2 files changed, 22 deletions(-) diff --git a/components/engine/api/server/router/build/build_routes.go b/components/engine/api/server/router/build/build_routes.go index 396797d9d5..09a167062a 100644 --- a/components/engine/api/server/router/build/build_routes.go +++ b/components/engine/api/server/router/build/build_routes.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "net/http" - "os" "runtime" "strconv" "strings" @@ -71,17 +70,7 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui options.Target = r.FormValue("target") options.RemoteContext = r.FormValue("remote") if versions.GreaterThanOrEqualTo(version, "1.32") { - // TODO @jhowardmsft. The following environment variable is an interim - // measure to allow the daemon to have a default platform if omitted by - // the client. This allows LCOW and WCOW to work with a down-level CLI - // for a short period of time, as the CLI changes can't be merged - // until after the daemon changes have been merged. Once the CLI is - // updated, this can be removed. PR for CLI is currently in - // https://github.com/docker/cli/pull/474. apiPlatform := r.FormValue("platform") - if system.LCOWSupported() && apiPlatform == "" { - apiPlatform = os.Getenv("LCOW_API_PLATFORM_IF_OMITTED") - } p := system.ParsePlatform(apiPlatform) if err := system.ValidatePlatform(p); err != nil { return nil, errdefs.InvalidParameter(errors.Errorf("invalid platform: %s", err)) diff --git a/components/engine/api/server/router/image/image_routes.go b/components/engine/api/server/router/image/image_routes.go index 618ccdf0d8..47ca467bb0 100644 --- a/components/engine/api/server/router/image/image_routes.go +++ b/components/engine/api/server/router/image/image_routes.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "net/http" - "os" "strconv" "strings" @@ -86,17 +85,7 @@ func (s *imageRouter) postImagesCreate(ctx context.Context, w http.ResponseWrite version := httputils.VersionFromContext(ctx) if versions.GreaterThanOrEqualTo(version, "1.32") { - // TODO @jhowardmsft. The following environment variable is an interim - // measure to allow the daemon to have a default platform if omitted by - // the client. This allows LCOW and WCOW to work with a down-level CLI - // for a short period of time, as the CLI changes can't be merged - // until after the daemon changes have been merged. Once the CLI is - // updated, this can be removed. PR for CLI is currently in - // https://github.com/docker/cli/pull/474. apiPlatform := r.FormValue("platform") - if system.LCOWSupported() && apiPlatform == "" { - apiPlatform = os.Getenv("LCOW_API_PLATFORM_IF_OMITTED") - } platform = system.ParsePlatform(apiPlatform) if err = system.ValidatePlatform(platform); err != nil { err = fmt.Errorf("invalid platform: %s", err) From 8382b77c1c4b766db3328eaaf1cc75ff57369b48 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 9 Feb 2018 18:24:57 -0500 Subject: [PATCH 05/19] Use TagImage in Commit Signed-off-by: Daniel Nephin Upstream-commit: afb3eda697efebf18d8ac3bbcfd911a5968081e3 Component: engine --- .../engine/api/server/router/image/backend.go | 2 +- .../api/server/router/image/image_routes.go | 2 +- components/engine/daemon/commit.go | 34 ++++--------------- components/engine/daemon/image_tag.go | 11 +++--- 4 files changed, 14 insertions(+), 35 deletions(-) diff --git a/components/engine/api/server/router/image/backend.go b/components/engine/api/server/router/image/backend.go index dcf554cef3..ffbc9c181a 100644 --- a/components/engine/api/server/router/image/backend.go +++ b/components/engine/api/server/router/image/backend.go @@ -29,7 +29,7 @@ type imageBackend interface { ImageHistory(imageName string) ([]*image.HistoryResponseItem, error) Images(imageFilters filters.Args, all bool, withExtraAttrs bool) ([]*types.ImageSummary, error) LookupImage(name string) (*types.ImageInspect, error) - TagImage(imageName, repository, tag string) error + TagImage(imageName, repository, tag string) (string, error) ImagesPrune(ctx context.Context, pruneFilters filters.Args) (*types.ImagesPruneReport, error) } diff --git a/components/engine/api/server/router/image/image_routes.go b/components/engine/api/server/router/image/image_routes.go index 618ccdf0d8..9358a52231 100644 --- a/components/engine/api/server/router/image/image_routes.go +++ b/components/engine/api/server/router/image/image_routes.go @@ -303,7 +303,7 @@ func (s *imageRouter) postImagesTag(ctx context.Context, w http.ResponseWriter, if err := httputils.ParseForm(r); err != nil { return err } - if err := s.backend.TagImage(vars["name"], r.Form.Get("repo"), r.Form.Get("tag")); err != nil { + if _, err := s.backend.TagImage(vars["name"], r.Form.Get("repo"), r.Form.Get("tag")); err != nil { return err } w.WriteHeader(http.StatusCreated) diff --git a/components/engine/daemon/commit.go b/components/engine/daemon/commit.go index 5f2e3c689d..8865d832f3 100644 --- a/components/engine/daemon/commit.go +++ b/components/engine/daemon/commit.go @@ -8,7 +8,6 @@ import ( "strings" "time" - "github.com/docker/distribution/reference" "github.com/docker/docker/api/types/backend" containertypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/builder/dockerfile" @@ -176,9 +175,12 @@ func (daemon *Daemon) CreateImageFromContainer(name string, c *backend.CreateIma return "", err } - imageRef, err := daemon.tagCommit(c.Repo, c.Tag, id) - if err != nil { - return "", err + var imageRef string + if c.Repo != "" { + imageRef, err = daemon.TagImage(string(id), c.Repo, c.Tag) + if err != nil { + return "", err + } } daemon.LogContainerEventWithAttributes(container, "commit", map[string]string{ "comment": c.Comment, @@ -247,30 +249,6 @@ func (daemon *Daemon) commitImage(c backend.CommitConfig) (image.ID, error) { return id, nil } -// TODO: remove from Daemon, move to api backend -func (daemon *Daemon) tagCommit(repo string, tag string, id image.ID) (string, error) { - imageRef := "" - if repo != "" { - newTag, err := reference.ParseNormalizedNamed(repo) // todo: should move this to API layer - if err != nil { - return "", err - } - if !reference.IsNameOnly(newTag) { - return "", errors.Errorf("unexpected repository name: %s", repo) - } - if tag != "" { - if newTag, err = reference.WithTag(newTag, tag); err != nil { - return "", err - } - } - if err := daemon.TagImageWithReference(id, newTag); err != nil { - return "", err - } - imageRef = reference.FamiliarString(newTag) - } - return imageRef, nil -} - func exportContainerRw(layerStore layer.Store, id, mountLabel string) (arch io.ReadCloser, err error) { rwlayer, err := layerStore.GetRWLayer(id) if err != nil { diff --git a/components/engine/daemon/image_tag.go b/components/engine/daemon/image_tag.go index 80abd9f158..56b325f66d 100644 --- a/components/engine/daemon/image_tag.go +++ b/components/engine/daemon/image_tag.go @@ -7,23 +7,24 @@ import ( // TagImage creates the tag specified by newTag, pointing to the image named // imageName (alternatively, imageName can also be an image ID). -func (daemon *Daemon) TagImage(imageName, repository, tag string) error { +func (daemon *Daemon) TagImage(imageName, repository, tag string) (string, error) { imageID, _, err := daemon.GetImageIDAndOS(imageName) if err != nil { - return err + return "", err } newTag, err := reference.ParseNormalizedNamed(repository) if err != nil { - return err + return "", err } if tag != "" { if newTag, err = reference.WithTag(reference.TrimNamed(newTag), tag); err != nil { - return err + return "", err } } - return daemon.TagImageWithReference(imageID, newTag) + err = daemon.TagImageWithReference(imageID, newTag) + return reference.FamiliarString(newTag), err } // TagImageWithReference adds the given reference to the image ID provided. From f0113d4e5a76f38716e74d0611b775055da5fb1d Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Mon, 12 Feb 2018 15:27:34 -0500 Subject: [PATCH 06/19] Use continuity fs package for volume copy Signed-off-by: Brian Goff Upstream-commit: b3aab5e31faf04d8a29f17be55562e4d0c0cb364 Component: engine --- components/engine/container/container_unix.go | 51 +-- components/engine/vendor.conf | 2 +- .../continuity/devices/devices_darwin.go | 15 - .../continuity/devices/devices_dummy.go | 23 -- .../continuity/devices/devices_freebsd.go | 15 - .../continuity/devices/devices_linux.go | 15 - .../continuity/devices/devices_solaris.go | 18 - .../continuity/devices/devices_unix.go | 23 +- .../containerd/continuity/fs/copy.go | 119 +++++++ .../containerd/continuity/fs/copy_linux.go | 95 ++++++ .../containerd/continuity/fs/copy_unix.go | 80 +++++ .../containerd/continuity/fs/copy_windows.go | 33 ++ .../containerd/continuity/fs/diff.go | 310 ++++++++++++++++++ .../containerd/continuity/fs/diff_unix.go | 58 ++++ .../containerd/continuity/fs/diff_windows.go | 32 ++ .../containerd/continuity/fs/dtype_linux.go | 87 +++++ .../github.com/containerd/continuity/fs/du.go | 22 ++ .../containerd/continuity/fs/du_unix.go | 88 +++++ .../containerd/continuity/fs/du_windows.go | 60 ++++ .../containerd/continuity/fs/hardlink.go | 27 ++ .../containerd/continuity/fs/hardlink_unix.go | 18 + .../continuity/fs/hardlink_windows.go | 7 + .../containerd/continuity/fs/path.go | 276 ++++++++++++++++ .../containerd/continuity/fs/stat_bsd.go | 28 ++ .../containerd/continuity/fs/stat_linux.go | 26 ++ .../containerd/continuity/fs/time.go | 13 + .../containerd/continuity/sysx/copy_linux.go | 11 - .../continuity/sysx/copy_linux_386.go | 20 -- .../continuity/sysx/copy_linux_amd64.go | 20 -- .../continuity/sysx/copy_linux_arm.go | 20 -- .../continuity/sysx/copy_linux_arm64.go | 20 -- .../continuity/sysx/copy_linux_ppc64le.go | 20 -- .../continuity/sysx/copy_linux_s390x.go | 20 -- .../continuity/sysx/sysnum_linux_386.go | 7 - .../continuity/sysx/sysnum_linux_amd64.go | 7 - .../continuity/sysx/sysnum_linux_arm.go | 7 - .../continuity/sysx/sysnum_linux_arm64.go | 7 - .../continuity/sysx/sysnum_linux_ppc64le.go | 7 - .../continuity/sysx/sysnum_linux_s390x.go | 7 - .../containerd/continuity/vendor.conf | 13 + 40 files changed, 1412 insertions(+), 315 deletions(-) delete mode 100644 components/engine/vendor/github.com/containerd/continuity/devices/devices_darwin.go delete mode 100644 components/engine/vendor/github.com/containerd/continuity/devices/devices_dummy.go delete mode 100644 components/engine/vendor/github.com/containerd/continuity/devices/devices_freebsd.go delete mode 100644 components/engine/vendor/github.com/containerd/continuity/devices/devices_linux.go delete mode 100644 components/engine/vendor/github.com/containerd/continuity/devices/devices_solaris.go create mode 100644 components/engine/vendor/github.com/containerd/continuity/fs/copy.go create mode 100644 components/engine/vendor/github.com/containerd/continuity/fs/copy_linux.go create mode 100644 components/engine/vendor/github.com/containerd/continuity/fs/copy_unix.go create mode 100644 components/engine/vendor/github.com/containerd/continuity/fs/copy_windows.go create mode 100644 components/engine/vendor/github.com/containerd/continuity/fs/diff.go create mode 100644 components/engine/vendor/github.com/containerd/continuity/fs/diff_unix.go create mode 100644 components/engine/vendor/github.com/containerd/continuity/fs/diff_windows.go create mode 100644 components/engine/vendor/github.com/containerd/continuity/fs/dtype_linux.go create mode 100644 components/engine/vendor/github.com/containerd/continuity/fs/du.go create mode 100644 components/engine/vendor/github.com/containerd/continuity/fs/du_unix.go create mode 100644 components/engine/vendor/github.com/containerd/continuity/fs/du_windows.go create mode 100644 components/engine/vendor/github.com/containerd/continuity/fs/hardlink.go create mode 100644 components/engine/vendor/github.com/containerd/continuity/fs/hardlink_unix.go create mode 100644 components/engine/vendor/github.com/containerd/continuity/fs/hardlink_windows.go create mode 100644 components/engine/vendor/github.com/containerd/continuity/fs/path.go create mode 100644 components/engine/vendor/github.com/containerd/continuity/fs/stat_bsd.go create mode 100644 components/engine/vendor/github.com/containerd/continuity/fs/stat_linux.go create mode 100644 components/engine/vendor/github.com/containerd/continuity/fs/time.go delete mode 100644 components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux.go delete mode 100644 components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_386.go delete mode 100644 components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_amd64.go delete mode 100644 components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_arm.go delete mode 100644 components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_arm64.go delete mode 100644 components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_ppc64le.go delete mode 100644 components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_s390x.go delete mode 100644 components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_386.go delete mode 100644 components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_amd64.go delete mode 100644 components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_arm.go delete mode 100644 components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_arm64.go delete mode 100644 components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_ppc64le.go delete mode 100644 components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_s390x.go create mode 100644 components/engine/vendor/github.com/containerd/continuity/vendor.conf diff --git a/components/engine/container/container_unix.go b/components/engine/container/container_unix.go index 32805e5271..6f4d91b919 100644 --- a/components/engine/container/container_unix.go +++ b/components/engine/container/container_unix.go @@ -6,13 +6,12 @@ import ( "io/ioutil" "os" + "github.com/containerd/continuity/fs" "github.com/docker/docker/api/types" containertypes "github.com/docker/docker/api/types/container" mounttypes "github.com/docker/docker/api/types/mount" - "github.com/docker/docker/pkg/chrootarchive" "github.com/docker/docker/pkg/mount" "github.com/docker/docker/pkg/stringid" - "github.com/docker/docker/pkg/system" "github.com/docker/docker/volume" "github.com/opencontainers/selinux/go-selinux/label" "github.com/pkg/errors" @@ -398,53 +397,15 @@ func (container *Container) DetachAndUnmount(volumeEventLog func(name, action st // copyExistingContents copies from the source to the destination and // ensures the ownership is appropriately set. func copyExistingContents(source, destination string) error { - volList, err := ioutil.ReadDir(source) + dstList, err := ioutil.ReadDir(destination) if err != nil { return err } - if len(volList) > 0 { - srcList, err := ioutil.ReadDir(destination) - if err != nil { - return err - } - if len(srcList) == 0 { - // If the source volume is empty, copies files from the root into the volume - if err := chrootarchive.NewArchiver(nil).CopyWithTar(source, destination); err != nil { - return err - } - } + if len(dstList) != 0 { + // destination is not empty, do not copy + return nil } - return copyOwnership(source, destination) -} - -// copyOwnership copies the permissions and uid:gid of the source file -// to the destination file -func copyOwnership(source, destination string) error { - stat, err := system.Stat(source) - if err != nil { - return err - } - - destStat, err := system.Stat(destination) - if err != nil { - return err - } - - // In some cases, even though UID/GID match and it would effectively be a no-op, - // this can return a permission denied error... for example if this is an NFS - // mount. - // Since it's not really an error that we can't chown to the same UID/GID, don't - // even bother trying in such cases. - if stat.UID() != destStat.UID() || stat.GID() != destStat.GID() { - if err := os.Chown(destination, int(stat.UID()), int(stat.GID())); err != nil { - return err - } - } - - if stat.Mode() != destStat.Mode() { - return os.Chmod(destination, os.FileMode(stat.Mode())) - } - return nil + return fs.CopyDir(destination, source) } // TmpfsMounts returns the list of tmpfs mounts diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index d2e1a21ed9..8f773f9bae 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -106,7 +106,7 @@ google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944 # containerd github.com/containerd/containerd 3fa104f843ec92328912e042b767d26825f202aa github.com/containerd/fifo fbfb6a11ec671efbe94ad1c12c2e98773f19e1e6 -github.com/containerd/continuity 35d55c5e8dd23b32037d56cf97174aff3efdfa83 +github.com/containerd/continuity 992a5f112bd2211d0983a1cc8562d2882848f3a3 github.com/containerd/cgroups 29da22c6171a4316169f9205ab6c49f59b5b852f github.com/containerd/console 84eeaae905fa414d03e07bcd6c8d3f19e7cf180e github.com/containerd/go-runc ed1cbe1fc31f5fb2359d3a54b6330d1a097858b7 diff --git a/components/engine/vendor/github.com/containerd/continuity/devices/devices_darwin.go b/components/engine/vendor/github.com/containerd/continuity/devices/devices_darwin.go deleted file mode 100644 index 5041e66611..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/devices/devices_darwin.go +++ /dev/null @@ -1,15 +0,0 @@ -package devices - -// from /usr/include/sys/types.h - -func getmajor(dev int32) uint64 { - return (uint64(dev) >> 24) & 0xff -} - -func getminor(dev int32) uint64 { - return uint64(dev) & 0xffffff -} - -func makedev(major int, minor int) int { - return ((major << 24) | minor) -} diff --git a/components/engine/vendor/github.com/containerd/continuity/devices/devices_dummy.go b/components/engine/vendor/github.com/containerd/continuity/devices/devices_dummy.go deleted file mode 100644 index 9a48330a56..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/devices/devices_dummy.go +++ /dev/null @@ -1,23 +0,0 @@ -// +build solaris,!cgo - -// -// Implementing the functions below requires cgo support. Non-cgo stubs -// versions are defined below to enable cross-compilation of source code -// that depends on these functions, but the resultant cross-compiled -// binaries cannot actually be used. If the stub function(s) below are -// actually invoked they will cause the calling process to exit. -// - -package devices - -func getmajor(dev uint64) uint64 { - panic("getmajor() support requires cgo.") -} - -func getminor(dev uint64) uint64 { - panic("getminor() support requires cgo.") -} - -func makedev(major int, minor int) int { - panic("makedev() support requires cgo.") -} diff --git a/components/engine/vendor/github.com/containerd/continuity/devices/devices_freebsd.go b/components/engine/vendor/github.com/containerd/continuity/devices/devices_freebsd.go deleted file mode 100644 index a5c7b93189..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/devices/devices_freebsd.go +++ /dev/null @@ -1,15 +0,0 @@ -package devices - -// from /usr/include/sys/types.h - -func getmajor(dev uint32) uint64 { - return (uint64(dev) >> 24) & 0xff -} - -func getminor(dev uint32) uint64 { - return uint64(dev) & 0xffffff -} - -func makedev(major int, minor int) int { - return ((major << 24) | minor) -} diff --git a/components/engine/vendor/github.com/containerd/continuity/devices/devices_linux.go b/components/engine/vendor/github.com/containerd/continuity/devices/devices_linux.go deleted file mode 100644 index 454cf668f5..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/devices/devices_linux.go +++ /dev/null @@ -1,15 +0,0 @@ -package devices - -// from /usr/include/linux/kdev_t.h - -func getmajor(dev uint64) uint64 { - return dev >> 8 -} - -func getminor(dev uint64) uint64 { - return dev & 0xff -} - -func makedev(major int, minor int) int { - return ((major << 8) | minor) -} diff --git a/components/engine/vendor/github.com/containerd/continuity/devices/devices_solaris.go b/components/engine/vendor/github.com/containerd/continuity/devices/devices_solaris.go deleted file mode 100644 index 8819ac82f5..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/devices/devices_solaris.go +++ /dev/null @@ -1,18 +0,0 @@ -// +build cgo - -package devices - -//#include -import "C" - -func getmajor(dev uint64) uint64 { - return uint64(C.major(C.dev_t(dev))) -} - -func getminor(dev uint64) uint64 { - return uint64(C.minor(C.dev_t(dev))) -} - -func makedev(major int, minor int) int { - return int(C.makedev(C.major_t(major), C.minor_t(minor))) -} diff --git a/components/engine/vendor/github.com/containerd/continuity/devices/devices_unix.go b/components/engine/vendor/github.com/containerd/continuity/devices/devices_unix.go index 85e9a68c49..97fe6b19d2 100644 --- a/components/engine/vendor/github.com/containerd/continuity/devices/devices_unix.go +++ b/components/engine/vendor/github.com/containerd/continuity/devices/devices_unix.go @@ -6,6 +6,8 @@ import ( "fmt" "os" "syscall" + + "golang.org/x/sys/unix" ) func DeviceInfo(fi os.FileInfo) (uint64, uint64, error) { @@ -14,42 +16,43 @@ func DeviceInfo(fi os.FileInfo) (uint64, uint64, error) { return 0, 0, fmt.Errorf("cannot extract device from os.FileInfo") } - return getmajor(sys.Rdev), getminor(sys.Rdev), nil + dev := uint64(sys.Rdev) + return uint64(unix.Major(dev)), uint64(unix.Minor(dev)), nil } // mknod provides a shortcut for syscall.Mknod func Mknod(p string, mode os.FileMode, maj, min int) error { var ( m = syscallMode(mode.Perm()) - dev int + dev uint64 ) if mode&os.ModeDevice != 0 { - dev = makedev(maj, min) + dev = unix.Mkdev(uint32(maj), uint32(min)) if mode&os.ModeCharDevice != 0 { - m |= syscall.S_IFCHR + m |= unix.S_IFCHR } else { - m |= syscall.S_IFBLK + m |= unix.S_IFBLK } } else if mode&os.ModeNamedPipe != 0 { - m |= syscall.S_IFIFO + m |= unix.S_IFIFO } - return syscall.Mknod(p, m, dev) + return unix.Mknod(p, m, int(dev)) } // syscallMode returns the syscall-specific mode bits from Go's portable mode bits. func syscallMode(i os.FileMode) (o uint32) { o |= uint32(i.Perm()) if i&os.ModeSetuid != 0 { - o |= syscall.S_ISUID + o |= unix.S_ISUID } if i&os.ModeSetgid != 0 { - o |= syscall.S_ISGID + o |= unix.S_ISGID } if i&os.ModeSticky != 0 { - o |= syscall.S_ISVTX + o |= unix.S_ISVTX } return } diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/copy.go b/components/engine/vendor/github.com/containerd/continuity/fs/copy.go new file mode 100644 index 0000000000..e8f452819b --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/copy.go @@ -0,0 +1,119 @@ +package fs + +import ( + "io/ioutil" + "os" + "path/filepath" + "sync" + + "github.com/pkg/errors" +) + +var bufferPool = &sync.Pool{ + New: func() interface{} { + buffer := make([]byte, 32*1024) + return &buffer + }, +} + +// CopyDir copies the directory from src to dst. +// Most efficient copy of files is attempted. +func CopyDir(dst, src string) error { + inodes := map[uint64]string{} + return copyDirectory(dst, src, inodes) +} + +func copyDirectory(dst, src string, inodes map[uint64]string) error { + stat, err := os.Stat(src) + if err != nil { + return errors.Wrapf(err, "failed to stat %s", src) + } + if !stat.IsDir() { + return errors.Errorf("source is not directory") + } + + if st, err := os.Stat(dst); err != nil { + if err := os.Mkdir(dst, stat.Mode()); err != nil { + return errors.Wrapf(err, "failed to mkdir %s", dst) + } + } else if !st.IsDir() { + return errors.Errorf("cannot copy to non-directory: %s", dst) + } else { + if err := os.Chmod(dst, stat.Mode()); err != nil { + return errors.Wrapf(err, "failed to chmod on %s", dst) + } + } + + fis, err := ioutil.ReadDir(src) + if err != nil { + return errors.Wrapf(err, "failed to read %s", src) + } + + if err := copyFileInfo(stat, dst); err != nil { + return errors.Wrapf(err, "failed to copy file info for %s", dst) + } + + for _, fi := range fis { + source := filepath.Join(src, fi.Name()) + target := filepath.Join(dst, fi.Name()) + + switch { + case fi.IsDir(): + if err := copyDirectory(target, source, inodes); err != nil { + return err + } + continue + case (fi.Mode() & os.ModeType) == 0: + link, err := getLinkSource(target, fi, inodes) + if err != nil { + return errors.Wrap(err, "failed to get hardlink") + } + if link != "" { + if err := os.Link(link, target); err != nil { + return errors.Wrap(err, "failed to create hard link") + } + } else if err := copyFile(source, target); err != nil { + return errors.Wrap(err, "failed to copy files") + } + case (fi.Mode() & os.ModeSymlink) == os.ModeSymlink: + link, err := os.Readlink(source) + if err != nil { + return errors.Wrapf(err, "failed to read link: %s", source) + } + if err := os.Symlink(link, target); err != nil { + return errors.Wrapf(err, "failed to create symlink: %s", target) + } + case (fi.Mode() & os.ModeDevice) == os.ModeDevice: + if err := copyDevice(target, fi); err != nil { + return errors.Wrapf(err, "failed to create device") + } + default: + // TODO: Support pipes and sockets + return errors.Wrapf(err, "unsupported mode %s", fi.Mode()) + } + if err := copyFileInfo(fi, target); err != nil { + return errors.Wrap(err, "failed to copy file info") + } + + if err := copyXAttrs(target, source); err != nil { + return errors.Wrap(err, "failed to copy xattrs") + } + } + + return nil +} + +func copyFile(source, target string) error { + src, err := os.Open(source) + if err != nil { + return errors.Wrapf(err, "failed to open source %s", source) + } + defer src.Close() + tgt, err := os.Create(target) + if err != nil { + return errors.Wrapf(err, "failed to open target %s", target) + } + defer tgt.Close() + + return copyFileContent(tgt, src) +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/copy_linux.go b/components/engine/vendor/github.com/containerd/continuity/fs/copy_linux.go new file mode 100644 index 0000000000..cfab6756b8 --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/copy_linux.go @@ -0,0 +1,95 @@ +package fs + +import ( + "io" + "os" + "syscall" + + "github.com/containerd/continuity/sysx" + "github.com/pkg/errors" + "golang.org/x/sys/unix" +) + +func copyFileInfo(fi os.FileInfo, name string) error { + st := fi.Sys().(*syscall.Stat_t) + if err := os.Lchown(name, int(st.Uid), int(st.Gid)); err != nil { + if os.IsPermission(err) { + // Normally if uid/gid are the same this would be a no-op, but some + // filesystems may still return EPERM... for instance NFS does this. + // In such a case, this is not an error. + if dstStat, err2 := os.Lstat(name); err2 == nil { + st2 := dstStat.Sys().(*syscall.Stat_t) + if st.Uid == st2.Uid && st.Gid == st2.Gid { + err = nil + } + } + } + if err != nil { + return errors.Wrapf(err, "failed to chown %s", name) + } + } + + if (fi.Mode() & os.ModeSymlink) != os.ModeSymlink { + if err := os.Chmod(name, fi.Mode()); err != nil { + return errors.Wrapf(err, "failed to chmod %s", name) + } + } + + timespec := []unix.Timespec{unix.Timespec(StatAtime(st)), unix.Timespec(StatMtime(st))} + if err := unix.UtimesNanoAt(unix.AT_FDCWD, name, timespec, unix.AT_SYMLINK_NOFOLLOW); err != nil { + return errors.Wrapf(err, "failed to utime %s", name) + } + + return nil +} + +func copyFileContent(dst, src *os.File) error { + st, err := src.Stat() + if err != nil { + return errors.Wrap(err, "unable to stat source") + } + + n, err := unix.CopyFileRange(int(src.Fd()), nil, int(dst.Fd()), nil, int(st.Size()), 0) + if err != nil { + if err != unix.ENOSYS && err != unix.EXDEV { + return errors.Wrap(err, "copy file range failed") + } + + buf := bufferPool.Get().(*[]byte) + _, err = io.CopyBuffer(dst, src, *buf) + bufferPool.Put(buf) + return err + } + + if int64(n) != st.Size() { + return errors.Wrapf(err, "short copy: %d of %d", int64(n), st.Size()) + } + + return nil +} + +func copyXAttrs(dst, src string) error { + xattrKeys, err := sysx.LListxattr(src) + if err != nil { + return errors.Wrapf(err, "failed to list xattrs on %s", src) + } + for _, xattr := range xattrKeys { + data, err := sysx.LGetxattr(src, xattr) + if err != nil { + return errors.Wrapf(err, "failed to get xattr %q on %s", xattr, src) + } + if err := sysx.LSetxattr(dst, xattr, data, 0); err != nil { + return errors.Wrapf(err, "failed to set xattr %q on %s", xattr, dst) + } + } + + return nil +} + +func copyDevice(dst string, fi os.FileInfo) error { + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return errors.New("unsupported stat type") + } + return unix.Mknod(dst, uint32(fi.Mode()), int(st.Rdev)) +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/copy_unix.go b/components/engine/vendor/github.com/containerd/continuity/fs/copy_unix.go new file mode 100644 index 0000000000..29cbb81ed5 --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/copy_unix.go @@ -0,0 +1,80 @@ +// +build solaris darwin freebsd + +package fs + +import ( + "io" + "os" + "syscall" + + "github.com/containerd/continuity/sysx" + "github.com/pkg/errors" + "golang.org/x/sys/unix" +) + +func copyFileInfo(fi os.FileInfo, name string) error { + st := fi.Sys().(*syscall.Stat_t) + if err := os.Lchown(name, int(st.Uid), int(st.Gid)); err != nil { + if os.IsPermission(err) { + // Normally if uid/gid are the same this would be a no-op, but some + // filesystems may still return EPERM... for instance NFS does this. + // In such a case, this is not an error. + if dstStat, err2 := os.Lstat(name); err2 == nil { + st2 := dstStat.Sys().(*syscall.Stat_t) + if st.Uid == st2.Uid && st.Gid == st2.Gid { + err = nil + } + } + } + if err != nil { + return errors.Wrapf(err, "failed to chown %s", name) + } + } + + if (fi.Mode() & os.ModeSymlink) != os.ModeSymlink { + if err := os.Chmod(name, fi.Mode()); err != nil { + return errors.Wrapf(err, "failed to chmod %s", name) + } + } + + timespec := []syscall.Timespec{StatAtime(st), StatMtime(st)} + if err := syscall.UtimesNano(name, timespec); err != nil { + return errors.Wrapf(err, "failed to utime %s", name) + } + + return nil +} + +func copyFileContent(dst, src *os.File) error { + buf := bufferPool.Get().(*[]byte) + _, err := io.CopyBuffer(dst, src, *buf) + bufferPool.Put(buf) + + return err +} + +func copyXAttrs(dst, src string) error { + xattrKeys, err := sysx.LListxattr(src) + if err != nil { + return errors.Wrapf(err, "failed to list xattrs on %s", src) + } + for _, xattr := range xattrKeys { + data, err := sysx.LGetxattr(src, xattr) + if err != nil { + return errors.Wrapf(err, "failed to get xattr %q on %s", xattr, src) + } + if err := sysx.LSetxattr(dst, xattr, data, 0); err != nil { + return errors.Wrapf(err, "failed to set xattr %q on %s", xattr, dst) + } + } + + return nil +} + +func copyDevice(dst string, fi os.FileInfo) error { + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return errors.New("unsupported stat type") + } + return unix.Mknod(dst, uint32(fi.Mode()), int(st.Rdev)) +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/copy_windows.go b/components/engine/vendor/github.com/containerd/continuity/fs/copy_windows.go new file mode 100644 index 0000000000..6fb3de5710 --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/copy_windows.go @@ -0,0 +1,33 @@ +package fs + +import ( + "io" + "os" + + "github.com/pkg/errors" +) + +func copyFileInfo(fi os.FileInfo, name string) error { + if err := os.Chmod(name, fi.Mode()); err != nil { + return errors.Wrapf(err, "failed to chmod %s", name) + } + + // TODO: copy windows specific metadata + + return nil +} + +func copyFileContent(dst, src *os.File) error { + buf := bufferPool.Get().(*[]byte) + _, err := io.CopyBuffer(dst, src, *buf) + bufferPool.Put(buf) + return err +} + +func copyXAttrs(dst, src string) error { + return nil +} + +func copyDevice(dst string, fi os.FileInfo) error { + return errors.New("device copy not supported") +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/diff.go b/components/engine/vendor/github.com/containerd/continuity/fs/diff.go new file mode 100644 index 0000000000..f2300e845d --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/diff.go @@ -0,0 +1,310 @@ +package fs + +import ( + "context" + "os" + "path/filepath" + "strings" + + "golang.org/x/sync/errgroup" + + "github.com/sirupsen/logrus" +) + +// ChangeKind is the type of modification that +// a change is making. +type ChangeKind int + +const ( + // ChangeKindUnmodified represents an unmodified + // file + ChangeKindUnmodified = iota + + // ChangeKindAdd represents an addition of + // a file + ChangeKindAdd + + // ChangeKindModify represents a change to + // an existing file + ChangeKindModify + + // ChangeKindDelete represents a delete of + // a file + ChangeKindDelete +) + +func (k ChangeKind) String() string { + switch k { + case ChangeKindUnmodified: + return "unmodified" + case ChangeKindAdd: + return "add" + case ChangeKindModify: + return "modify" + case ChangeKindDelete: + return "delete" + default: + return "" + } +} + +// Change represents single change between a diff and its parent. +type Change struct { + Kind ChangeKind + Path string +} + +// ChangeFunc is the type of function called for each change +// computed during a directory changes calculation. +type ChangeFunc func(ChangeKind, string, os.FileInfo, error) error + +// Changes computes changes between two directories calling the +// given change function for each computed change. The first +// directory is intended to the base directory and second +// directory the changed directory. +// +// The change callback is called by the order of path names and +// should be appliable in that order. +// Due to this apply ordering, the following is true +// - Removed directory trees only create a single change for the root +// directory removed. Remaining changes are implied. +// - A directory which is modified to become a file will not have +// delete entries for sub-path items, their removal is implied +// by the removal of the parent directory. +// +// Opaque directories will not be treated specially and each file +// removed from the base directory will show up as a removal. +// +// File content comparisons will be done on files which have timestamps +// which may have been truncated. If either of the files being compared +// has a zero value nanosecond value, each byte will be compared for +// differences. If 2 files have the same seconds value but different +// nanosecond values where one of those values is zero, the files will +// be considered unchanged if the content is the same. This behavior +// is to account for timestamp truncation during archiving. +func Changes(ctx context.Context, a, b string, changeFn ChangeFunc) error { + if a == "" { + logrus.Debugf("Using single walk diff for %s", b) + return addDirChanges(ctx, changeFn, b) + } else if diffOptions := detectDirDiff(b, a); diffOptions != nil { + logrus.Debugf("Using single walk diff for %s from %s", diffOptions.diffDir, a) + return diffDirChanges(ctx, changeFn, a, diffOptions) + } + + logrus.Debugf("Using double walk diff for %s from %s", b, a) + return doubleWalkDiff(ctx, changeFn, a, b) +} + +func addDirChanges(ctx context.Context, changeFn ChangeFunc, root string) error { + return filepath.Walk(root, func(path string, f os.FileInfo, err error) error { + if err != nil { + return err + } + + // Rebase path + path, err = filepath.Rel(root, path) + if err != nil { + return err + } + + path = filepath.Join(string(os.PathSeparator), path) + + // Skip root + if path == string(os.PathSeparator) { + return nil + } + + return changeFn(ChangeKindAdd, path, f, nil) + }) +} + +// diffDirOptions is used when the diff can be directly calculated from +// a diff directory to its base, without walking both trees. +type diffDirOptions struct { + diffDir string + skipChange func(string) (bool, error) + deleteChange func(string, string, os.FileInfo) (string, error) +} + +// diffDirChanges walks the diff directory and compares changes against the base. +func diffDirChanges(ctx context.Context, changeFn ChangeFunc, base string, o *diffDirOptions) error { + changedDirs := make(map[string]struct{}) + return filepath.Walk(o.diffDir, func(path string, f os.FileInfo, err error) error { + if err != nil { + return err + } + + // Rebase path + path, err = filepath.Rel(o.diffDir, path) + if err != nil { + return err + } + + path = filepath.Join(string(os.PathSeparator), path) + + // Skip root + if path == string(os.PathSeparator) { + return nil + } + + // TODO: handle opaqueness, start new double walker at this + // location to get deletes, and skip tree in single walker + + if o.skipChange != nil { + if skip, err := o.skipChange(path); skip { + return err + } + } + + var kind ChangeKind + + deletedFile, err := o.deleteChange(o.diffDir, path, f) + if err != nil { + return err + } + + // Find out what kind of modification happened + if deletedFile != "" { + path = deletedFile + kind = ChangeKindDelete + f = nil + } else { + // Otherwise, the file was added + kind = ChangeKindAdd + + // ...Unless it already existed in a base, in which case, it's a modification + stat, err := os.Stat(filepath.Join(base, path)) + if err != nil && !os.IsNotExist(err) { + return err + } + if err == nil { + // The file existed in the base, so that's a modification + + // However, if it's a directory, maybe it wasn't actually modified. + // If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar + if stat.IsDir() && f.IsDir() { + if f.Size() == stat.Size() && f.Mode() == stat.Mode() && sameFsTime(f.ModTime(), stat.ModTime()) { + // Both directories are the same, don't record the change + return nil + } + } + kind = ChangeKindModify + } + } + + // If /foo/bar/file.txt is modified, then /foo/bar must be part of the changed files. + // This block is here to ensure the change is recorded even if the + // modify time, mode and size of the parent directory in the rw and ro layers are all equal. + // Check https://github.com/docker/docker/pull/13590 for details. + if f.IsDir() { + changedDirs[path] = struct{}{} + } + if kind == ChangeKindAdd || kind == ChangeKindDelete { + parent := filepath.Dir(path) + if _, ok := changedDirs[parent]; !ok && parent != "/" { + pi, err := os.Stat(filepath.Join(o.diffDir, parent)) + if err := changeFn(ChangeKindModify, parent, pi, err); err != nil { + return err + } + changedDirs[parent] = struct{}{} + } + } + + return changeFn(kind, path, f, nil) + }) +} + +// doubleWalkDiff walks both directories to create a diff +func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b string) (err error) { + g, ctx := errgroup.WithContext(ctx) + + var ( + c1 = make(chan *currentPath) + c2 = make(chan *currentPath) + + f1, f2 *currentPath + rmdir string + ) + g.Go(func() error { + defer close(c1) + return pathWalk(ctx, a, c1) + }) + g.Go(func() error { + defer close(c2) + return pathWalk(ctx, b, c2) + }) + g.Go(func() error { + for c1 != nil || c2 != nil { + if f1 == nil && c1 != nil { + f1, err = nextPath(ctx, c1) + if err != nil { + return err + } + if f1 == nil { + c1 = nil + } + } + + if f2 == nil && c2 != nil { + f2, err = nextPath(ctx, c2) + if err != nil { + return err + } + if f2 == nil { + c2 = nil + } + } + if f1 == nil && f2 == nil { + continue + } + + var f os.FileInfo + k, p := pathChange(f1, f2) + switch k { + case ChangeKindAdd: + if rmdir != "" { + rmdir = "" + } + f = f2.f + f2 = nil + case ChangeKindDelete: + // Check if this file is already removed by being + // under of a removed directory + if rmdir != "" && strings.HasPrefix(f1.path, rmdir) { + f1 = nil + continue + } else if f1.f.IsDir() { + rmdir = f1.path + string(os.PathSeparator) + } else if rmdir != "" { + rmdir = "" + } + f1 = nil + case ChangeKindModify: + same, err := sameFile(f1, f2) + if err != nil { + return err + } + if f1.f.IsDir() && !f2.f.IsDir() { + rmdir = f1.path + string(os.PathSeparator) + } else if rmdir != "" { + rmdir = "" + } + f = f2.f + f1 = nil + f2 = nil + if same { + if !isLinked(f) { + continue + } + k = ChangeKindUnmodified + } + } + if err := changeFn(k, p, f, nil); err != nil { + return err + } + } + return nil + }) + + return g.Wait() +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/diff_unix.go b/components/engine/vendor/github.com/containerd/continuity/fs/diff_unix.go new file mode 100644 index 0000000000..3751814443 --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/diff_unix.go @@ -0,0 +1,58 @@ +// +build !windows + +package fs + +import ( + "bytes" + "os" + "syscall" + + "github.com/containerd/continuity/sysx" + "github.com/pkg/errors" +) + +// detectDirDiff returns diff dir options if a directory could +// be found in the mount info for upper which is the direct +// diff with the provided lower directory +func detectDirDiff(upper, lower string) *diffDirOptions { + // TODO: get mount options for upper + // TODO: detect AUFS + // TODO: detect overlay + return nil +} + +// compareSysStat returns whether the stats are equivalent, +// whether the files are considered the same file, and +// an error +func compareSysStat(s1, s2 interface{}) (bool, error) { + ls1, ok := s1.(*syscall.Stat_t) + if !ok { + return false, nil + } + ls2, ok := s2.(*syscall.Stat_t) + if !ok { + return false, nil + } + + return ls1.Mode == ls2.Mode && ls1.Uid == ls2.Uid && ls1.Gid == ls2.Gid && ls1.Rdev == ls2.Rdev, nil +} + +func compareCapabilities(p1, p2 string) (bool, error) { + c1, err := sysx.LGetxattr(p1, "security.capability") + if err != nil && err != sysx.ENODATA { + return false, errors.Wrapf(err, "failed to get xattr for %s", p1) + } + c2, err := sysx.LGetxattr(p2, "security.capability") + if err != nil && err != sysx.ENODATA { + return false, errors.Wrapf(err, "failed to get xattr for %s", p2) + } + return bytes.Equal(c1, c2), nil +} + +func isLinked(f os.FileInfo) bool { + s, ok := f.Sys().(*syscall.Stat_t) + if !ok { + return false + } + return !f.IsDir() && s.Nlink > 1 +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/diff_windows.go b/components/engine/vendor/github.com/containerd/continuity/fs/diff_windows.go new file mode 100644 index 0000000000..8eed36507e --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/diff_windows.go @@ -0,0 +1,32 @@ +package fs + +import ( + "os" + + "golang.org/x/sys/windows" +) + +func detectDirDiff(upper, lower string) *diffDirOptions { + return nil +} + +func compareSysStat(s1, s2 interface{}) (bool, error) { + f1, ok := s1.(windows.Win32FileAttributeData) + if !ok { + return false, nil + } + f2, ok := s2.(windows.Win32FileAttributeData) + if !ok { + return false, nil + } + return f1.FileAttributes == f2.FileAttributes, nil +} + +func compareCapabilities(p1, p2 string) (bool, error) { + // TODO: Use windows equivalent + return true, nil +} + +func isLinked(os.FileInfo) bool { + return false +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/dtype_linux.go b/components/engine/vendor/github.com/containerd/continuity/fs/dtype_linux.go new file mode 100644 index 0000000000..cc06573f1b --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/dtype_linux.go @@ -0,0 +1,87 @@ +// +build linux + +package fs + +import ( + "fmt" + "io/ioutil" + "os" + "syscall" + "unsafe" +) + +func locateDummyIfEmpty(path string) (string, error) { + children, err := ioutil.ReadDir(path) + if err != nil { + return "", err + } + if len(children) != 0 { + return "", nil + } + dummyFile, err := ioutil.TempFile(path, "fsutils-dummy") + if err != nil { + return "", err + } + name := dummyFile.Name() + err = dummyFile.Close() + return name, err +} + +// SupportsDType returns whether the filesystem mounted on path supports d_type +func SupportsDType(path string) (bool, error) { + // locate dummy so that we have at least one dirent + dummy, err := locateDummyIfEmpty(path) + if err != nil { + return false, err + } + if dummy != "" { + defer os.Remove(dummy) + } + + visited := 0 + supportsDType := true + fn := func(ent *syscall.Dirent) bool { + visited++ + if ent.Type == syscall.DT_UNKNOWN { + supportsDType = false + // stop iteration + return true + } + // continue iteration + return false + } + if err = iterateReadDir(path, fn); err != nil { + return false, err + } + if visited == 0 { + return false, fmt.Errorf("did not hit any dirent during iteration %s", path) + } + return supportsDType, nil +} + +func iterateReadDir(path string, fn func(*syscall.Dirent) bool) error { + d, err := os.Open(path) + if err != nil { + return err + } + defer d.Close() + fd := int(d.Fd()) + buf := make([]byte, 4096) + for { + nbytes, err := syscall.ReadDirent(fd, buf) + if err != nil { + return err + } + if nbytes == 0 { + break + } + for off := 0; off < nbytes; { + ent := (*syscall.Dirent)(unsafe.Pointer(&buf[off])) + if stop := fn(ent); stop { + return nil + } + off += int(ent.Reclen) + } + } + return nil +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/du.go b/components/engine/vendor/github.com/containerd/continuity/fs/du.go new file mode 100644 index 0000000000..26f5333154 --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/du.go @@ -0,0 +1,22 @@ +package fs + +import "context" + +// Usage of disk information +type Usage struct { + Inodes int64 + Size int64 +} + +// DiskUsage counts the number of inodes and disk usage for the resources under +// path. +func DiskUsage(roots ...string) (Usage, error) { + return diskUsage(roots...) +} + +// DiffUsage counts the numbers of inodes and disk usage in the +// diff between the 2 directories. The first path is intended +// as the base directory and the second as the changed directory. +func DiffUsage(ctx context.Context, a, b string) (Usage, error) { + return diffUsage(ctx, a, b) +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/du_unix.go b/components/engine/vendor/github.com/containerd/continuity/fs/du_unix.go new file mode 100644 index 0000000000..fe3426d278 --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/du_unix.go @@ -0,0 +1,88 @@ +// +build !windows + +package fs + +import ( + "context" + "os" + "path/filepath" + "syscall" +) + +type inode struct { + // TODO(stevvooe): Can probably reduce memory usage by not tracking + // device, but we can leave this right for now. + dev, ino uint64 +} + +func newInode(stat *syscall.Stat_t) inode { + return inode{ + // Dev is uint32 on darwin/bsd, uint64 on linux/solaris + dev: uint64(stat.Dev), // nolint: unconvert + // Ino is uint32 on bsd, uint64 on darwin/linux/solaris + ino: uint64(stat.Ino), // nolint: unconvert + } +} + +func diskUsage(roots ...string) (Usage, error) { + + var ( + size int64 + inodes = map[inode]struct{}{} // expensive! + ) + + for _, root := range roots { + if err := filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + + inoKey := newInode(fi.Sys().(*syscall.Stat_t)) + if _, ok := inodes[inoKey]; !ok { + inodes[inoKey] = struct{}{} + size += fi.Size() + } + + return nil + }); err != nil { + return Usage{}, err + } + } + + return Usage{ + Inodes: int64(len(inodes)), + Size: size, + }, nil +} + +func diffUsage(ctx context.Context, a, b string) (Usage, error) { + var ( + size int64 + inodes = map[inode]struct{}{} // expensive! + ) + + if err := Changes(ctx, a, b, func(kind ChangeKind, _ string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + + if kind == ChangeKindAdd || kind == ChangeKindModify { + inoKey := newInode(fi.Sys().(*syscall.Stat_t)) + if _, ok := inodes[inoKey]; !ok { + inodes[inoKey] = struct{}{} + size += fi.Size() + } + + return nil + + } + return nil + }); err != nil { + return Usage{}, err + } + + return Usage{ + Inodes: int64(len(inodes)), + Size: size, + }, nil +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/du_windows.go b/components/engine/vendor/github.com/containerd/continuity/fs/du_windows.go new file mode 100644 index 0000000000..3f852fc15e --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/du_windows.go @@ -0,0 +1,60 @@ +// +build windows + +package fs + +import ( + "context" + "os" + "path/filepath" +) + +func diskUsage(roots ...string) (Usage, error) { + var ( + size int64 + ) + + // TODO(stevvooe): Support inodes (or equivalent) for windows. + + for _, root := range roots { + if err := filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + + size += fi.Size() + return nil + }); err != nil { + return Usage{}, err + } + } + + return Usage{ + Size: size, + }, nil +} + +func diffUsage(ctx context.Context, a, b string) (Usage, error) { + var ( + size int64 + ) + + if err := Changes(ctx, a, b, func(kind ChangeKind, _ string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + + if kind == ChangeKindAdd || kind == ChangeKindModify { + size += fi.Size() + + return nil + + } + return nil + }); err != nil { + return Usage{}, err + } + + return Usage{ + Size: size, + }, nil +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/hardlink.go b/components/engine/vendor/github.com/containerd/continuity/fs/hardlink.go new file mode 100644 index 0000000000..38da93813c --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/hardlink.go @@ -0,0 +1,27 @@ +package fs + +import "os" + +// GetLinkInfo returns an identifier representing the node a hardlink is pointing +// to. If the file is not hard linked then 0 will be returned. +func GetLinkInfo(fi os.FileInfo) (uint64, bool) { + return getLinkInfo(fi) +} + +// getLinkSource returns a path for the given name and +// file info to its link source in the provided inode +// map. If the given file name is not in the map and +// has other links, it is added to the inode map +// to be a source for other link locations. +func getLinkSource(name string, fi os.FileInfo, inodes map[uint64]string) (string, error) { + inode, isHardlink := getLinkInfo(fi) + if !isHardlink { + return "", nil + } + + path, ok := inodes[inode] + if !ok { + inodes[inode] = name + } + return path, nil +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/hardlink_unix.go b/components/engine/vendor/github.com/containerd/continuity/fs/hardlink_unix.go new file mode 100644 index 0000000000..a6f99778de --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/hardlink_unix.go @@ -0,0 +1,18 @@ +// +build !windows + +package fs + +import ( + "os" + "syscall" +) + +func getLinkInfo(fi os.FileInfo) (uint64, bool) { + s, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return 0, false + } + + // Ino is uint32 on bsd, uint64 on darwin/linux/solaris + return uint64(s.Ino), !fi.IsDir() && s.Nlink > 1 // nolint: unconvert +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/hardlink_windows.go b/components/engine/vendor/github.com/containerd/continuity/fs/hardlink_windows.go new file mode 100644 index 0000000000..ad8845a7fb --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/hardlink_windows.go @@ -0,0 +1,7 @@ +package fs + +import "os" + +func getLinkInfo(fi os.FileInfo) (uint64, bool) { + return 0, false +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/path.go b/components/engine/vendor/github.com/containerd/continuity/fs/path.go new file mode 100644 index 0000000000..13fb826385 --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/path.go @@ -0,0 +1,276 @@ +package fs + +import ( + "bytes" + "context" + "io" + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" +) + +var ( + errTooManyLinks = errors.New("too many links") +) + +type currentPath struct { + path string + f os.FileInfo + fullPath string +} + +func pathChange(lower, upper *currentPath) (ChangeKind, string) { + if lower == nil { + if upper == nil { + panic("cannot compare nil paths") + } + return ChangeKindAdd, upper.path + } + if upper == nil { + return ChangeKindDelete, lower.path + } + // TODO: compare by directory + + switch i := strings.Compare(lower.path, upper.path); { + case i < 0: + // File in lower that is not in upper + return ChangeKindDelete, lower.path + case i > 0: + // File in upper that is not in lower + return ChangeKindAdd, upper.path + default: + return ChangeKindModify, upper.path + } +} + +func sameFile(f1, f2 *currentPath) (bool, error) { + if os.SameFile(f1.f, f2.f) { + return true, nil + } + + equalStat, err := compareSysStat(f1.f.Sys(), f2.f.Sys()) + if err != nil || !equalStat { + return equalStat, err + } + + if eq, err := compareCapabilities(f1.fullPath, f2.fullPath); err != nil || !eq { + return eq, err + } + + // If not a directory also check size, modtime, and content + if !f1.f.IsDir() { + if f1.f.Size() != f2.f.Size() { + return false, nil + } + t1 := f1.f.ModTime() + t2 := f2.f.ModTime() + + if t1.Unix() != t2.Unix() { + return false, nil + } + + // If the timestamp may have been truncated in both of the + // files, check content of file to determine difference + if t1.Nanosecond() == 0 && t2.Nanosecond() == 0 { + var eq bool + if (f1.f.Mode() & os.ModeSymlink) == os.ModeSymlink { + eq, err = compareSymlinkTarget(f1.fullPath, f2.fullPath) + } else if f1.f.Size() > 0 { + eq, err = compareFileContent(f1.fullPath, f2.fullPath) + } + if err != nil || !eq { + return eq, err + } + } else if t1.Nanosecond() != t2.Nanosecond() { + return false, nil + } + } + + return true, nil +} + +func compareSymlinkTarget(p1, p2 string) (bool, error) { + t1, err := os.Readlink(p1) + if err != nil { + return false, err + } + t2, err := os.Readlink(p2) + if err != nil { + return false, err + } + return t1 == t2, nil +} + +const compareChuckSize = 32 * 1024 + +// compareFileContent compares the content of 2 same sized files +// by comparing each byte. +func compareFileContent(p1, p2 string) (bool, error) { + f1, err := os.Open(p1) + if err != nil { + return false, err + } + defer f1.Close() + f2, err := os.Open(p2) + if err != nil { + return false, err + } + defer f2.Close() + + b1 := make([]byte, compareChuckSize) + b2 := make([]byte, compareChuckSize) + for { + n1, err1 := f1.Read(b1) + if err1 != nil && err1 != io.EOF { + return false, err1 + } + n2, err2 := f2.Read(b2) + if err2 != nil && err2 != io.EOF { + return false, err2 + } + if n1 != n2 || !bytes.Equal(b1[:n1], b2[:n2]) { + return false, nil + } + if err1 == io.EOF && err2 == io.EOF { + return true, nil + } + } +} + +func pathWalk(ctx context.Context, root string, pathC chan<- *currentPath) error { + return filepath.Walk(root, func(path string, f os.FileInfo, err error) error { + if err != nil { + return err + } + + // Rebase path + path, err = filepath.Rel(root, path) + if err != nil { + return err + } + + path = filepath.Join(string(os.PathSeparator), path) + + // Skip root + if path == string(os.PathSeparator) { + return nil + } + + p := ¤tPath{ + path: path, + f: f, + fullPath: filepath.Join(root, path), + } + + select { + case <-ctx.Done(): + return ctx.Err() + case pathC <- p: + return nil + } + }) +} + +func nextPath(ctx context.Context, pathC <-chan *currentPath) (*currentPath, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case p := <-pathC: + return p, nil + } +} + +// RootPath joins a path with a root, evaluating and bounding any +// symlink to the root directory. +func RootPath(root, path string) (string, error) { + if path == "" { + return root, nil + } + var linksWalked int // to protect against cycles + for { + i := linksWalked + newpath, err := walkLinks(root, path, &linksWalked) + if err != nil { + return "", err + } + path = newpath + if i == linksWalked { + newpath = filepath.Join("/", newpath) + if path == newpath { + return filepath.Join(root, newpath), nil + } + path = newpath + } + } +} + +func walkLink(root, path string, linksWalked *int) (newpath string, islink bool, err error) { + if *linksWalked > 255 { + return "", false, errTooManyLinks + } + + path = filepath.Join("/", path) + if path == "/" { + return path, false, nil + } + realPath := filepath.Join(root, path) + + fi, err := os.Lstat(realPath) + if err != nil { + // If path does not yet exist, treat as non-symlink + if os.IsNotExist(err) { + return path, false, nil + } + return "", false, err + } + if fi.Mode()&os.ModeSymlink == 0 { + return path, false, nil + } + newpath, err = os.Readlink(realPath) + if err != nil { + return "", false, err + } + if filepath.IsAbs(newpath) && strings.HasPrefix(newpath, root) { + newpath = newpath[:len(root)] + if !strings.HasPrefix(newpath, "/") { + newpath = "/" + newpath + } + } + *linksWalked++ + return newpath, true, nil +} + +func walkLinks(root, path string, linksWalked *int) (string, error) { + switch dir, file := filepath.Split(path); { + case dir == "": + newpath, _, err := walkLink(root, file, linksWalked) + return newpath, err + case file == "": + if os.IsPathSeparator(dir[len(dir)-1]) { + if dir == "/" { + return dir, nil + } + return walkLinks(root, dir[:len(dir)-1], linksWalked) + } + newpath, _, err := walkLink(root, dir, linksWalked) + return newpath, err + default: + newdir, err := walkLinks(root, dir, linksWalked) + if err != nil { + return "", err + } + newpath, islink, err := walkLink(root, filepath.Join(newdir, file), linksWalked) + if err != nil { + return "", err + } + if !islink { + return newpath, nil + } + if filepath.IsAbs(newpath) { + return newpath, nil + } + return filepath.Join(newdir, newpath), nil + } +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/stat_bsd.go b/components/engine/vendor/github.com/containerd/continuity/fs/stat_bsd.go new file mode 100644 index 0000000000..a1b776fdf5 --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/stat_bsd.go @@ -0,0 +1,28 @@ +// +build darwin freebsd + +package fs + +import ( + "syscall" + "time" +) + +// StatAtime returns the access time from a stat struct +func StatAtime(st *syscall.Stat_t) syscall.Timespec { + return st.Atimespec +} + +// StatCtime returns the created time from a stat struct +func StatCtime(st *syscall.Stat_t) syscall.Timespec { + return st.Ctimespec +} + +// StatMtime returns the modified time from a stat struct +func StatMtime(st *syscall.Stat_t) syscall.Timespec { + return st.Mtimespec +} + +// StatATimeAsTime returns the access time as a time.Time +func StatATimeAsTime(st *syscall.Stat_t) time.Time { + return time.Unix(int64(st.Atimespec.Sec), int64(st.Atimespec.Nsec)) // nolint: unconvert +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/stat_linux.go b/components/engine/vendor/github.com/containerd/continuity/fs/stat_linux.go new file mode 100644 index 0000000000..25bc652581 --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/stat_linux.go @@ -0,0 +1,26 @@ +package fs + +import ( + "syscall" + "time" +) + +// StatAtime returns the Atim +func StatAtime(st *syscall.Stat_t) syscall.Timespec { + return st.Atim +} + +// StatCtime returns the Ctim +func StatCtime(st *syscall.Stat_t) syscall.Timespec { + return st.Ctim +} + +// StatMtime returns the Mtim +func StatMtime(st *syscall.Stat_t) syscall.Timespec { + return st.Mtim +} + +// StatATimeAsTime returns st.Atim as a time.Time +func StatATimeAsTime(st *syscall.Stat_t) time.Time { + return time.Unix(st.Atim.Sec, st.Atim.Nsec) +} diff --git a/components/engine/vendor/github.com/containerd/continuity/fs/time.go b/components/engine/vendor/github.com/containerd/continuity/fs/time.go new file mode 100644 index 0000000000..c336f4d881 --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/fs/time.go @@ -0,0 +1,13 @@ +package fs + +import "time" + +// Gnu tar and the go tar writer don't have sub-second mtime +// precision, which is problematic when we apply changes via tar +// files, we handle this by comparing for exact times, *or* same +// second count and either a or b having exactly 0 nanoseconds +func sameFsTime(a, b time.Time) bool { + return a == b || + (a.Unix() == b.Unix() && + (a.Nanosecond() == 0 || b.Nanosecond() == 0)) +} diff --git a/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux.go b/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux.go deleted file mode 100644 index 4d8581284a..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux.go +++ /dev/null @@ -1,11 +0,0 @@ -package sysx - -// These functions will be generated by generate.sh -// $ GOOS=linux GOARCH=386 ./generate.sh copy -// $ GOOS=linux GOARCH=amd64 ./generate.sh copy -// $ GOOS=linux GOARCH=arm ./generate.sh copy -// $ GOOS=linux GOARCH=arm64 ./generate.sh copy -// $ GOOS=linux GOARCH=ppc64le ./generate.sh copy -// $ GOOS=linux GOARCH=s390x ./generate.sh copy - -//sys CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error) diff --git a/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_386.go b/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_386.go deleted file mode 100644 index c1368c5723..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_386.go +++ /dev/null @@ -1,20 +0,0 @@ -// mksyscall.pl -l32 copy_linux.go -// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT - -package sysx - -import ( - "syscall" - "unsafe" -) - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - -func CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error) { - r0, _, e1 := syscall.Syscall6(SYS_COPY_FILE_RANGE, uintptr(fdin), uintptr(unsafe.Pointer(offin)), uintptr(fdout), uintptr(unsafe.Pointer(offout)), uintptr(len), uintptr(flags)) - n = int(r0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} diff --git a/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_amd64.go b/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_amd64.go deleted file mode 100644 index 9941b01f09..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_amd64.go +++ /dev/null @@ -1,20 +0,0 @@ -// mksyscall.pl copy_linux.go -// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT - -package sysx - -import ( - "syscall" - "unsafe" -) - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - -func CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error) { - r0, _, e1 := syscall.Syscall6(SYS_COPY_FILE_RANGE, uintptr(fdin), uintptr(unsafe.Pointer(offin)), uintptr(fdout), uintptr(unsafe.Pointer(offout)), uintptr(len), uintptr(flags)) - n = int(r0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} diff --git a/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_arm.go b/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_arm.go deleted file mode 100644 index c1368c5723..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_arm.go +++ /dev/null @@ -1,20 +0,0 @@ -// mksyscall.pl -l32 copy_linux.go -// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT - -package sysx - -import ( - "syscall" - "unsafe" -) - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - -func CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error) { - r0, _, e1 := syscall.Syscall6(SYS_COPY_FILE_RANGE, uintptr(fdin), uintptr(unsafe.Pointer(offin)), uintptr(fdout), uintptr(unsafe.Pointer(offout)), uintptr(len), uintptr(flags)) - n = int(r0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} diff --git a/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_arm64.go b/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_arm64.go deleted file mode 100644 index 9941b01f09..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_arm64.go +++ /dev/null @@ -1,20 +0,0 @@ -// mksyscall.pl copy_linux.go -// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT - -package sysx - -import ( - "syscall" - "unsafe" -) - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - -func CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error) { - r0, _, e1 := syscall.Syscall6(SYS_COPY_FILE_RANGE, uintptr(fdin), uintptr(unsafe.Pointer(offin)), uintptr(fdout), uintptr(unsafe.Pointer(offout)), uintptr(len), uintptr(flags)) - n = int(r0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} diff --git a/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_ppc64le.go b/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_ppc64le.go deleted file mode 100644 index 9941b01f09..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_ppc64le.go +++ /dev/null @@ -1,20 +0,0 @@ -// mksyscall.pl copy_linux.go -// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT - -package sysx - -import ( - "syscall" - "unsafe" -) - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - -func CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error) { - r0, _, e1 := syscall.Syscall6(SYS_COPY_FILE_RANGE, uintptr(fdin), uintptr(unsafe.Pointer(offin)), uintptr(fdout), uintptr(unsafe.Pointer(offout)), uintptr(len), uintptr(flags)) - n = int(r0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} diff --git a/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_s390x.go b/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_s390x.go deleted file mode 100644 index 9941b01f09..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/sysx/copy_linux_s390x.go +++ /dev/null @@ -1,20 +0,0 @@ -// mksyscall.pl copy_linux.go -// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT - -package sysx - -import ( - "syscall" - "unsafe" -) - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - -func CopyFileRange(fdin uintptr, offin *int64, fdout uintptr, offout *int64, len int, flags int) (n int, err error) { - r0, _, e1 := syscall.Syscall6(SYS_COPY_FILE_RANGE, uintptr(fdin), uintptr(unsafe.Pointer(offin)), uintptr(fdout), uintptr(unsafe.Pointer(offout)), uintptr(len), uintptr(flags)) - n = int(r0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} diff --git a/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_386.go b/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_386.go deleted file mode 100644 index 0063f8a913..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_386.go +++ /dev/null @@ -1,7 +0,0 @@ -package sysx - -const ( - // SYS_COPYFILERANGE defined in Kernel 4.5+ - // Number defined in /usr/include/asm/unistd_32.h - SYS_COPY_FILE_RANGE = 377 -) diff --git a/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_amd64.go b/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_amd64.go deleted file mode 100644 index 4170540c5d..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_amd64.go +++ /dev/null @@ -1,7 +0,0 @@ -package sysx - -const ( - // SYS_COPYFILERANGE defined in Kernel 4.5+ - // Number defined in /usr/include/asm/unistd_64.h - SYS_COPY_FILE_RANGE = 326 -) diff --git a/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_arm.go b/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_arm.go deleted file mode 100644 index a05dcbb5ef..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_arm.go +++ /dev/null @@ -1,7 +0,0 @@ -package sysx - -const ( - // SYS_COPY_FILE_RANGE defined in Kernel 4.5+ - // Number defined in /usr/include/arm-linux-gnueabihf/asm/unistd.h - SYS_COPY_FILE_RANGE = 391 -) diff --git a/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_arm64.go b/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_arm64.go deleted file mode 100644 index da31bbd908..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_arm64.go +++ /dev/null @@ -1,7 +0,0 @@ -package sysx - -const ( - // SYS_COPY_FILE_RANGE defined in Kernel 4.5+ - // Number defined in /usr/include/asm-generic/unistd.h - SYS_COPY_FILE_RANGE = 285 -) diff --git a/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_ppc64le.go b/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_ppc64le.go deleted file mode 100644 index 5dea25a3c4..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_ppc64le.go +++ /dev/null @@ -1,7 +0,0 @@ -package sysx - -const ( - // SYS_COPYFILERANGE defined in Kernel 4.5+ - // Number defined in /usr/include/asm/unistd_64.h - SYS_COPY_FILE_RANGE = 379 -) diff --git a/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_s390x.go b/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_s390x.go deleted file mode 100644 index 8a6f2a7ec0..0000000000 --- a/components/engine/vendor/github.com/containerd/continuity/sysx/sysnum_linux_s390x.go +++ /dev/null @@ -1,7 +0,0 @@ -package sysx - -const ( - // SYS_COPYFILERANGE defined in Kernel 4.5+ - // Number defined in /usr/include/asm/unistd_64.h - SYS_COPY_FILE_RANGE = 375 -) diff --git a/components/engine/vendor/github.com/containerd/continuity/vendor.conf b/components/engine/vendor/github.com/containerd/continuity/vendor.conf new file mode 100644 index 0000000000..7c80deec58 --- /dev/null +++ b/components/engine/vendor/github.com/containerd/continuity/vendor.conf @@ -0,0 +1,13 @@ +bazil.org/fuse 371fbbdaa8987b715bdd21d6adc4c9b20155f748 +github.com/dustin/go-humanize bb3d318650d48840a39aa21a027c6630e198e626 +github.com/golang/protobuf 1e59b77b52bf8e4b449a57e6f79f21226d571845 +github.com/inconshreveable/mousetrap 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75 +github.com/opencontainers/go-digest 279bed98673dd5bef374d3b6e4b09e2af76183bf +github.com/pkg/errors f15c970de5b76fac0b59abb32d62c17cc7bed265 +github.com/sirupsen/logrus 89742aefa4b206dcf400792f3bd35b542998eb3b +github.com/spf13/cobra 2da4a54c5ceefcee7ca5dd0eea1e18a3b6366489 +github.com/spf13/pflag 4c012f6dcd9546820e378d0bdda4d8fc772cdfea +golang.org/x/crypto 9f005a07e0d31d45e6656d241bb5c0f2efd4bc94 +golang.org/x/net a337091b0525af65de94df2eb7e98bd9962dcbe2 +golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c +golang.org/x/sys 665f6529cca930e27b831a0d1dafffbe1c172924 From cd33455198e9b4d06b97e8b8c9ded8c4ffade5df Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Tue, 13 Feb 2018 11:14:23 +0100 Subject: [PATCH 07/19] =?UTF-8?q?Clean=20some=20maintainers=20=F0=9F=91=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit albers, aluzzardi, ehazlett, icecrime, lk4d4, mavenugo 🤗 Signed-off-by: Vincent Demeester Upstream-commit: de664ac749ed25271e6b498aa5b7735dd2f1e026 Component: engine --- components/engine/MAINTAINERS | 53 +++++++++++++++++++++++++++-------- components/engine/poule.yml | 3 -- 2 files changed, 41 insertions(+), 15 deletions(-) diff --git a/components/engine/MAINTAINERS b/components/engine/MAINTAINERS index 4c831d7832..e9a5566caf 100644 --- a/components/engine/MAINTAINERS +++ b/components/engine/MAINTAINERS @@ -23,29 +23,19 @@ # a subsystem, they are responsible for doing so and holding the # subsystem maintainers accountable. If ownership is unclear, they are the de facto owners. - # For each release (including minor releases), a "release captain" is assigned from the - # pool of core maintainers. Rotation is encouraged across all maintainers, to ensure - # the release process is clear and up-to-date. - people = [ "aaronlehmann", "akihirosuda", - "albers", - "aluzzardi", "anusha", "coolljt0725", "cpuguy83", "crosbymichael", "dnephin", "duglin", - "ehazlett", "estesp", - "icecrime", "jhowardmsft", "johnstep", "justincormack", - "lk4d4", - "mavenugo", "mhbauer", "mlaventure", "runcom", @@ -103,6 +93,16 @@ # Thank you! people = [ + # Harald Albers is the mastermind behind the bash completion scripts for the + # Docker CLI. The completion scripts moved to the Docker CLI repository, so + # you can now find him perform his magic in the https://github.com/docker/cli repository. + "albers", + + # Andrea Luzzardi started contributing to the Docker codebase in the "dotCloud" + # era, even before it was called "Docker". He is one of the architects of both + # Swarm and SwarmKit, and its integration into the Docker engine. + "aluzzardi", + # David Calavera contributed many features to Docker, such as an improved # event system, dynamic configuration reloading, volume plugins, fancy # new templating options, and an external client credential store. As a @@ -120,6 +120,24 @@ # still stumble into him in our issue tracker, or on IRC. "erikh", + # Evan Hazlett is the creator of of the Shipyard and Interlock open source projects, + # and the author of "Orca", which became the foundation of Docker Universal Control + # Plane (UCP). As a maintainer, Evan helped integrating SwarmKit (secrets, tasks) + # into the Docker engine. + "ehazlett", + + # Arnaud Porterie (AKA "icecrime") was in charge of maintaining the maintainers. + # As a maintainer, he made life easier for contributors to the Docker open-source + # projects, bringing order in the chaos by designing a triage- and review workflow + # using labels (see https://icecrime.net/technology/a-structured-approach-to-labeling/), + # and automating the hell out of things with his buddies GordonTheTurtle and Poule + # (a chicken!). + # + # A lesser-known fact is that he created the first commit in the libnetwork repository + # even though he didn't know anything about it. Some say, he's now selling stuff on + # the internet ;-) + "icecrime", + # After a false start with his first PR being rejected, James Turnbull became a frequent # contributor to the documentation, and became a docs maintainer on December 5, 2013. As # a maintainer, James lifted the docs to a higher standard, and introduced the community @@ -139,13 +157,24 @@ # containers a lot more secure). Besides being a maintainer, she # set up the CI infrastructure for the project, giving everyone # something to shout at if a PR failed ("noooo Janky!"). - # Jess is currently working on the DCOS security team at Mesosphere, - # and contributing to various open source projects. # Be sure you don't miss her talks at a conference near you (a must-see), # read her blog at https://blog.jessfraz.com (a must-read), and # check out her open source projects on GitHub https://github.com/jessfraz (a must-try). "jessfraz", + # Alexander Morozov contributed many features to Docker, worked on the premise of + # what later became containerd (and worked on that too), and made a "stupid" Go + # vendor tool specificaly for docker/docker needs: vndr (https://github.com/LK4D4/vndr). + # Not many know that Alexander is a master negotiator, being able to change course + # of action with a single "Nope, we're not gonna do that". + "lk4d4", + + # Madhu Venugopal was part of the SocketPlane team that joined Docker. + # As a maintainer, he was working with Jana for the Container Network + # Model (CNM) implemented through libnetwork, and the "routing mesh" powering + # Swarm mode networking. + "mavenugo", + # As a docs maintainer, Mary Anthony contributed greatly to the Docker # docs. She wrote the Docker Contributor Guide and Getting Started # Guides. She helped create a doc build system independent of diff --git a/components/engine/poule.yml b/components/engine/poule.yml index 2abf0df7f0..7afb79eada 100644 --- a/components/engine/poule.yml +++ b/components/engine/poule.yml @@ -107,16 +107,13 @@ users: [ "aaronlehmann", "akihirosuda", - "aluzzardi", "coolljt0725", "cpuguy83", "crosbymichael", "dnephin", "duglin", - "ehazlett", "johnstep", "justincormack", - "lk4d4", "mhbauer", "mlaventure", "runcom", From 7ebcfdf8bd9eb28cd6c79f991d7ca7e2c1c80966 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 12 Feb 2018 23:08:25 +0000 Subject: [PATCH 08/19] Update api tests to use container.Run/Create in helper package This fix is a sync up with 36266 so that relevant api tests use the newly added container.Run/Create in helper package Signed-off-by: Yong Tang Upstream-commit: 9fcd2a05106af98e6ffd6efb9f124d64426956e4 Component: engine --- .../engine/integration/container/exec_test.go | 23 ++----- .../integration/container/health_test.go | 36 ++++------- .../integration/container/links_linux_test.go | 23 ++----- .../engine/integration/container/nat_test.go | 61 ++++++------------- .../integration/container/stats_test.go | 21 ++----- .../plugin/authz/authz_plugin_test.go | 39 +++++------- .../plugin/authz/authz_plugin_v2_test.go | 13 ++-- 7 files changed, 68 insertions(+), 148 deletions(-) diff --git a/components/engine/integration/container/exec_test.go b/components/engine/integration/container/exec_test.go index a14284806d..9c27524696 100644 --- a/components/engine/integration/container/exec_test.go +++ b/components/engine/integration/container/exec_test.go @@ -6,9 +6,8 @@ import ( "testing" "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/api/types/network" "github.com/docker/docker/api/types/strslice" + "github.com/docker/docker/integration/internal/container" "github.com/docker/docker/integration/internal/request" "github.com/stretchr/testify/require" ) @@ -18,22 +17,12 @@ func TestExec(t *testing.T) { ctx := context.Background() client := request.NewAPIClient(t) - container, err := client.ContainerCreate(ctx, - &container.Config{ - Image: "busybox", - Tty: true, - WorkingDir: "/root", - Cmd: strslice.StrSlice([]string{"top"}), - }, - &container.HostConfig{}, - &network.NetworkingConfig{}, - "foo", - ) - require.NoError(t, err) - err = client.ContainerStart(ctx, container.ID, types.ContainerStartOptions{}) - require.NoError(t, err) + cID := container.Run(t, ctx, client, func(c *container.TestContainerConfig) { + c.Config.Tty = true + c.Config.WorkingDir = "/root" + }) - id, err := client.ContainerExecCreate(ctx, container.ID, + id, err := client.ContainerExecCreate(ctx, cID, types.ExecConfig{ WorkingDir: "/tmp", Env: strslice.StrSlice([]string{"FOO=BAR"}), diff --git a/components/engine/integration/container/health_test.go b/components/engine/integration/container/health_test.go index a5c62edb57..ec9604c901 100644 --- a/components/engine/integration/container/health_test.go +++ b/components/engine/integration/container/health_test.go @@ -6,13 +6,11 @@ import ( "time" "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/api/types/network" - "github.com/docker/docker/api/types/strslice" + containertypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/client" + "github.com/docker/docker/integration/internal/container" "github.com/docker/docker/integration/internal/request" "github.com/gotestyourself/gotestyourself/poll" - "github.com/stretchr/testify/require" ) // TestHealthCheckWorkdir verifies that health-checks inherit the containers' @@ -22,27 +20,17 @@ func TestHealthCheckWorkdir(t *testing.T) { ctx := context.Background() client := request.NewAPIClient(t) - c, err := client.ContainerCreate(ctx, - &container.Config{ - Image: "busybox", - Tty: true, - WorkingDir: "/foo", - Cmd: strslice.StrSlice([]string{"top"}), - Healthcheck: &container.HealthConfig{ - Test: []string{"CMD-SHELL", "if [ \"$PWD\" = \"/foo\" ]; then exit 0; else exit 1; fi;"}, - Interval: 50 * time.Millisecond, - Retries: 3, - }, - }, - &container.HostConfig{}, - &network.NetworkingConfig{}, - "healthtest", - ) - require.NoError(t, err) - err = client.ContainerStart(ctx, c.ID, types.ContainerStartOptions{}) - require.NoError(t, err) + cID := container.Run(t, ctx, client, func(c *container.TestContainerConfig) { + c.Config.Tty = true + c.Config.WorkingDir = "/foo" + c.Config.Healthcheck = &containertypes.HealthConfig{ + Test: []string{"CMD-SHELL", "if [ \"$PWD\" = \"/foo\" ]; then exit 0; else exit 1; fi;"}, + Interval: 50 * time.Millisecond, + Retries: 3, + } + }) - poll.WaitOn(t, pollForHealthStatus(ctx, client, c.ID, types.Healthy), poll.WithDelay(100*time.Millisecond)) + poll.WaitOn(t, pollForHealthStatus(ctx, client, cID, types.Healthy), poll.WithDelay(100*time.Millisecond)) } func pollForHealthStatus(ctx context.Context, client client.APIClient, containerID string, healthStatus string) func(log poll.LogT) poll.Result { diff --git a/components/engine/integration/container/links_linux_test.go b/components/engine/integration/container/links_linux_test.go index b1dc654c20..550b02d874 100644 --- a/components/engine/integration/container/links_linux_test.go +++ b/components/engine/integration/container/links_linux_test.go @@ -9,7 +9,7 @@ import ( "time" "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" + "github.com/docker/docker/integration/internal/container" "github.com/docker/docker/integration/internal/request" "github.com/docker/docker/pkg/stdcopy" "github.com/gotestyourself/gotestyourself/poll" @@ -28,24 +28,13 @@ func TestLinksEtcHostsContentMatch(t *testing.T) { client := request.NewAPIClient(t) ctx := context.Background() - c, err := client.ContainerCreate(ctx, - &container.Config{ - Image: "busybox", - Cmd: []string{"cat", "/etc/hosts"}, - }, - &container.HostConfig{ - NetworkMode: "host", - }, - nil, - "") - require.NoError(t, err) + cID := container.Run(t, ctx, client, container.WithCmd("cat", "/etc/hosts"), func(c *container.TestContainerConfig) { + c.HostConfig.NetworkMode = "host" + }) - err = client.ContainerStart(ctx, c.ID, types.ContainerStartOptions{}) - require.NoError(t, err) + poll.WaitOn(t, containerIsStopped(ctx, client, cID), poll.WithDelay(100*time.Millisecond)) - poll.WaitOn(t, containerIsStopped(ctx, client, c.ID), poll.WithDelay(100*time.Millisecond)) - - body, err := client.ContainerLogs(ctx, c.ID, types.ContainerLogsOptions{ + body, err := client.ContainerLogs(ctx, cID, types.ContainerLogsOptions{ ShowStdout: true, }) require.NoError(t, err) diff --git a/components/engine/integration/container/nat_test.go b/components/engine/integration/container/nat_test.go index 0732e2d852..1b41ae9608 100644 --- a/components/engine/integration/container/nat_test.go +++ b/components/engine/integration/container/nat_test.go @@ -12,8 +12,7 @@ import ( "time" "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/api/types/network" + "github.com/docker/docker/integration/internal/container" "github.com/docker/docker/integration/internal/request" "github.com/docker/go-connections/nat" "github.com/gotestyourself/gotestyourself/poll" @@ -67,25 +66,15 @@ func TestNetworkLoopbackNat(t *testing.T) { client := request.NewAPIClient(t) ctx := context.Background() - c, err := client.ContainerCreate(ctx, - &container.Config{ - Image: "busybox", - Cmd: []string{"sh", "-c", fmt.Sprintf("stty raw && nc -w 5 %s 8080", endpoint.String())}, - Tty: true, - }, - &container.HostConfig{ - NetworkMode: "container:server", - }, - nil, - "") - require.NoError(t, err) - err = client.ContainerStart(ctx, c.ID, types.ContainerStartOptions{}) - require.NoError(t, err) + cID := container.Run(t, ctx, client, container.WithCmd("sh", "-c", fmt.Sprintf("stty raw && nc -w 5 %s 8080", endpoint.String())), func(c *container.TestContainerConfig) { + c.Config.Tty = true + c.HostConfig.NetworkMode = "container:server" + }) - poll.WaitOn(t, containerIsStopped(ctx, client, c.ID), poll.WithDelay(100*time.Millisecond)) + poll.WaitOn(t, containerIsStopped(ctx, client, cID), poll.WithDelay(100*time.Millisecond)) - body, err := client.ContainerLogs(ctx, c.ID, types.ContainerLogsOptions{ + body, err := client.ContainerLogs(ctx, cID, types.ContainerLogsOptions{ ShowStdout: true, }) require.NoError(t, err) @@ -102,34 +91,22 @@ func startServerContainer(t *testing.T, msg string, port int) string { client := request.NewAPIClient(t) ctx := context.Background() - c, err := client.ContainerCreate(ctx, - &container.Config{ - Image: "busybox", - Cmd: []string{"sh", "-c", fmt.Sprintf("echo %q | nc -lp %d", msg, port)}, - ExposedPorts: map[nat.Port]struct{}{ - nat.Port(fmt.Sprintf("%d/tcp", port)): {}, - }, - }, - &container.HostConfig{ - PortBindings: nat.PortMap{ - nat.Port(fmt.Sprintf("%d/tcp", port)): []nat.PortBinding{ - { - HostPort: fmt.Sprintf("%d", port), - }, + cID := container.Run(t, ctx, client, container.WithCmd("sh", "-c", fmt.Sprintf("echo %q | nc -lp %d", msg, port)), func(c *container.TestContainerConfig) { + c.Config.ExposedPorts = map[nat.Port]struct{}{ + nat.Port(fmt.Sprintf("%d/tcp", port)): {}, + } + c.HostConfig.PortBindings = nat.PortMap{ + nat.Port(fmt.Sprintf("%d/tcp", port)): []nat.PortBinding{ + { + HostPort: fmt.Sprintf("%d", port), }, }, - }, - &network.NetworkingConfig{}, - "server", - ) - require.NoError(t, err) + } + }) - err = client.ContainerStart(ctx, c.ID, types.ContainerStartOptions{}) - require.NoError(t, err) + poll.WaitOn(t, containerIsInState(ctx, client, cID, "running"), poll.WithDelay(100*time.Millisecond)) - poll.WaitOn(t, containerIsInState(ctx, client, c.ID, "running"), poll.WithDelay(100*time.Millisecond)) - - return c.ID + return cID } func getExternalAddress(t *testing.T) net.IP { diff --git a/components/engine/integration/container/stats_test.go b/components/engine/integration/container/stats_test.go index 577d446d15..fdf85f44f6 100644 --- a/components/engine/integration/container/stats_test.go +++ b/components/engine/integration/container/stats_test.go @@ -8,8 +8,7 @@ import ( "time" "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/api/types/network" + "github.com/docker/docker/integration/internal/container" "github.com/docker/docker/integration/internal/request" "github.com/gotestyourself/gotestyourself/poll" "github.com/gotestyourself/gotestyourself/skip" @@ -27,23 +26,11 @@ func TestStats(t *testing.T) { info, err := client.Info(ctx) require.NoError(t, err) - c, err := client.ContainerCreate(ctx, - &container.Config{ - Cmd: []string{"top"}, - Image: "busybox", - }, - &container.HostConfig{}, - &network.NetworkingConfig{}, - "", - ) - require.NoError(t, err) + cID := container.Run(t, ctx, client) - err = client.ContainerStart(ctx, c.ID, types.ContainerStartOptions{}) - require.NoError(t, err) + poll.WaitOn(t, containerIsInState(ctx, client, cID, "running"), poll.WithDelay(100*time.Millisecond)) - poll.WaitOn(t, containerIsInState(ctx, client, c.ID, "running"), poll.WithDelay(100*time.Millisecond)) - - resp, err := client.ContainerStats(context.Background(), c.ID, false) + resp, err := client.ContainerStats(ctx, cID, false) require.NoError(t, err) defer resp.Body.Close() diff --git a/components/engine/integration/plugin/authz/authz_plugin_test.go b/components/engine/integration/plugin/authz/authz_plugin_test.go index befebe4080..94f7b896a7 100644 --- a/components/engine/integration/plugin/authz/authz_plugin_test.go +++ b/components/engine/integration/plugin/authz/authz_plugin_test.go @@ -19,10 +19,9 @@ import ( "time" "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" eventtypes "github.com/docker/docker/api/types/events" - networktypes "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" + "github.com/docker/docker/integration/internal/container" "github.com/docker/docker/integration/internal/request" "github.com/docker/docker/internal/test/environment" "github.com/docker/docker/pkg/authorization" @@ -91,17 +90,15 @@ func TestAuthZPluginAllowRequest(t *testing.T) { client, err := d.NewClient() require.Nil(t, err) - // Ensure command successful - createResponse, err := client.ContainerCreate(context.Background(), &container.Config{Cmd: []string{"top"}, Image: "busybox"}, &container.HostConfig{}, &networktypes.NetworkingConfig{}, "") - require.Nil(t, err) + ctx := context.Background() - err = client.ContainerStart(context.Background(), createResponse.ID, types.ContainerStartOptions{}) - require.Nil(t, err) + // Ensure command successful + cID := container.Run(t, ctx, client) assertURIRecorded(t, ctrl.requestsURIs, "/containers/create") - assertURIRecorded(t, ctrl.requestsURIs, fmt.Sprintf("/containers/%s/start", createResponse.ID)) + assertURIRecorded(t, ctrl.requestsURIs, fmt.Sprintf("/containers/%s/start", cID)) - _, err = client.ServerVersion(context.Background()) + _, err = client.ServerVersion(ctx) require.Nil(t, err) require.Equal(t, 1, ctrl.versionReqCount) require.Equal(t, 1, ctrl.versionResCount) @@ -213,19 +210,17 @@ func TestAuthZPluginAllowEventStream(t *testing.T) { client, err := d.NewClient() require.Nil(t, err) + ctx := context.Background() + startTime := strconv.FormatInt(systemTime(t, client, testEnv).Unix(), 10) events, errs, cancel := systemEventsSince(client, startTime) defer cancel() // Create a container and wait for the creation events - createResponse, err := client.ContainerCreate(context.Background(), &container.Config{Cmd: []string{"top"}, Image: "busybox"}, &container.HostConfig{}, &networktypes.NetworkingConfig{}, "") - require.Nil(t, err) - - err = client.ContainerStart(context.Background(), createResponse.ID, types.ContainerStartOptions{}) - require.Nil(t, err) + cID := container.Run(t, ctx, client) for i := 0; i < 100; i++ { - c, err := client.ContainerInspect(context.Background(), createResponse.ID) + c, err := client.ContainerInspect(ctx, cID) require.Nil(t, err) if c.State.Running { break @@ -241,7 +236,7 @@ func TestAuthZPluginAllowEventStream(t *testing.T) { for !created && !started { select { case event := <-events: - if event.Type == eventtypes.ContainerEventType && event.Actor.ID == createResponse.ID { + if event.Type == eventtypes.ContainerEventType && event.Actor.ID == cID { if event.Action == "create" { created = true } @@ -264,7 +259,7 @@ func TestAuthZPluginAllowEventStream(t *testing.T) { // authorization plugin assertURIRecorded(t, ctrl.requestsURIs, "/events") assertURIRecorded(t, ctrl.requestsURIs, "/containers/create") - assertURIRecorded(t, ctrl.requestsURIs, fmt.Sprintf("/containers/%s/start", createResponse.ID)) + assertURIRecorded(t, ctrl.requestsURIs, fmt.Sprintf("/containers/%s/start", cID)) } func systemTime(t *testing.T, client client.APIClient, testEnv *environment.Execution) time.Time { @@ -347,6 +342,8 @@ func TestAuthZPluginEnsureLoadImportWorking(t *testing.T) { client, err := d.NewClient() require.Nil(t, err) + ctx := context.Background() + tmp, err := ioutil.TempDir("", "test-authz-load-import") require.Nil(t, err) defer os.RemoveAll(tmp) @@ -360,13 +357,9 @@ func TestAuthZPluginEnsureLoadImportWorking(t *testing.T) { exportedImagePath := filepath.Join(tmp, "export.tar") - createResponse, err := client.ContainerCreate(context.Background(), &container.Config{Cmd: []string{}, Image: "busybox"}, &container.HostConfig{}, &networktypes.NetworkingConfig{}, "") - require.Nil(t, err) + cID := container.Run(t, ctx, client) - err = client.ContainerStart(context.Background(), createResponse.ID, types.ContainerStartOptions{}) - require.Nil(t, err) - - responseReader, err := client.ContainerExport(context.Background(), createResponse.ID) + responseReader, err := client.ContainerExport(context.Background(), cID) require.Nil(t, err) defer responseReader.Close() file, err := os.Create(exportedImagePath) diff --git a/components/engine/integration/plugin/authz/authz_plugin_v2_test.go b/components/engine/integration/plugin/authz/authz_plugin_v2_test.go index 3f07f48f01..5efa421e88 100644 --- a/components/engine/integration/plugin/authz/authz_plugin_v2_test.go +++ b/components/engine/integration/plugin/authz/authz_plugin_v2_test.go @@ -11,11 +11,10 @@ import ( "testing" "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/filters" - networktypes "github.com/docker/docker/api/types/network" volumetypes "github.com/docker/docker/api/types/volume" "github.com/docker/docker/client" + "github.com/docker/docker/integration/internal/container" "github.com/docker/docker/integration/internal/requirement" "github.com/gotestyourself/gotestyourself/skip" "github.com/stretchr/testify/require" @@ -47,6 +46,8 @@ func TestAuthZPluginV2AllowNonVolumeRequest(t *testing.T) { client, err := d.NewClient() require.Nil(t, err) + ctx := context.Background() + // Install authz plugin err = pluginInstallGrantAllPermissions(client, authzPluginNameWithTag) require.Nil(t, err) @@ -56,13 +57,9 @@ func TestAuthZPluginV2AllowNonVolumeRequest(t *testing.T) { d.LoadBusybox(t) // Ensure docker run command and accompanying docker ps are successful - createResponse, err := client.ContainerCreate(context.Background(), &container.Config{Cmd: []string{"top"}, Image: "busybox"}, &container.HostConfig{}, &networktypes.NetworkingConfig{}, "") - require.Nil(t, err) + cID := container.Run(t, ctx, client) - err = client.ContainerStart(context.Background(), createResponse.ID, types.ContainerStartOptions{}) - require.Nil(t, err) - - _, err = client.ContainerInspect(context.Background(), createResponse.ID) + _, err = client.ContainerInspect(ctx, cID) require.Nil(t, err) } From 366c7398c0555de5ab89a4a8924b2badd799877d Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 13 Feb 2018 15:45:40 +0000 Subject: [PATCH 09/19] Add WithNetworkMode, WithExposedPorts, WithTty, WithWorkingDir to container helper functions Signed-off-by: Yong Tang Upstream-commit: eaa1a0c218454c7f102a1a56c657e806e30d1b1b Component: engine --- .../engine/integration/container/exec_test.go | 5 +-- .../integration/container/health_test.go | 4 +- .../integration/container/links_linux_test.go | 4 +- .../engine/integration/container/nat_test.go | 10 +---- .../integration/internal/container/ops.go | 37 ++++++++++++++++++- 5 files changed, 41 insertions(+), 19 deletions(-) diff --git a/components/engine/integration/container/exec_test.go b/components/engine/integration/container/exec_test.go index 9c27524696..06835678f0 100644 --- a/components/engine/integration/container/exec_test.go +++ b/components/engine/integration/container/exec_test.go @@ -17,10 +17,7 @@ func TestExec(t *testing.T) { ctx := context.Background() client := request.NewAPIClient(t) - cID := container.Run(t, ctx, client, func(c *container.TestContainerConfig) { - c.Config.Tty = true - c.Config.WorkingDir = "/root" - }) + cID := container.Run(t, ctx, client, container.WithTty(true), container.WithWorkingDir("/root")) id, err := client.ContainerExecCreate(ctx, cID, types.ExecConfig{ diff --git a/components/engine/integration/container/health_test.go b/components/engine/integration/container/health_test.go index ec9604c901..651cb2b00c 100644 --- a/components/engine/integration/container/health_test.go +++ b/components/engine/integration/container/health_test.go @@ -20,9 +20,7 @@ func TestHealthCheckWorkdir(t *testing.T) { ctx := context.Background() client := request.NewAPIClient(t) - cID := container.Run(t, ctx, client, func(c *container.TestContainerConfig) { - c.Config.Tty = true - c.Config.WorkingDir = "/foo" + cID := container.Run(t, ctx, client, container.WithTty(true), container.WithWorkingDir("/foo"), func(c *container.TestContainerConfig) { c.Config.Healthcheck = &containertypes.HealthConfig{ Test: []string{"CMD-SHELL", "if [ \"$PWD\" = \"/foo\" ]; then exit 0; else exit 1; fi;"}, Interval: 50 * time.Millisecond, diff --git a/components/engine/integration/container/links_linux_test.go b/components/engine/integration/container/links_linux_test.go index 550b02d874..94e87451de 100644 --- a/components/engine/integration/container/links_linux_test.go +++ b/components/engine/integration/container/links_linux_test.go @@ -28,9 +28,7 @@ func TestLinksEtcHostsContentMatch(t *testing.T) { client := request.NewAPIClient(t) ctx := context.Background() - cID := container.Run(t, ctx, client, container.WithCmd("cat", "/etc/hosts"), func(c *container.TestContainerConfig) { - c.HostConfig.NetworkMode = "host" - }) + cID := container.Run(t, ctx, client, container.WithCmd("cat", "/etc/hosts"), container.WithNetworkMode("host")) poll.WaitOn(t, containerIsStopped(ctx, client, cID), poll.WithDelay(100*time.Millisecond)) diff --git a/components/engine/integration/container/nat_test.go b/components/engine/integration/container/nat_test.go index 1b41ae9608..df3451fbc5 100644 --- a/components/engine/integration/container/nat_test.go +++ b/components/engine/integration/container/nat_test.go @@ -67,10 +67,7 @@ func TestNetworkLoopbackNat(t *testing.T) { client := request.NewAPIClient(t) ctx := context.Background() - cID := container.Run(t, ctx, client, container.WithCmd("sh", "-c", fmt.Sprintf("stty raw && nc -w 5 %s 8080", endpoint.String())), func(c *container.TestContainerConfig) { - c.Config.Tty = true - c.HostConfig.NetworkMode = "container:server" - }) + cID := container.Run(t, ctx, client, container.WithCmd("sh", "-c", fmt.Sprintf("stty raw && nc -w 5 %s 8080", endpoint.String())), container.WithTty(true), container.WithNetworkMode("container:server")) poll.WaitOn(t, containerIsStopped(ctx, client, cID), poll.WithDelay(100*time.Millisecond)) @@ -91,10 +88,7 @@ func startServerContainer(t *testing.T, msg string, port int) string { client := request.NewAPIClient(t) ctx := context.Background() - cID := container.Run(t, ctx, client, container.WithCmd("sh", "-c", fmt.Sprintf("echo %q | nc -lp %d", msg, port)), func(c *container.TestContainerConfig) { - c.Config.ExposedPorts = map[nat.Port]struct{}{ - nat.Port(fmt.Sprintf("%d/tcp", port)): {}, - } + cID := container.Run(t, ctx, client, container.WithName("server"), container.WithCmd("sh", "-c", fmt.Sprintf("echo %q | nc -lp %d", msg, port)), container.WithExposedPorts(fmt.Sprintf("%d/tcp", port)), func(c *container.TestContainerConfig) { c.HostConfig.PortBindings = nat.PortMap{ nat.Port(fmt.Sprintf("%d/tcp", port)): []nat.PortBinding{ { diff --git a/components/engine/integration/internal/container/ops.go b/components/engine/integration/internal/container/ops.go index 940fd68923..e3d538bef1 100644 --- a/components/engine/integration/internal/container/ops.go +++ b/components/engine/integration/internal/container/ops.go @@ -1,6 +1,10 @@ package container -import "github.com/docker/docker/api/types/strslice" +import ( + containertypes "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/strslice" + "github.com/docker/go-connections/nat" +) // WithName sets the name of the container func WithName(name string) func(*TestContainerConfig) { @@ -22,3 +26,34 @@ func WithCmd(cmds ...string) func(*TestContainerConfig) { c.Config.Cmd = strslice.StrSlice(cmds) } } + +// WithNetworkMode sets the network mode of the container +func WithNetworkMode(mode string) func(*TestContainerConfig) { + return func(c *TestContainerConfig) { + c.HostConfig.NetworkMode = containertypes.NetworkMode(mode) + } +} + +// WithExposedPorts sets the exposed ports of the container +func WithExposedPorts(ports ...string) func(*TestContainerConfig) { + return func(c *TestContainerConfig) { + c.Config.ExposedPorts = map[nat.Port]struct{}{} + for _, port := range ports { + c.Config.ExposedPorts[nat.Port(port)] = struct{}{} + } + } +} + +// WithTty sets the TTY mode of the container +func WithTty(tty bool) func(*TestContainerConfig) { + return func(c *TestContainerConfig) { + c.Config.Tty = tty + } +} + +// WithWorkingDir sets the working dir of the container +func WithWorkingDir(dir string) func(*TestContainerConfig) { + return func(c *TestContainerConfig) { + c.Config.WorkingDir = dir + } +} From 65db1dafe3ea0aafb8bb07a31f926b8ae07ffbeb Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 6 Feb 2018 17:48:18 +0000 Subject: [PATCH 10/19] Update runc to 6c55f98695e902427906eed2c799e566e3d3dfb5 This fix is related to 36219 This fix updates runc to: ``` -RUNC_COMMIT=9f9c96235cc97674e935002fc3d78361b696a69e +RUNC_COMMIT=6c55f98695e902427906eed2c799e566e3d3dfb5 -github.com/opencontainers/runc 9f9c96235cc97674e935002fc3d78361b696a69e +github.com/opencontainers/runc 6c55f98695e902427906eed2c799e566e3d3dfb5 ``` Signed-off-by: Yong Tang Upstream-commit: d644050db2a2e341726df49b7a43fc37c05d554a Component: engine --- .../engine/hack/dockerfile/binaries-commits | 2 +- components/engine/vendor.conf | 2 +- .../runc/libcontainer/nsenter/nsexec.c | 107 +++++++++--------- 3 files changed, 55 insertions(+), 56 deletions(-) diff --git a/components/engine/hack/dockerfile/binaries-commits b/components/engine/hack/dockerfile/binaries-commits index ca84b44e58..6333db533d 100644 --- a/components/engine/hack/dockerfile/binaries-commits +++ b/components/engine/hack/dockerfile/binaries-commits @@ -3,7 +3,7 @@ TOMLV_COMMIT=9baf8a8a9f2ed20a8e54160840c492f937eeaf9a # When updating RUNC_COMMIT, also update runc in vendor.conf accordingly -RUNC_COMMIT=9f9c96235cc97674e935002fc3d78361b696a69e +RUNC_COMMIT=6c55f98695e902427906eed2c799e566e3d3dfb5 # containerd is also pinned in vendor.conf. When updating the binary # version you may also need to update the vendor version to pick up bug diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index d2e1a21ed9..ad30591094 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -66,7 +66,7 @@ github.com/pborman/uuid v1.0 google.golang.org/grpc v1.3.0 # When updating, also update RUNC_COMMIT in hack/dockerfile/binaries-commits accordingly -github.com/opencontainers/runc 9f9c96235cc97674e935002fc3d78361b696a69e +github.com/opencontainers/runc 6c55f98695e902427906eed2c799e566e3d3dfb5 github.com/opencontainers/runtime-spec v1.0.1 github.com/opencontainers/image-spec v1.0.1 github.com/seccomp/libseccomp-golang 32f571b70023028bd57d9288c20efbcb237f3ce0 diff --git a/components/engine/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c b/components/engine/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c index a6a107e6e6..2c69cee5d6 100644 --- a/components/engine/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c +++ b/components/engine/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c @@ -22,7 +22,6 @@ #include #include - #include #include #include @@ -32,15 +31,15 @@ /* Synchronisation values. */ enum sync_t { - SYNC_USERMAP_PLS = 0x40, /* Request parent to map our users. */ - SYNC_USERMAP_ACK = 0x41, /* Mapping finished by the parent. */ - SYNC_RECVPID_PLS = 0x42, /* Tell parent we're sending the PID. */ - SYNC_RECVPID_ACK = 0x43, /* PID was correctly received by parent. */ - SYNC_GRANDCHILD = 0x44, /* The grandchild is ready to run. */ - SYNC_CHILD_READY = 0x45, /* The child or grandchild is ready to return. */ + SYNC_USERMAP_PLS = 0x40, /* Request parent to map our users. */ + SYNC_USERMAP_ACK = 0x41, /* Mapping finished by the parent. */ + SYNC_RECVPID_PLS = 0x42, /* Tell parent we're sending the PID. */ + SYNC_RECVPID_ACK = 0x43, /* PID was correctly received by parent. */ + SYNC_GRANDCHILD = 0x44, /* The grandchild is ready to run. */ + SYNC_CHILD_READY = 0x45, /* The child or grandchild is ready to return. */ /* XXX: This doesn't help with segfaults and other such issues. */ - SYNC_ERR = 0xFF, /* Fatal error, no turning back. The error code follows. */ + SYNC_ERR = 0xFF, /* Fatal error, no turning back. The error code follows. */ }; /* longjmp() arguments. */ @@ -73,7 +72,7 @@ struct nlconfig_t { char *oom_score_adj; size_t oom_score_adj_len; - /* User namespace settings.*/ + /* User namespace settings. */ char *uidmap; size_t uidmap_len; char *gidmap; @@ -82,7 +81,7 @@ struct nlconfig_t { size_t namespaces_len; uint8_t is_setgroup; - /* Rootless container settings.*/ + /* Rootless container settings. */ uint8_t is_rootless; char *uidmappath; size_t uidmappath_len; @@ -167,7 +166,7 @@ static int write_file(char *data, size_t data_len, char *pathfmt, ...) goto out; } -out: + out: close(fd); return ret; } @@ -184,16 +183,16 @@ static void update_setgroups(int pid, enum policy_t setgroup) char *policy; switch (setgroup) { - case SETGROUPS_ALLOW: - policy = "allow"; - break; - case SETGROUPS_DENY: - policy = "deny"; - break; - case SETGROUPS_DEFAULT: - default: - /* Nothing to do. */ - return; + case SETGROUPS_ALLOW: + policy = "allow"; + break; + case SETGROUPS_DENY: + policy = "deny"; + break; + case SETGROUPS_DEFAULT: + default: + /* Nothing to do. */ + return; } if (write_file(policy, strlen(policy), "/proc/%d/setgroups", pid) < 0) { @@ -226,14 +225,14 @@ static int try_mapping_tool(const char *app, int pid, char *map, size_t map_len) if (!child) { #define MAX_ARGV 20 char *argv[MAX_ARGV]; - char *envp[] = {NULL}; + char *envp[] = { NULL }; char pid_fmt[16]; int argc = 0; char *next; snprintf(pid_fmt, 16, "%d", pid); - argv[argc++] = (char *) app; + argv[argc++] = (char *)app; argv[argc++] = pid_fmt; /* * Convert the map string into a list of argument that @@ -319,7 +318,7 @@ static int clone_parent(jmp_buf *env, int jmpval) __attribute__ ((noinline)); static int clone_parent(jmp_buf *env, int jmpval) { struct clone_t ca = { - .env = env, + .env = env, .jmpval = jmpval, }; @@ -533,7 +532,7 @@ void nsexec(void) int pipenum; jmp_buf env; int sync_child_pipe[2], sync_grandchild_pipe[2]; - struct nlconfig_t config = {0}; + struct nlconfig_t config = { 0 }; /* * If we don't have an init pipe, just return to the go routine. @@ -630,21 +629,21 @@ void nsexec(void) */ switch (setjmp(env)) { - /* - * Stage 0: We're in the parent. Our job is just to create a new child - * (stage 1: JUMP_CHILD) process and write its uid_map and - * gid_map. That process will go on to create a new process, then - * it will send us its PID which we will send to the bootstrap - * process. - */ - case JUMP_PARENT: { + /* + * Stage 0: We're in the parent. Our job is just to create a new child + * (stage 1: JUMP_CHILD) process and write its uid_map and + * gid_map. That process will go on to create a new process, then + * it will send us its PID which we will send to the bootstrap + * process. + */ + case JUMP_PARENT:{ int len; pid_t child, first_child = -1; char buf[JSON_MAX]; bool ready = false; /* For debugging. */ - prctl(PR_SET_NAME, (unsigned long) "runc:[0:PARENT]", 0, 0, 0); + prctl(PR_SET_NAME, (unsigned long)"runc:[0:PARENT]", 0, 0, 0); /* Start the process of getting a container. */ child = clone_parent(&env, JUMP_CHILD); @@ -702,7 +701,7 @@ void nsexec(void) bail("failed to sync with child: write(SYNC_USERMAP_ACK)"); } break; - case SYNC_RECVPID_PLS: { + case SYNC_RECVPID_PLS:{ first_child = child; /* Get the init_func pid. */ @@ -781,16 +780,16 @@ void nsexec(void) exit(0); } - /* - * Stage 1: We're in the first child process. Our job is to join any - * provided namespaces in the netlink payload and unshare all - * of the requested namespaces. If we've been asked to - * CLONE_NEWUSER, we will ask our parent (stage 0) to set up - * our user mappings for us. Then, we create a new child - * (stage 2: JUMP_INIT) for PID namespace. We then send the - * child's PID to our parent (stage 0). - */ - case JUMP_CHILD: { + /* + * Stage 1: We're in the first child process. Our job is to join any + * provided namespaces in the netlink payload and unshare all + * of the requested namespaces. If we've been asked to + * CLONE_NEWUSER, we will ask our parent (stage 0) to set up + * our user mappings for us. Then, we create a new child + * (stage 2: JUMP_INIT) for PID namespace. We then send the + * child's PID to our parent (stage 0). + */ + case JUMP_CHILD:{ pid_t child; enum sync_t s; @@ -799,7 +798,7 @@ void nsexec(void) close(sync_child_pipe[1]); /* For debugging. */ - prctl(PR_SET_NAME, (unsigned long) "runc:[1:CHILD]", 0, 0, 0); + prctl(PR_SET_NAME, (unsigned long)"runc:[1:CHILD]", 0, 0, 0); /* * We need to setns first. We cannot do this earlier (in stage 0) @@ -901,13 +900,13 @@ void nsexec(void) exit(0); } - /* - * Stage 2: We're the final child process, and the only process that will - * actually return to the Go runtime. Our job is to just do the - * final cleanup steps and then return to the Go runtime to allow - * init_linux.go to run. - */ - case JUMP_INIT: { + /* + * Stage 2: We're the final child process, and the only process that will + * actually return to the Go runtime. Our job is to just do the + * final cleanup steps and then return to the Go runtime to allow + * init_linux.go to run. + */ + case JUMP_INIT:{ /* * We're inside the child now, having jumped from the * start_child() code after forking in the parent. @@ -921,7 +920,7 @@ void nsexec(void) close(sync_child_pipe[1]); /* For debugging. */ - prctl(PR_SET_NAME, (unsigned long) "runc:[2:INIT]", 0, 0, 0); + prctl(PR_SET_NAME, (unsigned long)"runc:[2:INIT]", 0, 0, 0); if (read(syncfd, &s, sizeof(s)) != sizeof(s)) bail("failed to sync with parent: read(SYNC_GRANDCHILD)"); From 6d509c76c47ab928a2db96eb2f9f136d8d6797c3 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 14 Feb 2018 01:35:22 +0100 Subject: [PATCH 11/19] Bump containerd to 1.0.2 (cfd04396dc68220d1cecbe686a6cc3aa5ce3667c) Signed-off-by: Sebastiaan van Stijn Upstream-commit: c2fb6db55be08da95b15bee730a191094f846577 Component: engine --- components/engine/hack/dockerfile/binaries-commits | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/hack/dockerfile/binaries-commits b/components/engine/hack/dockerfile/binaries-commits index 6333db533d..afab51c42a 100644 --- a/components/engine/hack/dockerfile/binaries-commits +++ b/components/engine/hack/dockerfile/binaries-commits @@ -8,7 +8,7 @@ RUNC_COMMIT=6c55f98695e902427906eed2c799e566e3d3dfb5 # containerd is also pinned in vendor.conf. When updating the binary # version you may also need to update the vendor version to pick up bug # fixes or new APIs. -CONTAINERD_COMMIT=9b55aab90508bd389d7654c4baf173a981477d55 # v1.0.1 +CONTAINERD_COMMIT=cfd04396dc68220d1cecbe686a6cc3aa5ce3667c # v1.0.2 TINI_COMMIT=949e6facb77383876aeff8a6944dde66b3089574 LIBNETWORK_COMMIT=fcf1c3b5e57833aaaa756ae3c4140ea54da00319 VNDR_COMMIT=a6e196d8b4b0cbbdc29aebdb20c59ac6926bb384 From 281df74045fe6728cbd7318e8499b7a347e17205 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Wed, 14 Feb 2018 02:03:03 +0100 Subject: [PATCH 12/19] Update containerd dependencies to match 1.0.2 - https://github.com/containerd/go-runc/compare/ed1cbe1fc31f5fb2359d3a54b6330d1a097858b7...4f6e87ae043f859a38255247b49c9abc262d002f - https://github.com/containerd/cgroups/compare/29da22c6171a4316169f9205ab6c49f59b5b852f...c0710c92e8b3a44681d1321dcfd1360fc5c6c089 - runc (already ahead) - https://github.com/stevvooe/ttrpc/compare/76e68349ad9ab4d03d764c713826d31216715e4f...d4528379866b0ce7e9d71f3eb96f0582fc374577 Signed-off-by: Sebastiaan van Stijn Upstream-commit: 175cfdcfb521aa83f6a1441b0a4b99cb159f058d Component: engine --- components/engine/vendor.conf | 6 +- .../github.com/containerd/cgroups/blkio.go | 71 ++++++++----------- .../github.com/containerd/cgroups/errors.go | 2 +- .../github.com/containerd/go-runc/runc.go | 12 ++-- .../github.com/containerd/go-runc/utils.go | 17 +++++ .../github.com/stevvooe/ttrpc/channel.go | 9 ++- .../github.com/stevvooe/ttrpc/client.go | 69 +++++++++++++++--- .../github.com/stevvooe/ttrpc/server.go | 4 +- 8 files changed, 126 insertions(+), 64 deletions(-) diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index ef8a045311..12f0d352b6 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -107,12 +107,12 @@ google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944 github.com/containerd/containerd 3fa104f843ec92328912e042b767d26825f202aa github.com/containerd/fifo fbfb6a11ec671efbe94ad1c12c2e98773f19e1e6 github.com/containerd/continuity 992a5f112bd2211d0983a1cc8562d2882848f3a3 -github.com/containerd/cgroups 29da22c6171a4316169f9205ab6c49f59b5b852f +github.com/containerd/cgroups c0710c92e8b3a44681d1321dcfd1360fc5c6c089 github.com/containerd/console 84eeaae905fa414d03e07bcd6c8d3f19e7cf180e -github.com/containerd/go-runc ed1cbe1fc31f5fb2359d3a54b6330d1a097858b7 +github.com/containerd/go-runc 4f6e87ae043f859a38255247b49c9abc262d002f github.com/containerd/typeurl f6943554a7e7e88b3c14aad190bf05932da84788 github.com/dmcgowan/go-tar go1.10 -github.com/stevvooe/ttrpc 76e68349ad9ab4d03d764c713826d31216715e4f +github.com/stevvooe/ttrpc d4528379866b0ce7e9d71f3eb96f0582fc374577 # cluster github.com/docker/swarmkit 68a376dc30d8c4001767c39456b990dbd821371b diff --git a/components/engine/vendor/github.com/containerd/cgroups/blkio.go b/components/engine/vendor/github.com/containerd/cgroups/blkio.go index 078d12b2a4..9b15b8a62f 100644 --- a/components/engine/vendor/github.com/containerd/cgroups/blkio.go +++ b/components/engine/vendor/github.com/containerd/cgroups/blkio.go @@ -3,12 +3,12 @@ package cgroups import ( "bufio" "fmt" + "io" "io/ioutil" "os" "path/filepath" "strconv" "strings" - "syscall" specs "github.com/opencontainers/runtime-spec/specs-go" ) @@ -105,8 +105,13 @@ func (b *blkioController) Stat(path string, stats *Metrics) error { }, ) } + f, err := os.Open("/proc/diskstats") + if err != nil { + return err + } + defer f.Close() - devices, err := getDevices("/dev") + devices, err := getDevices(f) if err != nil { return err } @@ -268,50 +273,32 @@ type deviceKey struct { // getDevices makes a best effort attempt to read all the devices into a map // keyed by major and minor number. Since devices may be mapped multiple times, // we err on taking the first occurrence. -func getDevices(path string) (map[deviceKey]string, error) { - // TODO(stevvooe): We are ignoring lots of errors. It might be kind of - // challenging to debug this if we aren't mapping devices correctly. - // Consider logging these errors. - devices := map[deviceKey]string{} - if err := filepath.Walk(path, func(p string, fi os.FileInfo, err error) error { +func getDevices(r io.Reader) (map[deviceKey]string, error) { + + var ( + s = bufio.NewScanner(r) + devices = make(map[deviceKey]string) + ) + for s.Scan() { + fields := strings.Fields(s.Text()) + major, err := strconv.Atoi(fields[0]) if err != nil { - return err + return nil, err } - switch { - case fi.IsDir(): - switch fi.Name() { - case "pts", "shm", "fd", "mqueue", ".lxc", ".lxd-mounts": - return filepath.SkipDir - default: - return nil - } - case fi.Name() == "console": - return nil - default: - if fi.Mode()&os.ModeDevice == 0 { - // skip non-devices - return nil - } - - st, ok := fi.Sys().(*syscall.Stat_t) - if !ok { - return fmt.Errorf("%s: unable to convert to system stat", p) - } - - key := deviceKey{major(st.Rdev), minor(st.Rdev)} - if _, ok := devices[key]; ok { - return nil // skip it if we have already populated the path. - } - - devices[key] = p + minor, err := strconv.Atoi(fields[1]) + if err != nil { + return nil, err } - - return nil - }); err != nil { - return nil, err + key := deviceKey{ + major: uint64(major), + minor: uint64(minor), + } + if _, ok := devices[key]; ok { + continue + } + devices[key] = filepath.Join("/dev", fields[2]) } - - return devices, nil + return devices, s.Err() } func major(devNumber uint64) uint64 { diff --git a/components/engine/vendor/github.com/containerd/cgroups/errors.go b/components/engine/vendor/github.com/containerd/cgroups/errors.go index d18b4b1df6..a5824fe23f 100644 --- a/components/engine/vendor/github.com/containerd/cgroups/errors.go +++ b/components/engine/vendor/github.com/containerd/cgroups/errors.go @@ -12,7 +12,7 @@ var ( ErrFreezerNotSupported = errors.New("cgroups: freezer cgroup not supported on this system") ErrMemoryNotSupported = errors.New("cgroups: memory cgroup not supported on this system") ErrCgroupDeleted = errors.New("cgroups: cgroup deleted") - ErrNoCgroupMountDestination = errors.New("cgroups: cannot found cgroup mount destination") + ErrNoCgroupMountDestination = errors.New("cgroups: cannot find cgroup mount destination") ) // ErrorHandler is a function that handles and acts on errors diff --git a/components/engine/vendor/github.com/containerd/go-runc/runc.go b/components/engine/vendor/github.com/containerd/go-runc/runc.go index c5a66a1990..df76ad77a6 100644 --- a/components/engine/vendor/github.com/containerd/go-runc/runc.go +++ b/components/engine/vendor/github.com/containerd/go-runc/runc.go @@ -1,7 +1,6 @@ package runc import ( - "bytes" "context" "encoding/json" "errors" @@ -532,7 +531,9 @@ func (r *Runc) Restore(context context.Context, id, bundle string, opts *Restore // Update updates the current container with the provided resource spec func (r *Runc) Update(context context.Context, id string, resources *specs.LinuxResources) error { - buf := bytes.NewBuffer(nil) + buf := getBuf() + defer putBuf(buf) + if err := json.NewEncoder(buf).Encode(resources); err != nil { return err } @@ -638,11 +639,12 @@ func (r *Runc) runOrError(cmd *exec.Cmd) error { } func cmdOutput(cmd *exec.Cmd, combined bool) ([]byte, error) { - var b bytes.Buffer + b := getBuf() + defer putBuf(b) - cmd.Stdout = &b + cmd.Stdout = b if combined { - cmd.Stderr = &b + cmd.Stderr = b } ec, err := Monitor.Start(cmd) if err != nil { diff --git a/components/engine/vendor/github.com/containerd/go-runc/utils.go b/components/engine/vendor/github.com/containerd/go-runc/utils.go index 81fcd3f2d8..8cb241aca7 100644 --- a/components/engine/vendor/github.com/containerd/go-runc/utils.go +++ b/components/engine/vendor/github.com/containerd/go-runc/utils.go @@ -1,8 +1,10 @@ package runc import ( + "bytes" "io/ioutil" "strconv" + "sync" "syscall" ) @@ -26,3 +28,18 @@ func exitStatus(status syscall.WaitStatus) int { } return status.ExitStatus() } + +var bytesBufferPool = sync.Pool{ + New: func() interface{} { + return bytes.NewBuffer(nil) + }, +} + +func getBuf() *bytes.Buffer { + return bytesBufferPool.Get().(*bytes.Buffer) +} + +func putBuf(b *bytes.Buffer) { + b.Reset() + bytesBufferPool.Put(b) +} diff --git a/components/engine/vendor/github.com/stevvooe/ttrpc/channel.go b/components/engine/vendor/github.com/stevvooe/ttrpc/channel.go index 4a33827a43..9493d68624 100644 --- a/components/engine/vendor/github.com/stevvooe/ttrpc/channel.go +++ b/components/engine/vendor/github.com/stevvooe/ttrpc/channel.go @@ -5,6 +5,7 @@ import ( "context" "encoding/binary" "io" + "net" "sync" "github.com/pkg/errors" @@ -60,16 +61,18 @@ func writeMessageHeader(w io.Writer, p []byte, mh messageHeader) error { var buffers sync.Pool type channel struct { + conn net.Conn bw *bufio.Writer br *bufio.Reader hrbuf [messageHeaderLength]byte // avoid alloc when reading header hwbuf [messageHeaderLength]byte } -func newChannel(w io.Writer, r io.Reader) *channel { +func newChannel(conn net.Conn) *channel { return &channel{ - bw: bufio.NewWriter(w), - br: bufio.NewReader(r), + conn: conn, + bw: bufio.NewWriter(conn), + br: bufio.NewReader(conn), } } diff --git a/components/engine/vendor/github.com/stevvooe/ttrpc/client.go b/components/engine/vendor/github.com/stevvooe/ttrpc/client.go index ca76afe19a..f047181678 100644 --- a/components/engine/vendor/github.com/stevvooe/ttrpc/client.go +++ b/components/engine/vendor/github.com/stevvooe/ttrpc/client.go @@ -2,8 +2,12 @@ package ttrpc import ( "context" + "io" "net" + "os" + "strings" "sync" + "syscall" "github.com/containerd/containerd/log" "github.com/gogo/protobuf/proto" @@ -11,6 +15,10 @@ import ( "google.golang.org/grpc/status" ) +// ErrClosed is returned by client methods when the underlying connection is +// closed. +var ErrClosed = errors.New("ttrpc: closed") + type Client struct { codec codec conn net.Conn @@ -19,18 +27,20 @@ type Client struct { closed chan struct{} closeOnce sync.Once + closeFunc func() done chan struct{} err error } func NewClient(conn net.Conn) *Client { c := &Client{ - codec: codec{}, - conn: conn, - channel: newChannel(conn, conn), - calls: make(chan *callRequest), - closed: make(chan struct{}), - done: make(chan struct{}), + codec: codec{}, + conn: conn, + channel: newChannel(conn), + calls: make(chan *callRequest), + closed: make(chan struct{}), + done: make(chan struct{}), + closeFunc: func() {}, } go c.run() @@ -91,7 +101,7 @@ func (c *Client) dispatch(ctx context.Context, req *Request, resp *Response) err select { case err := <-errs: - return err + return filterCloseErr(err) case <-c.done: return c.err } @@ -105,6 +115,11 @@ func (c *Client) Close() error { return nil } +// OnClose allows a close func to be called when the server is closed +func (c *Client) OnClose(closer func()) { + c.closeFunc = closer +} + type message struct { messageHeader p []byte @@ -150,6 +165,7 @@ func (c *Client) run() { defer c.conn.Close() defer close(c.done) + defer c.closeFunc() for { select { @@ -171,7 +187,14 @@ func (c *Client) run() { call.errs <- c.recv(call.resp, msg) delete(waiters, msg.StreamID) case <-shutdown: + if shutdownErr != nil { + shutdownErr = filterCloseErr(shutdownErr) + } else { + shutdownErr = ErrClosed + } + shutdownErr = errors.Wrapf(shutdownErr, "ttrpc: client shutting down") + c.err = shutdownErr for _, waiter := range waiters { waiter.errs <- shutdownErr @@ -179,9 +202,12 @@ func (c *Client) run() { c.Close() return case <-c.closed: + if c.err == nil { + c.err = ErrClosed + } // broadcast the shutdown error to the remaining waiters. for _, waiter := range waiters { - waiter.errs <- shutdownErr + waiter.errs <- c.err } return } @@ -209,3 +235,30 @@ func (c *Client) recv(resp *Response, msg *message) error { defer c.channel.putmbuf(msg.p) return proto.Unmarshal(msg.p, resp) } + +// filterCloseErr rewrites EOF and EPIPE errors to ErrClosed. Use when +// returning from call or handling errors from main read loop. +// +// This purposely ignores errors with a wrapped cause. +func filterCloseErr(err error) error { + if err == nil { + return nil + } + + if err == io.EOF { + return ErrClosed + } + + if strings.Contains(err.Error(), "use of closed network connection") { + return ErrClosed + } + + // if we have an epipe on a write, we cast to errclosed + if oerr, ok := err.(*net.OpError); ok && oerr.Op == "write" { + if serr, ok := oerr.Err.(*os.SyscallError); ok && serr.Err == syscall.EPIPE { + return ErrClosed + } + } + + return err +} diff --git a/components/engine/vendor/github.com/stevvooe/ttrpc/server.go b/components/engine/vendor/github.com/stevvooe/ttrpc/server.go index edfca0c52c..fd29b719e4 100644 --- a/components/engine/vendor/github.com/stevvooe/ttrpc/server.go +++ b/components/engine/vendor/github.com/stevvooe/ttrpc/server.go @@ -16,7 +16,7 @@ import ( ) var ( - ErrServerClosed = errors.New("ttrpc: server close") + ErrServerClosed = errors.New("ttrpc: server closed") ) type Server struct { @@ -281,7 +281,7 @@ func (c *serverConn) run(sctx context.Context) { ) var ( - ch = newChannel(c.conn, c.conn) + ch = newChannel(c.conn) ctx, cancel = context.WithCancel(sctx) active int state connState = connStateIdle From 0fb8610c546b604bd2f03e48a618e0318bfbd70f Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Tue, 13 Feb 2018 15:03:56 -0500 Subject: [PATCH 13/19] Fix log tail with empty logs When tailing a container log, if the log file is empty it will cause the log stream to abort with an unexpected `EOF`. Note that this only applies to the "current" log file as rotated files cannot be empty. This fix just skips adding the "current" file the log tail if it is empty. Signed-off-by: Brian Goff Upstream-commit: f40860c5f3d3575629d4a932207e866c1fea625d Component: engine --- .../daemon/logger/loggerutils/logfile.go | 8 +++-- .../engine/integration/container/logs_test.go | 33 +++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 components/engine/integration/container/logs_test.go diff --git a/components/engine/daemon/logger/loggerutils/logfile.go b/components/engine/daemon/logger/loggerutils/logfile.go index ff7a3323f9..a9d9d633db 100644 --- a/components/engine/daemon/logger/loggerutils/logfile.go +++ b/components/engine/daemon/logger/loggerutils/logfile.go @@ -192,8 +192,12 @@ func (w *LogFile) ReadLogs(config logger.ReadConfig, watcher *logger.LogWatcher) for _, f := range files { seekers = append(seekers, f) } - seekers = append(seekers, currentChunk) - tailFile(multireader.MultiReadSeeker(seekers...), watcher, w.createDecoder, config) + if currentChunk.Size() > 0 { + seekers = append(seekers, currentChunk) + } + if len(seekers) > 0 { + tailFile(multireader.MultiReadSeeker(seekers...), watcher, w.createDecoder, config) + } } w.mu.RLock() diff --git a/components/engine/integration/container/logs_test.go b/components/engine/integration/container/logs_test.go new file mode 100644 index 0000000000..1157da14b8 --- /dev/null +++ b/components/engine/integration/container/logs_test.go @@ -0,0 +1,33 @@ +package container + +import ( + "context" + "io/ioutil" + "testing" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/integration/internal/container" + "github.com/docker/docker/integration/internal/request" + "github.com/docker/docker/pkg/stdcopy" + "github.com/stretchr/testify/assert" +) + +// Regression test for #35370 +// Makes sure that when following we don't get an EOF error when there are no logs +func TestLogsFollowTailEmpty(t *testing.T) { + defer setupTest(t)() + client := request.NewAPIClient(t) + ctx := context.Background() + + id := container.Run(t, ctx, client, container.WithCmd("sleep", "100000")) + defer client.ContainerRemove(ctx, id, types.ContainerRemoveOptions{Force: true}) + + logs, err := client.ContainerLogs(ctx, id, types.ContainerLogsOptions{ShowStdout: true, Tail: "2"}) + if logs != nil { + defer logs.Close() + } + assert.NoError(t, err) + + _, err = stdcopy.StdCopy(ioutil.Discard, ioutil.Discard, logs) + assert.NoError(t, err) +} From b660bf165d1197716d38360c1d8d209417b65fbd Mon Sep 17 00:00:00 2001 From: Dennis Chen Date: Tue, 13 Feb 2018 04:52:10 +0000 Subject: [PATCH 14/19] Unify the frozen images to the multi-arch version Update and unify the `busybox` images on all arches to the `glibc` multi-arch version and remove the temp workaround on amd64 which uses the old version busybox (v1.26) before this PR to bypass the failure of those network related test cases. Also, this PR will fix all the network related issues with `glibc` version `busybox` image. Signed-off-by: Dennis Chen Upstream-commit: 3a971009763387856bb7f162accdf6714100e39b Component: engine --- components/engine/Dockerfile | 3 +-- components/engine/Dockerfile.aarch64 | 2 +- components/engine/Dockerfile.armhf | 2 +- components/engine/Dockerfile.e2e | 8 ++++---- components/engine/Dockerfile.ppc64le | 2 +- components/engine/Dockerfile.s390x | 2 +- .../engine/integration-cli/docker_cli_build_test.go | 8 +------- components/engine/integration-cli/docker_cli_run_test.go | 9 +-------- .../engine/integration-cli/fixtures/load/frozen.go | 3 +++ components/engine/internal/test/environment/protect.go | 2 +- 10 files changed, 15 insertions(+), 26 deletions(-) diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile index aa910c751f..718855ee96 100644 --- a/components/engine/Dockerfile +++ b/components/engine/Dockerfile @@ -168,10 +168,9 @@ RUN echo "source $PWD/hack/make/.integration-test-helpers" >> /etc/bash.bashrc # Get useful and necessary Hub images so we can "docker load" locally instead of pulling COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/ -# TODO: when issue #35963 fixed, we can upgrade the busybox to multi-arch RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ - busybox:latest@sha256:32f093055929dbc23dec4d03e09dfe971f5973a9ca5cf059cbfb644c206aa83f \ + busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \ debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) diff --git a/components/engine/Dockerfile.aarch64 b/components/engine/Dockerfile.aarch64 index c654207b20..031cfbb5f4 100644 --- a/components/engine/Dockerfile.aarch64 +++ b/components/engine/Dockerfile.aarch64 @@ -144,7 +144,7 @@ RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ - busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \ + busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \ debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) diff --git a/components/engine/Dockerfile.armhf b/components/engine/Dockerfile.armhf index 0a8f06c59e..401f5f42c7 100644 --- a/components/engine/Dockerfile.armhf +++ b/components/engine/Dockerfile.armhf @@ -133,7 +133,7 @@ RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ - busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \ + busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \ debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) diff --git a/components/engine/Dockerfile.e2e b/components/engine/Dockerfile.e2e index 294a6aa85a..bfac86d291 100644 --- a/components/engine/Dockerfile.e2e +++ b/components/engine/Dockerfile.e2e @@ -16,10 +16,10 @@ WORKDIR /go/src/github.com/docker/docker/ # Generate frozen images COPY contrib/download-frozen-image-v2.sh contrib/download-frozen-image-v2.sh RUN contrib/download-frozen-image-v2.sh /output/docker-frozen-images \ - buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ - busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \ - debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ - hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c + buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ + busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \ + debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ + hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # Download Docker CLI binary COPY hack/dockerfile hack/dockerfile diff --git a/components/engine/Dockerfile.ppc64le b/components/engine/Dockerfile.ppc64le index 41c7318ce8..f9c0a0ba32 100644 --- a/components/engine/Dockerfile.ppc64le +++ b/components/engine/Dockerfile.ppc64le @@ -131,7 +131,7 @@ RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ - busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \ + busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \ debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) diff --git a/components/engine/Dockerfile.s390x b/components/engine/Dockerfile.s390x index f6cbe7dab3..4f954487ca 100644 --- a/components/engine/Dockerfile.s390x +++ b/components/engine/Dockerfile.s390x @@ -125,7 +125,7 @@ RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ - busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \ + busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \ debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) diff --git a/components/engine/integration-cli/docker_cli_build_test.go b/components/engine/integration-cli/docker_cli_build_test.go index dda77c94a1..5dd1d8b355 100644 --- a/components/engine/integration-cli/docker_cli_build_test.go +++ b/components/engine/integration-cli/docker_cli_build_test.go @@ -400,13 +400,7 @@ func (s *DockerSuite) TestBuildLastModified(c *check.C) { defer server.Close() var out, out2 string - var args []string - // Temopray workaround for #35963. Will remove this when that issue fixed - if runtime.GOARCH == "amd64" { - args = []string{"run", name, "ls", "-le", "/file"} - } else { - args = []string{"run", name, "ls", "-l", "--full-time", "/file"} - } + args := []string{"run", name, "ls", "-l", "--full-time", "/file"} dFmt := `FROM busybox ADD %s/file /` diff --git a/components/engine/integration-cli/docker_cli_run_test.go b/components/engine/integration-cli/docker_cli_run_test.go index 2a652e35e4..776f0e5ba5 100644 --- a/components/engine/integration-cli/docker_cli_run_test.go +++ b/components/engine/integration-cli/docker_cli_run_test.go @@ -2239,14 +2239,7 @@ func (s *DockerSuite) TestRunSlowStdoutConsumer(c *check.C) { // alternate to /dev/zero and /dev/stdout. testRequires(c, DaemonIsLinux) - // TODO will remove this if issue #35963 fixed - var args []string - if runtime.GOARCH == "amd64" { - args = []string{"run", "--rm", "busybox", "/bin/sh", "-c", "dd if=/dev/zero of=/dev/stdout bs=1024 count=2000 | catv"} - } else { - args = []string{"run", "--rm", "busybox", "/bin/sh", "-c", "dd if=/dev/zero of=/dev/stdout bs=1024 count=2000 | cat -v"} - } - + args := []string{"run", "--rm", "busybox", "/bin/sh", "-c", "dd if=/dev/zero of=/dev/stdout bs=1024 count=2000 | cat -v"} cont := exec.Command(dockerBinary, args...) stdout, err := cont.StdoutPipe() diff --git a/components/engine/integration-cli/fixtures/load/frozen.go b/components/engine/integration-cli/fixtures/load/frozen.go index 5701a216a2..7e104fe76c 100644 --- a/components/engine/integration-cli/fixtures/load/frozen.go +++ b/components/engine/integration-cli/fixtures/load/frozen.go @@ -37,6 +37,9 @@ func FrozenImagesLinux(client client.APIClient, images ...string) error { if img == "hello-world:frozen" { srcName = "hello-world:latest" } + if img == "busybox:1.27-glibc" { + img = "busybox:latest" + } loadImages = append(loadImages, struct{ srcName, destName string }{ srcName: srcName, destName: img, diff --git a/components/engine/internal/test/environment/protect.go b/components/engine/internal/test/environment/protect.go index 9aa6fdd602..482dde60b6 100644 --- a/components/engine/internal/test/environment/protect.go +++ b/components/engine/internal/test/environment/protect.go @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/require" ) -var frozenImages = []string{"busybox:latest", "hello-world:frozen", "debian:jessie"} +var frozenImages = []string{"busybox:1.27-glibc", "hello-world:frozen", "debian:jessie"} type protectedElements struct { containers map[string]struct{} From 1d51022fe81c77d3b394b80996310fa5e6812c1e Mon Sep 17 00:00:00 2001 From: Brett Randall Date: Wed, 14 Feb 2018 22:03:52 +1100 Subject: [PATCH 15/19] Updated docker-on-docker build-notes. These are now more in-line with wiki instructions. Also removes broken/deprecated make target test-unit. Signed-off-by: Brett Randall Upstream-commit: ba49e8c49830b69c833edff3c393716da20f897a Component: engine --- components/engine/Dockerfile | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile index 560adc79a2..6b137c3990 100644 --- a/components/engine/Dockerfile +++ b/components/engine/Dockerfile @@ -2,14 +2,23 @@ # # Usage: # -# # Assemble the full dev environment. This is slow the first time. -# docker build -t docker . +# # Use make to build a development environment image and run it in a container. +# # This is slow the first time. +# make BIND_DIR=. shell # -# # Mount your source in an interactive container for quick testing: -# docker run -v `pwd`:/go/src/github.com/docker/docker --privileged -i -t docker bash +# The following commands are executed inside the running container. + +# # Make a dockerd binary. +# # hack/make.sh binary # -# # Run the test suite: -# docker run -e DOCKER_GITCOMMIT=foo --privileged docker hack/make.sh test-unit test-integration test-docker-py +# # Install dockerd to /usr/local/bin +# # make install +# +# # Run unit tests +# # hack/test/unit +# +# # Run tests e.g. integration, py +# # hack/make.sh binary test-integration test-docker-py # # # Publish a release: # docker run --privileged \ From 0e750709d335ec5a8c5070038bd53f192cefbf1e Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 7 Feb 2018 15:33:20 -0500 Subject: [PATCH 16/19] Move commit to container backend Signed-off-by: Daniel Nephin Upstream-commit: e574c5ae73f2f54c47319e5e4a17b16bd93213be Component: engine --- .../api/server/router/container/backend.go | 5 +++ .../api/server/router/container/container.go | 1 + .../router/container/container_routes.go | 39 ++++++++++++++++++ .../engine/api/server/router/image/backend.go | 6 --- .../engine/api/server/router/image/image.go | 10 +---- .../api/server/router/image/image_routes.go | 41 ------------------- components/engine/cmd/dockerd/daemon.go | 2 +- 7 files changed, 48 insertions(+), 56 deletions(-) diff --git a/components/engine/api/server/router/container/backend.go b/components/engine/api/server/router/container/backend.go index b6b1dec94c..5072083c69 100644 --- a/components/engine/api/server/router/container/backend.go +++ b/components/engine/api/server/router/container/backend.go @@ -68,8 +68,13 @@ type systemBackend interface { ContainersPrune(ctx context.Context, pruneFilters filters.Args) (*types.ContainersPruneReport, error) } +type commitBackend interface { + CreateImageFromContainer(name string, config *backend.CreateImageConfig) (imageID string, err error) +} + // Backend is all the methods that need to be implemented to provide container specific functionality. type Backend interface { + commitBackend execBackend copyBackend stateBackend diff --git a/components/engine/api/server/router/container/container.go b/components/engine/api/server/router/container/container.go index c763900c60..358f2bc2c1 100644 --- a/components/engine/api/server/router/container/container.go +++ b/components/engine/api/server/router/container/container.go @@ -61,6 +61,7 @@ func (r *containerRouter) initRoutes() { router.NewPostRoute("/containers/{name:.*}/rename", r.postContainerRename), router.NewPostRoute("/containers/{name:.*}/update", r.postContainerUpdate), router.NewPostRoute("/containers/prune", r.postContainersPrune, router.WithCancel), + router.NewPostRoute("/commit", r.postCommit), // PUT router.NewPutRoute("/containers/{name:.*}/archive", r.putContainersArchive), // DELETE diff --git a/components/engine/api/server/router/container/container_routes.go b/components/engine/api/server/router/container/container_routes.go index 5de01e7639..0527810f23 100644 --- a/components/engine/api/server/router/container/container_routes.go +++ b/components/engine/api/server/router/container/container_routes.go @@ -24,6 +24,45 @@ import ( "golang.org/x/net/websocket" ) +func (s *containerRouter) postCommit(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + if err := httputils.ParseForm(r); err != nil { + return err + } + + if err := httputils.CheckForJSON(r); err != nil { + return err + } + + // TODO: remove pause arg, and always pause in backend + pause := httputils.BoolValue(r, "pause") + version := httputils.VersionFromContext(ctx) + if r.FormValue("pause") == "" && versions.GreaterThanOrEqualTo(version, "1.13") { + pause = true + } + + config, _, _, err := s.decoder.DecodeConfig(r.Body) + if err != nil && err != io.EOF { //Do not fail if body is empty. + return err + } + + commitCfg := &backend.CreateImageConfig{ + Pause: pause, + Repo: r.Form.Get("repo"), + Tag: r.Form.Get("tag"), + Author: r.Form.Get("author"), + Comment: r.Form.Get("comment"), + Config: config, + Changes: r.Form["changes"], + } + + imgID, err := s.backend.CreateImageFromContainer(r.Form.Get("container"), commitCfg) + if err != nil { + return err + } + + return httputils.WriteJSON(w, http.StatusCreated, &types.IDResponse{ID: imgID}) +} + func (s *containerRouter) getContainersJSON(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { if err := httputils.ParseForm(r); err != nil { return err diff --git a/components/engine/api/server/router/image/backend.go b/components/engine/api/server/router/image/backend.go index ffbc9c181a..e4d4b7d1bb 100644 --- a/components/engine/api/server/router/image/backend.go +++ b/components/engine/api/server/router/image/backend.go @@ -4,7 +4,6 @@ import ( "io" "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/backend" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/image" "github.com/docker/docker/api/types/registry" @@ -14,16 +13,11 @@ import ( // Backend is all the methods that need to be implemented // to provide image specific functionality. type Backend interface { - containerBackend imageBackend importExportBackend registryBackend } -type containerBackend interface { - CreateImageFromContainer(name string, config *backend.CreateImageConfig) (imageID string, err error) -} - type imageBackend interface { ImageDelete(imageRef string, force, prune bool) ([]types.ImageDeleteResponseItem, error) ImageHistory(imageName string) ([]*image.HistoryResponseItem, error) diff --git a/components/engine/api/server/router/image/image.go b/components/engine/api/server/router/image/image.go index 980bbd2421..6d5d87f63c 100644 --- a/components/engine/api/server/router/image/image.go +++ b/components/engine/api/server/router/image/image.go @@ -1,23 +1,18 @@ package image // import "github.com/docker/docker/api/server/router/image" import ( - "github.com/docker/docker/api/server/httputils" "github.com/docker/docker/api/server/router" ) // imageRouter is a router to talk with the image controller type imageRouter struct { backend Backend - decoder httputils.ContainerDecoder routes []router.Route } // NewRouter initializes a new image router -func NewRouter(backend Backend, decoder httputils.ContainerDecoder) router.Router { - r := &imageRouter{ - backend: backend, - decoder: decoder, - } +func NewRouter(backend Backend) router.Router { + r := &imageRouter{backend: backend} r.initRoutes() return r } @@ -38,7 +33,6 @@ func (r *imageRouter) initRoutes() { router.NewGetRoute("/images/{name:.*}/history", r.getImagesHistory), router.NewGetRoute("/images/{name:.*}/json", r.getImagesByName), // POST - router.NewPostRoute("/commit", r.postCommit), router.NewPostRoute("/images/load", r.postImagesLoad), router.NewPostRoute("/images/create", r.postImagesCreate, router.WithCancel), router.NewPostRoute("/images/{name:.*}/push", r.postImagesPush, router.WithCancel), diff --git a/components/engine/api/server/router/image/image_routes.go b/components/engine/api/server/router/image/image_routes.go index 19a9e74f45..cb497134a2 100644 --- a/components/engine/api/server/router/image/image_routes.go +++ b/components/engine/api/server/router/image/image_routes.go @@ -4,14 +4,12 @@ import ( "encoding/base64" "encoding/json" "fmt" - "io" "net/http" "strconv" "strings" "github.com/docker/docker/api/server/httputils" "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/backend" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/versions" "github.com/docker/docker/errdefs" @@ -24,45 +22,6 @@ import ( "golang.org/x/net/context" ) -func (s *imageRouter) postCommit(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - if err := httputils.ParseForm(r); err != nil { - return err - } - - if err := httputils.CheckForJSON(r); err != nil { - return err - } - - // TODO: remove pause arg, and always pause in backend - pause := httputils.BoolValue(r, "pause") - version := httputils.VersionFromContext(ctx) - if r.FormValue("pause") == "" && versions.GreaterThanOrEqualTo(version, "1.13") { - pause = true - } - - config, _, _, err := s.decoder.DecodeConfig(r.Body) - if err != nil && err != io.EOF { //Do not fail if body is empty. - return err - } - - commitCfg := &backend.CreateImageConfig{ - Pause: pause, - Repo: r.Form.Get("repo"), - Tag: r.Form.Get("tag"), - Author: r.Form.Get("author"), - Comment: r.Form.Get("comment"), - Config: config, - Changes: r.Form["changes"], - } - - imgID, err := s.backend.CreateImageFromContainer(r.Form.Get("container"), commitCfg) - if err != nil { - return err - } - - return httputils.WriteJSON(w, http.StatusCreated, &types.IDResponse{ID: imgID}) -} - // Creates an image from Pull or from Import func (s *imageRouter) postImagesCreate(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error { diff --git a/components/engine/cmd/dockerd/daemon.go b/components/engine/cmd/dockerd/daemon.go index d73b63a0f0..6665cf032e 100644 --- a/components/engine/cmd/dockerd/daemon.go +++ b/components/engine/cmd/dockerd/daemon.go @@ -514,7 +514,7 @@ func initRouter(opts routerOptions) { // we need to add the checkpoint router before the container router or the DELETE gets masked checkpointrouter.NewRouter(opts.daemon, decoder), container.NewRouter(opts.daemon, decoder), - image.NewRouter(opts.daemon, decoder), + image.NewRouter(opts.daemon), systemrouter.NewRouter(opts.daemon, opts.cluster, opts.buildCache), volume.NewRouter(opts.daemon), build.NewRouter(opts.buildBackend, opts.daemon), From d80026fd7e8bd7f8902508b7bb2e8d299076c628 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 15 Feb 2018 07:43:44 +0000 Subject: [PATCH 17/19] Update docker-py to 5e28dcaace5f7b70cbe44c313b7a3b288fa38916 This fix updates docker-py: ``` -ENV DOCKER_PY_COMMIT 1d6b5b203222ba5df7dedfcd1ee061a452f99c8a +ENV DOCKER_PY_COMMIT 5e28dcaace5f7b70cbe44c313b7a3b288fa38916 ``` The updated docker-py includes https://github.com/docker/docker-py/pull/1909 which is required to have #36292 pass the tests. Full diff is in https://github.com/docker/docker-py/compare/1d6b5b203222ba5df7dedfcd1ee061a452f99c8a...5e28dcaace5f7b70cbe44c313b7a3b288fa38916. Signed-off-by: Yong Tang Upstream-commit: 9d9af83b0fd70ff6a7faa15cf8746669f0f3b588 Component: engine --- components/engine/Dockerfile | 2 +- components/engine/Dockerfile.aarch64 | 2 +- components/engine/Dockerfile.armhf | 2 +- components/engine/Dockerfile.ppc64le | 2 +- components/engine/Dockerfile.s390x | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile index 6b137c3990..3c02cf6d92 100644 --- a/components/engine/Dockerfile +++ b/components/engine/Dockerfile @@ -142,7 +142,7 @@ RUN set -x \ && rm -rf "$GOPATH" # Get the "docker-py" source so we can run their integration tests -ENV DOCKER_PY_COMMIT 1d6b5b203222ba5df7dedfcd1ee061a452f99c8a +ENV DOCKER_PY_COMMIT 5e28dcaace5f7b70cbe44c313b7a3b288fa38916 # To run integration tests docker-pycreds is required. RUN git clone https://github.com/docker/docker-py.git /docker-py \ && cd /docker-py \ diff --git a/components/engine/Dockerfile.aarch64 b/components/engine/Dockerfile.aarch64 index e1c7706c01..7928f95919 100644 --- a/components/engine/Dockerfile.aarch64 +++ b/components/engine/Dockerfile.aarch64 @@ -106,7 +106,7 @@ RUN set -x \ && rm -rf "$GOPATH" # Get the "docker-py" source so we can run their integration tests -ENV DOCKER_PY_COMMIT 1d6b5b203222ba5df7dedfcd1ee061a452f99c8a +ENV DOCKER_PY_COMMIT 5e28dcaace5f7b70cbe44c313b7a3b288fa38916 # To run integration tests docker-pycreds is required. RUN git clone https://github.com/docker/docker-py.git /docker-py \ && cd /docker-py \ diff --git a/components/engine/Dockerfile.armhf b/components/engine/Dockerfile.armhf index 0a8f06c59e..e695324b8d 100644 --- a/components/engine/Dockerfile.armhf +++ b/components/engine/Dockerfile.armhf @@ -104,7 +104,7 @@ RUN set -x \ && rm -rf "$GOPATH" # Get the "docker-py" source so we can run their integration tests -ENV DOCKER_PY_COMMIT 1d6b5b203222ba5df7dedfcd1ee061a452f99c8a +ENV DOCKER_PY_COMMIT 5e28dcaace5f7b70cbe44c313b7a3b288fa38916 # To run integration tests docker-pycreds is required. RUN git clone https://github.com/docker/docker-py.git /docker-py \ && cd /docker-py \ diff --git a/components/engine/Dockerfile.ppc64le b/components/engine/Dockerfile.ppc64le index 41c7318ce8..930e5378ce 100644 --- a/components/engine/Dockerfile.ppc64le +++ b/components/engine/Dockerfile.ppc64le @@ -102,7 +102,7 @@ RUN set -x \ && rm -rf "$GOPATH" # Get the "docker-py" source so we can run their integration tests -ENV DOCKER_PY_COMMIT 1d6b5b203222ba5df7dedfcd1ee061a452f99c8a +ENV DOCKER_PY_COMMIT 5e28dcaace5f7b70cbe44c313b7a3b288fa38916 # To run integration tests docker-pycreds is required. RUN git clone https://github.com/docker/docker-py.git /docker-py \ && cd /docker-py \ diff --git a/components/engine/Dockerfile.s390x b/components/engine/Dockerfile.s390x index f6cbe7dab3..b56636cca8 100644 --- a/components/engine/Dockerfile.s390x +++ b/components/engine/Dockerfile.s390x @@ -96,7 +96,7 @@ RUN set -x \ && rm -rf "$GOPATH" # Get the "docker-py" source so we can run their integration tests -ENV DOCKER_PY_COMMIT 1d6b5b203222ba5df7dedfcd1ee061a452f99c8a +ENV DOCKER_PY_COMMIT 5e28dcaace5f7b70cbe44c313b7a3b288fa38916 # To run integration tests docker-pycreds is required. RUN git clone https://github.com/docker/docker-py.git /docker-py \ && cd /docker-py \ From 1903b720b8845d683b3e17dc8490f123e9f7bc8c Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 15 Feb 2018 01:01:53 -0800 Subject: [PATCH 18/19] Remove docker_cli_diff_test.go from integration-cli Signed-off-by: Yong Tang Upstream-commit: f19bea20c91e1cd5a748aa241e7083e8c6246634 Component: engine --- .../integration-cli/docker_cli_diff_test.go | 98 ------------------- 1 file changed, 98 deletions(-) delete mode 100644 components/engine/integration-cli/docker_cli_diff_test.go diff --git a/components/engine/integration-cli/docker_cli_diff_test.go b/components/engine/integration-cli/docker_cli_diff_test.go deleted file mode 100644 index 614beee43c..0000000000 --- a/components/engine/integration-cli/docker_cli_diff_test.go +++ /dev/null @@ -1,98 +0,0 @@ -package main - -import ( - "strings" - "time" - - "github.com/docker/docker/integration-cli/checker" - "github.com/docker/docker/integration-cli/cli" - "github.com/go-check/check" -) - -// ensure that an added file shows up in docker diff -func (s *DockerSuite) TestDiffFilenameShownInOutput(c *check.C) { - containerCmd := `mkdir /foo; echo xyzzy > /foo/bar` - out := cli.DockerCmd(c, "run", "-d", "busybox", "sh", "-c", containerCmd).Combined() - - // Wait for it to exit as cannot diff a running container on Windows, and - // it will take a few seconds to exit. Also there's no way in Windows to - // differentiate between an Add or a Modify, and all files are under - // a "Files/" prefix. - containerID := strings.TrimSpace(out) - lookingFor := "A /foo/bar" - if testEnv.OSType == "windows" { - cli.WaitExited(c, containerID, 60*time.Second) - lookingFor = "C Files/foo/bar" - } - - cleanCID := strings.TrimSpace(out) - out = cli.DockerCmd(c, "diff", cleanCID).Combined() - - found := false - for _, line := range strings.Split(out, "\n") { - if strings.Contains(line, lookingFor) { - found = true - break - } - } - c.Assert(found, checker.True) -} - -// test to ensure GH #3840 doesn't occur any more -func (s *DockerSuite) TestDiffEnsureInitLayerFilesAreIgnored(c *check.C) { - testRequires(c, DaemonIsLinux) - // this is a list of files which shouldn't show up in `docker diff` - initLayerFiles := []string{"/etc/resolv.conf", "/etc/hostname", "/etc/hosts", "/.dockerenv"} - containerCount := 5 - - // we might not run into this problem from the first run, so start a few containers - for i := 0; i < containerCount; i++ { - containerCmd := `echo foo > /root/bar` - out, _ := dockerCmd(c, "run", "-d", "busybox", "sh", "-c", containerCmd) - - cleanCID := strings.TrimSpace(out) - out, _ = dockerCmd(c, "diff", cleanCID) - - for _, filename := range initLayerFiles { - c.Assert(out, checker.Not(checker.Contains), filename) - } - } -} - -func (s *DockerSuite) TestDiffEnsureDefaultDevs(c *check.C) { - testRequires(c, DaemonIsLinux) - out, _ := dockerCmd(c, "run", "-d", "busybox", "sleep", "0") - - cleanCID := strings.TrimSpace(out) - out, _ = dockerCmd(c, "diff", cleanCID) - - expected := map[string]bool{ - "C /dev": true, - "A /dev/full": true, // busybox - "C /dev/ptmx": true, // libcontainer - "A /dev/mqueue": true, - "A /dev/kmsg": true, - "A /dev/fd": true, - "A /dev/ptmx": true, - "A /dev/null": true, - "A /dev/random": true, - "A /dev/stdout": true, - "A /dev/stderr": true, - "A /dev/tty1": true, - "A /dev/stdin": true, - "A /dev/tty": true, - "A /dev/urandom": true, - "A /dev/zero": true, - } - - for _, line := range strings.Split(out, "\n") { - c.Assert(line == "" || expected[line], checker.True, check.Commentf(line)) - } -} - -// https://github.com/docker/docker/pull/14381#discussion_r33859347 -func (s *DockerSuite) TestDiffEmptyArgClientError(c *check.C) { - out, _, err := dockerCmdWithError("diff", "") - c.Assert(err, checker.NotNil) - c.Assert(strings.TrimSpace(out), checker.Contains, "Container name cannot be empty") -} From 90c87d3cd766f5382f7e145fa298301765060eea Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 15 Feb 2018 01:02:27 -0800 Subject: [PATCH 19/19] Migrate container diff tests in integration-cli to api tests. This fix migreates container diff tests in integration-cli to api tests. Signed-off-by: Yong Tang Upstream-commit: 9537498cedc4e28ee0c8c26ba3d9e59ebb59fcad Component: engine --- .../engine/integration/container/diff_test.go | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 components/engine/integration/container/diff_test.go diff --git a/components/engine/integration/container/diff_test.go b/components/engine/integration/container/diff_test.go new file mode 100644 index 0000000000..63ac19e9ad --- /dev/null +++ b/components/engine/integration/container/diff_test.go @@ -0,0 +1,98 @@ +package container // import "github.com/docker/docker/integration/container" + +import ( + "context" + "testing" + "time" + + containertypes "github.com/docker/docker/api/types/container" + "github.com/docker/docker/integration/internal/container" + "github.com/docker/docker/integration/internal/request" + "github.com/docker/docker/pkg/archive" + "github.com/gotestyourself/gotestyourself/poll" + "github.com/gotestyourself/gotestyourself/skip" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// ensure that an added file shows up in docker diff +func TestDiffFilenameShownInOutput(t *testing.T) { + defer setupTest(t)() + client := request.NewAPIClient(t) + ctx := context.Background() + + cID := container.Run(t, ctx, client, container.WithCmd("sh", "-c", `mkdir /foo; echo xyzzy > /foo/bar`)) + + // Wait for it to exit as cannot diff a running container on Windows, and + // it will take a few seconds to exit. Also there's no way in Windows to + // differentiate between an Add or a Modify, and all files are under + // a "Files/" prefix. + lookingFor := containertypes.ContainerChangeResponseItem{Kind: archive.ChangeAdd, Path: "/foo/bar"} + if testEnv.OSType == "windows" { + poll.WaitOn(t, containerIsInState(ctx, client, cID, "exited"), poll.WithDelay(100*time.Millisecond), poll.WithTimeout(60*time.Second)) + lookingFor = containertypes.ContainerChangeResponseItem{Kind: archive.ChangeModify, Path: "Files/foo/bar"} + } + + items, err := client.ContainerDiff(ctx, cID) + require.NoError(t, err) + assert.Contains(t, items, lookingFor) +} + +// test to ensure GH #3840 doesn't occur any more +func TestDiffEnsureInitLayerFilesAreIgnored(t *testing.T) { + skip.If(t, testEnv.DaemonInfo.OSType != "linux") + + defer setupTest(t)() + client := request.NewAPIClient(t) + ctx := context.Background() + + // this is a list of files which shouldn't show up in `docker diff` + initLayerFiles := []string{"/etc/resolv.conf", "/etc/hostname", "/etc/hosts", "/.dockerenv"} + containerCount := 5 + + // we might not run into this problem from the first run, so start a few containers + for i := 0; i < containerCount; i++ { + cID := container.Run(t, ctx, client, container.WithCmd("sh", "-c", `echo foo > /root/bar`)) + + items, err := client.ContainerDiff(ctx, cID) + require.NoError(t, err) + for _, item := range items { + assert.NotContains(t, initLayerFiles, item.Path) + } + } +} + +func TestDiffEnsureDefaultDevs(t *testing.T) { + skip.If(t, testEnv.DaemonInfo.OSType != "linux") + + defer setupTest(t)() + client := request.NewAPIClient(t) + ctx := context.Background() + + cID := container.Run(t, ctx, client, container.WithCmd("sleep", "0")) + + items, err := client.ContainerDiff(ctx, cID) + require.NoError(t, err) + + expected := []containertypes.ContainerChangeResponseItem{ + {Kind: archive.ChangeModify, Path: "/dev"}, + {Kind: archive.ChangeAdd, Path: "/dev/full"}, // busybox + {Kind: archive.ChangeModify, Path: "/dev/ptmx"}, // libcontainer + {Kind: archive.ChangeAdd, Path: "/dev/mqueue"}, + {Kind: archive.ChangeAdd, Path: "/dev/kmsg"}, + {Kind: archive.ChangeAdd, Path: "/dev/fd"}, + {Kind: archive.ChangeAdd, Path: "/dev/ptmx"}, + {Kind: archive.ChangeAdd, Path: "/dev/null"}, + {Kind: archive.ChangeAdd, Path: "/dev/random"}, + {Kind: archive.ChangeAdd, Path: "/dev/stdout"}, + {Kind: archive.ChangeAdd, Path: "/dev/stderr"}, + {Kind: archive.ChangeAdd, Path: "/dev/tty1"}, + {Kind: archive.ChangeAdd, Path: "/dev/stdin"}, + {Kind: archive.ChangeAdd, Path: "/dev/tty"}, + {Kind: archive.ChangeAdd, Path: "/dev/urandom"}, + } + + for _, item := range items { + assert.Contains(t, expected, item) + } +}