From 4f8240aeabca5ad6ce74ea0148051131c03cb439 Mon Sep 17 00:00:00 2001 From: bin liu Date: Sat, 10 Feb 2018 19:27:14 +0800 Subject: [PATCH 01/20] Fix typos in builder and client Signed-off-by: bin liu Upstream-commit: ce0993270436c16a17e0afb08b9130556346b983 Component: engine --- components/engine/builder/dockerfile/instructions/commands.go | 4 ++-- components/engine/client/client_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/components/engine/builder/dockerfile/instructions/commands.go b/components/engine/builder/dockerfile/instructions/commands.go index ef435b27b3..d4f55ceb43 100644 --- a/components/engine/builder/dockerfile/instructions/commands.go +++ b/components/engine/builder/dockerfile/instructions/commands.go @@ -9,7 +9,7 @@ import ( "github.com/docker/docker/api/types/strslice" ) -// KeyValuePair represent an arbitrary named value (useful in slice insted of map[string] string to preserve ordering) +// KeyValuePair represent an arbitrary named value (useful in slice instead of map[string] string to preserve ordering) type KeyValuePair struct { Key string Value string @@ -195,7 +195,7 @@ func (c *WorkdirCommand) Expand(expander SingleWordExpander) error { return nil } -// ShellDependantCmdLine represents a cmdline optionaly prepended with the shell +// ShellDependantCmdLine represents a cmdline optionally prepended with the shell type ShellDependantCmdLine struct { CmdLine strslice.StrSlice PrependShell bool diff --git a/components/engine/client/client_test.go b/components/engine/client/client_test.go index 59b08bd4c3..69a0460855 100644 --- a/components/engine/client/client_test.go +++ b/components/engine/client/client_test.go @@ -287,7 +287,7 @@ func TestNegotiateAPIVersion(t *testing.T) { } // TestNegotiateAPIVersionOverride asserts that we honor -// the environment variable DOCKER_API_VERSION when negotianing versions +// the environment variable DOCKER_API_VERSION when negotiating versions func TestNegotiateAPVersionOverride(t *testing.T) { env := envToMap() defer mapToEnv(env) From 692e4bf089c1381c8b5d1c0aa4d14668bf1f1b5a Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Fri, 16 Feb 2018 13:50:57 -0800 Subject: [PATCH 02/20] builder: fix layer lifecycle leak Signed-off-by: Tonis Tiigi Upstream-commit: 7ad41d53df94c4277574d14809211b42dca2becc Component: engine --- components/engine/builder/builder.go | 16 +- components/engine/builder/dockerfile/copy.go | 20 ++- .../builder/dockerfile/dispatchers_test.go | 6 +- .../engine/builder/dockerfile/imagecontext.go | 31 +--- .../engine/builder/dockerfile/internals.go | 26 +-- .../builder/dockerfile/mockbackend_test.go | 27 ++- components/engine/daemon/build.go | 156 +++++++++--------- .../engine/integration/build/build_test.go | 40 +++++ 8 files changed, 186 insertions(+), 136 deletions(-) diff --git a/components/engine/builder/builder.go b/components/engine/builder/builder.go index 95320c9a6c..6da0e49e10 100644 --- a/components/engine/builder/builder.go +++ b/components/engine/builder/builder.go @@ -53,7 +53,7 @@ type Backend interface { // ImageBackend are the interface methods required from an image component type ImageBackend interface { - GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (Image, ReleaseableLayer, error) + GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (Image, ROLayer, error) } // ExecBackend contains the interface methods required for executing containers @@ -100,10 +100,16 @@ type Image interface { OperatingSystem() string } -// ReleaseableLayer is an image layer that can be mounted and released -type ReleaseableLayer interface { +// ROLayer is a reference to image rootfs layer +type ROLayer interface { Release() error - Mount() (containerfs.ContainerFS, error) - Commit() (ReleaseableLayer, error) + NewRWLayer() (RWLayer, error) DiffID() layer.DiffID } + +// RWLayer is active layer that can be read/modified +type RWLayer interface { + Release() error + Root() containerfs.ContainerFS + Commit() (ROLayer, error) +} diff --git a/components/engine/builder/dockerfile/copy.go b/components/engine/builder/dockerfile/copy.go index f0b27b4c68..cb9f24d205 100644 --- a/components/engine/builder/dockerfile/copy.go +++ b/components/engine/builder/dockerfile/copy.go @@ -72,8 +72,12 @@ type copier struct { source builder.Source pathCache pathCache download sourceDownloader - tmpPaths []string platform string + // for cleanup. TODO: having copier.cleanup() is error prone and hard to + // follow. Code calling performCopy should manage the lifecycle of its params. + // Copier should take override source as input, not imageMount. + activeLayer builder.RWLayer + tmpPaths []string } func copierFromDispatchRequest(req dispatchRequest, download sourceDownloader, imageSource *imageMount) copier { @@ -155,6 +159,10 @@ func (o *copier) Cleanup() { os.RemoveAll(path) } o.tmpPaths = []string{} + if o.activeLayer != nil { + o.activeLayer.Release() + o.activeLayer = nil + } } // TODO: allowWildcards can probably be removed by refactoring this function further. @@ -166,9 +174,15 @@ func (o *copier) calcCopyInfo(origPath string, allowWildcards bool) ([]copyInfo, // done on image Source? if imageSource != nil { var err error - o.source, err = imageSource.Source() + rwLayer, err := imageSource.NewRWLayer() if err != nil { - return nil, errors.Wrapf(err, "failed to copy from %s", imageSource.ImageID()) + return nil, err + } + o.activeLayer = rwLayer + + o.source, err = remotecontext.NewLazySource(rwLayer.Root()) + if err != nil { + return nil, errors.Wrapf(err, "failed to create context for copy from %s", rwLayer.Root().Path()) } } diff --git a/components/engine/builder/dockerfile/dispatchers_test.go b/components/engine/builder/dockerfile/dispatchers_test.go index a96d579905..6d52e7e619 100644 --- a/components/engine/builder/dockerfile/dispatchers_test.go +++ b/components/engine/builder/dockerfile/dispatchers_test.go @@ -127,7 +127,7 @@ func TestFromScratch(t *testing.T) { func TestFromWithArg(t *testing.T) { tag, expected := ":sometag", "expectedthisid" - getImage := func(name string) (builder.Image, builder.ReleaseableLayer, error) { + getImage := func(name string) (builder.Image, builder.ROLayer, error) { assert.Equal(t, "alpine"+tag, name) return &mockImage{id: "expectedthisid"}, nil, nil } @@ -159,7 +159,7 @@ func TestFromWithArg(t *testing.T) { func TestFromWithUndefinedArg(t *testing.T) { tag, expected := "sometag", "expectedthisid" - getImage := func(name string) (builder.Image, builder.ReleaseableLayer, error) { + getImage := func(name string) (builder.Image, builder.ROLayer, error) { assert.Equal(t, "alpine", name) return &mockImage{id: "expectedthisid"}, nil, nil } @@ -433,7 +433,7 @@ func TestRunWithBuildArgs(t *testing.T) { return imageCache } b.imageProber = newImageProber(mockBackend, nil, false) - mockBackend.getImageFunc = func(_ string) (builder.Image, builder.ReleaseableLayer, error) { + mockBackend.getImageFunc = func(_ string) (builder.Image, builder.ROLayer, error) { return &mockImage{ id: "abcdef", config: &container.Config{Cmd: origCmd}, diff --git a/components/engine/builder/dockerfile/imagecontext.go b/components/engine/builder/dockerfile/imagecontext.go index f83528e104..0d4af384af 100644 --- a/components/engine/builder/dockerfile/imagecontext.go +++ b/components/engine/builder/dockerfile/imagecontext.go @@ -5,7 +5,6 @@ import ( "github.com/docker/docker/api/types/backend" "github.com/docker/docker/builder" - "github.com/docker/docker/builder/remotecontext" dockerimage "github.com/docker/docker/image" "github.com/docker/docker/pkg/system" "github.com/pkg/errors" @@ -13,7 +12,7 @@ import ( "golang.org/x/net/context" ) -type getAndMountFunc func(string, bool) (builder.Image, builder.ReleaseableLayer, error) +type getAndMountFunc func(string, bool) (builder.Image, builder.ROLayer, error) // imageSources mounts images and provides a cache for mounted images. It tracks // all images so they can be unmounted at the end of the build. @@ -24,7 +23,7 @@ type imageSources struct { } func newImageSources(ctx context.Context, options builderOptions) *imageSources { - getAndMount := func(idOrRef string, localOnly bool) (builder.Image, builder.ReleaseableLayer, error) { + getAndMount := func(idOrRef string, localOnly bool) (builder.Image, builder.ROLayer, error) { pullOption := backend.PullOptionNoPull if !localOnly { if options.Options.PullParent { @@ -92,32 +91,14 @@ func (m *imageSources) Add(im *imageMount) { type imageMount struct { image builder.Image source builder.Source - layer builder.ReleaseableLayer + layer builder.ROLayer } -func newImageMount(image builder.Image, layer builder.ReleaseableLayer) *imageMount { +func newImageMount(image builder.Image, layer builder.ROLayer) *imageMount { im := &imageMount{image: image, layer: layer} return im } -func (im *imageMount) Source() (builder.Source, error) { - if im.source == nil { - if im.layer == nil { - return nil, errors.Errorf("empty context") - } - mountPath, err := im.layer.Mount() - if err != nil { - return nil, errors.Wrapf(err, "failed to mount %s", im.image.ImageID()) - } - source, err := remotecontext.NewLazySource(mountPath) - if err != nil { - return nil, errors.Wrapf(err, "failed to create lazycontext for %s", mountPath) - } - im.source = source - } - return im.source, nil -} - func (im *imageMount) unmount() error { if im.layer == nil { return nil @@ -133,8 +114,8 @@ func (im *imageMount) Image() builder.Image { return im.image } -func (im *imageMount) Layer() builder.ReleaseableLayer { - return im.layer +func (im *imageMount) NewRWLayer() (builder.RWLayer, error) { + return im.layer.NewRWLayer() } func (im *imageMount) ImageID() string { diff --git a/components/engine/builder/dockerfile/internals.go b/components/engine/builder/dockerfile/internals.go index 4b4f638b1f..c8b34f8f65 100644 --- a/components/engine/builder/dockerfile/internals.go +++ b/components/engine/builder/dockerfile/internals.go @@ -17,6 +17,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/backend" "github.com/docker/docker/api/types/container" + "github.com/docker/docker/builder" "github.com/docker/docker/image" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/chrootarchive" @@ -114,8 +115,8 @@ func (b *Builder) commitContainer(dispatchState *dispatchState, id string, conta return err } -func (b *Builder) exportImage(state *dispatchState, imageMount *imageMount, runConfig *container.Config) error { - newLayer, err := imageMount.Layer().Commit() +func (b *Builder) exportImage(state *dispatchState, layer builder.RWLayer, parent builder.Image, runConfig *container.Config) error { + newLayer, err := layer.Commit() if err != nil { return err } @@ -124,7 +125,7 @@ func (b *Builder) exportImage(state *dispatchState, imageMount *imageMount, runC // if there is an error before we can add the full mount with image b.imageSources.Add(newImageMount(nil, newLayer)) - parentImage, ok := imageMount.Image().(*image.Image) + parentImage, ok := parent.(*image.Image) if !ok { return errors.Errorf("unexpected image type") } @@ -177,7 +178,13 @@ func (b *Builder) performCopy(state *dispatchState, inst copyInstruction) error return errors.Wrapf(err, "failed to get destination image %q", state.imageID) } - destInfo, err := createDestInfo(state.runConfig.WorkingDir, inst, imageMount, b.options.Platform) + rwLayer, err := imageMount.NewRWLayer() + if err != nil { + return err + } + defer rwLayer.Release() + + destInfo, err := createDestInfo(state.runConfig.WorkingDir, inst, rwLayer, b.options.Platform) if err != nil { return err } @@ -203,10 +210,10 @@ func (b *Builder) performCopy(state *dispatchState, inst copyInstruction) error return errors.Wrapf(err, "failed to copy files") } } - return b.exportImage(state, imageMount, runConfigWithCommentCmd) + return b.exportImage(state, rwLayer, imageMount.Image(), runConfigWithCommentCmd) } -func createDestInfo(workingDir string, inst copyInstruction, imageMount *imageMount, platform string) (copyInfo, error) { +func createDestInfo(workingDir string, inst copyInstruction, rwLayer builder.RWLayer, platform string) (copyInfo, error) { // Twiddle the destination when it's a relative path - meaning, make it // relative to the WORKINGDIR dest, err := normalizeDest(workingDir, inst.dest, platform) @@ -214,12 +221,7 @@ func createDestInfo(workingDir string, inst copyInstruction, imageMount *imageMo return copyInfo{}, errors.Wrapf(err, "invalid %s", inst.cmdName) } - destMount, err := imageMount.Source() - if err != nil { - return copyInfo{}, errors.Wrapf(err, "failed to mount copy source") - } - - return newCopyInfoFromSource(destMount, dest, ""), nil + return copyInfo{root: rwLayer.Root(), path: dest}, nil } // normalizeDest normalises the destination of a COPY/ADD command in a diff --git a/components/engine/builder/dockerfile/mockbackend_test.go b/components/engine/builder/dockerfile/mockbackend_test.go index 06adb928fd..77bc8b5715 100644 --- a/components/engine/builder/dockerfile/mockbackend_test.go +++ b/components/engine/builder/dockerfile/mockbackend_test.go @@ -20,7 +20,7 @@ import ( type MockBackend struct { containerCreateFunc func(config types.ContainerCreateConfig) (container.ContainerCreateCreatedBody, error) commitFunc func(backend.CommitConfig) (image.ID, error) - getImageFunc func(string) (builder.Image, builder.ReleaseableLayer, error) + getImageFunc func(string) (builder.Image, builder.ROLayer, error) makeImageCacheFunc func(cacheFrom []string) builder.ImageCache } @@ -66,7 +66,7 @@ func (m *MockBackend) CopyOnBuild(containerID string, destPath string, srcRoot s return nil } -func (m *MockBackend) GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (builder.Image, builder.ReleaseableLayer, error) { +func (m *MockBackend) GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (builder.Image, builder.ROLayer, error) { if m.getImageFunc != nil { return m.getImageFunc(refOrID) } @@ -124,14 +124,25 @@ func (l *mockLayer) Release() error { return nil } -func (l *mockLayer) Mount() (containerfs.ContainerFS, error) { - return containerfs.NewLocalContainerFS("mountPath"), nil -} - -func (l *mockLayer) Commit() (builder.ReleaseableLayer, error) { - return nil, nil +func (l *mockLayer) NewRWLayer() (builder.RWLayer, error) { + return &mockRWLayer{}, nil } func (l *mockLayer) DiffID() layer.DiffID { return layer.DiffID("abcdef") } + +type mockRWLayer struct { +} + +func (l *mockRWLayer) Release() error { + return nil +} + +func (l *mockRWLayer) Commit() (builder.ROLayer, error) { + return nil, nil +} + +func (l *mockRWLayer) Root() containerfs.ContainerFS { + return nil +} diff --git a/components/engine/daemon/build.go b/components/engine/daemon/build.go index 0384c13f32..d1396fb50e 100644 --- a/components/engine/daemon/build.go +++ b/components/engine/daemon/build.go @@ -15,130 +15,126 @@ import ( "github.com/docker/docker/pkg/system" "github.com/docker/docker/registry" "github.com/pkg/errors" - "github.com/sirupsen/logrus" "golang.org/x/net/context" ) -type releaseableLayer struct { +type roLayer struct { released bool layerStore layer.Store roLayer layer.Layer - rwLayer layer.RWLayer } -func (rl *releaseableLayer) Mount() (containerfs.ContainerFS, error) { - var err error - var mountPath containerfs.ContainerFS +func (l *roLayer) DiffID() layer.DiffID { + if l.roLayer == nil { + return layer.DigestSHA256EmptyTar + } + return l.roLayer.DiffID() +} + +func (l *roLayer) Release() error { + if l.released { + return nil + } + if l.roLayer != nil { + metadata, err := l.layerStore.Release(l.roLayer) + layer.LogReleaseMetadata(metadata) + if err != nil { + return errors.Wrap(err, "failed to release ROLayer") + } + } + l.roLayer = nil + l.released = true + return nil +} + +func (l *roLayer) NewRWLayer() (builder.RWLayer, error) { var chainID layer.ChainID - if rl.roLayer != nil { - chainID = rl.roLayer.ChainID() + if l.roLayer != nil { + chainID = l.roLayer.ChainID() } mountID := stringid.GenerateRandomID() - rl.rwLayer, err = rl.layerStore.CreateRWLayer(mountID, chainID, nil) + newLayer, err := l.layerStore.CreateRWLayer(mountID, chainID, nil) if err != nil { return nil, errors.Wrap(err, "failed to create rwlayer") } - mountPath, err = rl.rwLayer.Mount("") + rwLayer := &rwLayer{layerStore: l.layerStore, rwLayer: newLayer} + + fs, err := newLayer.Mount("") if err != nil { - // Clean up the layer if we fail to mount it here. - metadata, err := rl.layerStore.ReleaseRWLayer(rl.rwLayer) - layer.LogReleaseMetadata(metadata) - if err != nil { - logrus.Errorf("Failed to release RWLayer: %s", err) - } - rl.rwLayer = nil + rwLayer.Release() return nil, err } - return mountPath, nil + rwLayer.fs = fs + + return rwLayer, nil } -func (rl *releaseableLayer) Commit() (builder.ReleaseableLayer, error) { - var chainID layer.ChainID - if rl.roLayer != nil { - chainID = rl.roLayer.ChainID() - } +type rwLayer struct { + released bool + layerStore layer.Store + rwLayer layer.RWLayer + fs containerfs.ContainerFS +} - stream, err := rl.rwLayer.TarStream() +func (l *rwLayer) Root() containerfs.ContainerFS { + return l.fs +} + +func (l *rwLayer) Commit() (builder.ROLayer, error) { + stream, err := l.rwLayer.TarStream() if err != nil { return nil, err } defer stream.Close() - newLayer, err := rl.layerStore.Register(stream, chainID) + var chainID layer.ChainID + if parent := l.rwLayer.Parent(); parent != nil { + chainID = parent.ChainID() + } + + newLayer, err := l.layerStore.Register(stream, chainID) if err != nil { return nil, err } // TODO: An optimization would be to handle empty layers before returning - return &releaseableLayer{layerStore: rl.layerStore, roLayer: newLayer}, nil + return &roLayer{layerStore: l.layerStore, roLayer: newLayer}, nil } -func (rl *releaseableLayer) DiffID() layer.DiffID { - if rl.roLayer == nil { - return layer.DigestSHA256EmptyTar - } - return rl.roLayer.DiffID() -} - -func (rl *releaseableLayer) Release() error { - if rl.released { +func (l *rwLayer) Release() error { + if l.released { return nil } - if err := rl.releaseRWLayer(); err != nil { - // Best effort attempt at releasing read-only layer before returning original error. - rl.releaseROLayer() - return err + + if l.fs != nil { + if err := l.rwLayer.Unmount(); err != nil { + return errors.Wrap(err, "failed to unmount RWLayer") + } + l.fs = nil } - if err := rl.releaseROLayer(); err != nil { - return err + + metadata, err := l.layerStore.ReleaseRWLayer(l.rwLayer) + layer.LogReleaseMetadata(metadata) + if err != nil { + return errors.Wrap(err, "failed to release RWLayer") } - rl.released = true + l.released = true return nil } -func (rl *releaseableLayer) releaseRWLayer() error { - if rl.rwLayer == nil { - return nil - } - if err := rl.rwLayer.Unmount(); err != nil { - logrus.Errorf("Failed to unmount RWLayer: %s", err) - return err - } - metadata, err := rl.layerStore.ReleaseRWLayer(rl.rwLayer) - layer.LogReleaseMetadata(metadata) - if err != nil { - logrus.Errorf("Failed to release RWLayer: %s", err) - } - rl.rwLayer = nil - return err -} - -func (rl *releaseableLayer) releaseROLayer() error { - if rl.roLayer == nil { - return nil - } - metadata, err := rl.layerStore.Release(rl.roLayer) - layer.LogReleaseMetadata(metadata) - if err != nil { - logrus.Errorf("Failed to release ROLayer: %s", err) - } - rl.roLayer = nil - return err -} - -func newReleasableLayerForImage(img *image.Image, layerStore layer.Store) (builder.ReleaseableLayer, error) { +func newROLayerForImage(img *image.Image, layerStore layer.Store) (builder.ROLayer, error) { if img == nil || img.RootFS.ChainID() == "" { - return &releaseableLayer{layerStore: layerStore}, nil + return &roLayer{layerStore: layerStore}, nil } // Hold a reference to the image layer so that it can't be removed before // it is released - roLayer, err := layerStore.Get(img.RootFS.ChainID()) + layer, err := layerStore.Get(img.RootFS.ChainID()) if err != nil { return nil, errors.Wrapf(err, "failed to get layer for image %s", img.ImageID()) } - return &releaseableLayer{layerStore: layerStore, roLayer: roLayer}, nil + return &roLayer{layerStore: layerStore, roLayer: layer}, nil } // TODO: could this use the regular daemon PullImage ? @@ -170,12 +166,12 @@ func (daemon *Daemon) pullForBuilder(ctx context.Context, name string, authConfi // GetImageAndReleasableLayer returns an image and releaseable layer for a reference or ID. // Every call to GetImageAndReleasableLayer MUST call releasableLayer.Release() to prevent // leaking of layers. -func (daemon *Daemon) GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (builder.Image, builder.ReleaseableLayer, error) { +func (daemon *Daemon) GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (builder.Image, builder.ROLayer, error) { if refOrID == "" { if !system.IsOSSupported(opts.OS) { return nil, nil, system.ErrNotSupportedOperatingSystem } - layer, err := newReleasableLayerForImage(nil, daemon.layerStores[opts.OS]) + layer, err := newROLayerForImage(nil, daemon.layerStores[opts.OS]) return nil, layer, err } @@ -189,7 +185,7 @@ func (daemon *Daemon) GetImageAndReleasableLayer(ctx context.Context, refOrID st if !system.IsOSSupported(image.OperatingSystem()) { return nil, nil, system.ErrNotSupportedOperatingSystem } - layer, err := newReleasableLayerForImage(image, daemon.layerStores[image.OperatingSystem()]) + layer, err := newROLayerForImage(image, daemon.layerStores[image.OperatingSystem()]) return image, layer, err } } @@ -201,7 +197,7 @@ func (daemon *Daemon) GetImageAndReleasableLayer(ctx context.Context, refOrID st if !system.IsOSSupported(image.OperatingSystem()) { return nil, nil, system.ErrNotSupportedOperatingSystem } - layer, err := newReleasableLayerForImage(image, daemon.layerStores[image.OperatingSystem()]) + layer, err := newROLayerForImage(image, daemon.layerStores[image.OperatingSystem()]) return image, layer, err } diff --git a/components/engine/integration/build/build_test.go b/components/engine/integration/build/build_test.go index 3f7b1dcf34..124f1107fb 100644 --- a/components/engine/integration/build/build_test.go +++ b/components/engine/integration/build/build_test.go @@ -301,6 +301,46 @@ COPY bar /` require.NotContains(t, out.String(), "Using cache") } +// docker/for-linux#135 +// #35641 +func TestBuildMultiStageLayerLeak(t *testing.T) { + ctx := context.TODO() + defer setupTest(t)() + + // all commands need to match until COPY + dockerfile := `FROM busybox +WORKDIR /foo +COPY foo . +FROM busybox +WORKDIR /foo +COPY bar . +RUN [ -f bar ] +RUN [ ! -f foo ] +` + + source := fakecontext.New(t, "", + fakecontext.WithFile("foo", "0"), + fakecontext.WithFile("bar", "1"), + fakecontext.WithDockerfile(dockerfile)) + defer source.Close() + + apiclient := testEnv.APIClient() + resp, err := apiclient.ImageBuild(ctx, + source.AsTarReader(t), + types.ImageBuildOptions{ + Remove: true, + ForceRemove: true, + }) + + out := bytes.NewBuffer(nil) + require.NoError(t, err) + _, err = io.Copy(out, resp.Body) + resp.Body.Close() + require.NoError(t, err) + + assert.Contains(t, out.String(), "Successfully built") +} + func writeTarRecord(t *testing.T, w *tar.Writer, fn, contents string) { err := w.WriteHeader(&tar.Header{ Name: fn, From 94dbb42ee9bd3cd0c808f175089d1312c79da408 Mon Sep 17 00:00:00 2001 From: junzhe and mnussbaum Date: Sat, 10 Feb 2018 00:03:47 +0000 Subject: [PATCH 03/20] Fix empty LogPath with non-blocking logging mode This fixes an issue where the container LogPath was empty when the non-blocking logging mode was enabled. This change sets the LogPath on the container as soon as the path is generated, instead of setting the LogPath on a logger struct and then attempting to pull it off that logger at a later point. That attempt to pull the LogPath off the logger was error prone since it assumed that the logger would only ever be a single type. Prior to this change docker inspect returned an empty string for LogPath. This caused issues with tools that rely on docker inspect output to discover container logs, e.g. Kubernetes. This commit also removes some LogPath methods that are now unnecessary and are never invoked. Signed-off-by: junzhe and mnussbaum Upstream-commit: 20ca612a59c45c0bd58c71c199a7ebd2a6bf1a9e Component: engine --- components/engine/container/container.go | 9 ++-- .../engine/container/container_unit_test.go | 53 +++++++++++++++++++ .../daemon/logger/jsonfilelog/jsonfilelog.go | 5 -- .../daemon/logger/loggerutils/logfile.go | 7 --- 4 files changed, 56 insertions(+), 18 deletions(-) diff --git a/components/engine/container/container.go b/components/engine/container/container.go index 938d66f889..451e0f833e 100644 --- a/components/engine/container/container.go +++ b/components/engine/container/container.go @@ -38,7 +38,7 @@ import ( "github.com/docker/docker/runconfig" "github.com/docker/docker/volume" "github.com/docker/go-connections/nat" - "github.com/docker/go-units" + units "github.com/docker/go-units" "github.com/docker/libnetwork" "github.com/docker/libnetwork/netlabel" "github.com/docker/libnetwork/options" @@ -391,6 +391,8 @@ func (container *Container) StartLogger() (logger.Logger, error) { if err != nil { return nil, err } + + container.LogPath = info.LogPath } l, err := initDriver(info) @@ -974,11 +976,6 @@ func (container *Container) startLogging() error { copier.Run() container.LogDriver = l - // set LogPath field only for json-file logdriver - if jl, ok := l.(*jsonfilelog.JSONFileLogger); ok { - container.LogPath = jl.LogPath() - } - return nil } diff --git a/components/engine/container/container_unit_test.go b/components/engine/container/container_unit_test.go index f6ded8fa9d..858124abdb 100644 --- a/components/engine/container/container_unit_test.go +++ b/components/engine/container/container_unit_test.go @@ -1,12 +1,16 @@ package container // import "github.com/docker/docker/container" import ( + "fmt" + "io/ioutil" "path/filepath" "testing" "github.com/docker/docker/api/types/container" swarmtypes "github.com/docker/docker/api/types/swarm" + "github.com/docker/docker/daemon/logger/jsonfilelog" "github.com/docker/docker/pkg/signal" + "github.com/stretchr/testify/require" ) func TestContainerStopSignal(t *testing.T) { @@ -66,3 +70,52 @@ func TestContainerSecretReferenceDestTarget(t *testing.T) { t.Fatalf("expected secret dest %q; received %q", expected, d) } } + +func TestContainerLogPathSetForJSONFileLogger(t *testing.T) { + containerRoot, err := ioutil.TempDir("", "TestContainerLogPathSetForJSONFileLogger") + require.NoError(t, err) + + c := &Container{ + Config: &container.Config{}, + HostConfig: &container.HostConfig{ + LogConfig: container.LogConfig{ + Type: jsonfilelog.Name, + }, + }, + ID: "TestContainerLogPathSetForJSONFileLogger", + Root: containerRoot, + } + + _, err = c.StartLogger() + require.NoError(t, err) + + expectedLogPath, err := filepath.Abs(filepath.Join(containerRoot, fmt.Sprintf("%s-json.log", c.ID))) + require.NoError(t, err) + require.Equal(t, c.LogPath, expectedLogPath) +} + +func TestContainerLogPathSetForRingLogger(t *testing.T) { + containerRoot, err := ioutil.TempDir("", "TestContainerLogPathSetForRingLogger") + require.NoError(t, err) + + c := &Container{ + Config: &container.Config{}, + HostConfig: &container.HostConfig{ + LogConfig: container.LogConfig{ + Type: jsonfilelog.Name, + Config: map[string]string{ + "mode": string(container.LogModeNonBlock), + }, + }, + }, + ID: "TestContainerLogPathSetForRingLogger", + Root: containerRoot, + } + + _, err = c.StartLogger() + require.NoError(t, err) + + expectedLogPath, err := filepath.Abs(filepath.Join(containerRoot, fmt.Sprintf("%s-json.log", c.ID))) + require.NoError(t, err) + require.Equal(t, c.LogPath, expectedLogPath) +} diff --git a/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go b/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go index 92484697d5..7d637f8573 100644 --- a/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go +++ b/components/engine/daemon/logger/jsonfilelog/jsonfilelog.go @@ -150,11 +150,6 @@ func ValidateLogOpt(cfg map[string]string) error { return nil } -// LogPath returns the location the given json logger logs to. -func (l *JSONFileLogger) LogPath() string { - return l.writer.LogPath() -} - // Close closes underlying file and signals all readers to stop. func (l *JSONFileLogger) Close() error { l.mu.Lock() diff --git a/components/engine/daemon/logger/loggerutils/logfile.go b/components/engine/daemon/logger/loggerutils/logfile.go index ff7a3323f9..8c6cbfb42b 100644 --- a/components/engine/daemon/logger/loggerutils/logfile.go +++ b/components/engine/daemon/logger/loggerutils/logfile.go @@ -130,13 +130,6 @@ func rotate(name string, maxFiles int) error { return nil } -// LogPath returns the location the given writer logs to. -func (w *LogFile) LogPath() string { - w.mu.Lock() - defer w.mu.Unlock() - return w.f.Name() -} - // MaxFiles return maximum number of files func (w *LogFile) MaxFiles() int { return w.maxFiles From 55a9ea06ee39ea82350e64f76b45dfca3b306e3b Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 22 Feb 2018 03:25:10 -0800 Subject: [PATCH 04/20] Migrate some config secret tests to api test This fix migrates some secret create tests to api tests, and remove redundant TestConfigCreate. Signed-off-by: Yong Tang Upstream-commit: 99e28188507bbcb925b0c09df6b53cdd882d24c5 Component: engine --- .../docker_cli_secret_create_test.go | 39 -------------- .../engine/integration/secret/secret_test.go | 54 +++++++++---------- 2 files changed, 24 insertions(+), 69 deletions(-) diff --git a/components/engine/integration-cli/docker_cli_secret_create_test.go b/components/engine/integration-cli/docker_cli_secret_create_test.go index 839c3922ac..a807e4e7e7 100644 --- a/components/engine/integration-cli/docker_cli_secret_create_test.go +++ b/components/engine/integration-cli/docker_cli_secret_create_test.go @@ -12,45 +12,6 @@ import ( "github.com/go-check/check" ) -func (s *DockerSwarmSuite) TestSecretCreate(c *check.C) { - d := s.AddDaemon(c, true, true) - - testName := "test_secret" - id := d.CreateSecret(c, swarm.SecretSpec{ - Annotations: swarm.Annotations{ - Name: testName, - }, - Data: []byte("TESTINGDATA"), - }) - c.Assert(id, checker.Not(checker.Equals), "", check.Commentf("secrets: %s", id)) - - secret := d.GetSecret(c, id) - c.Assert(secret.Spec.Name, checker.Equals, testName) -} - -func (s *DockerSwarmSuite) TestSecretCreateWithLabels(c *check.C) { - d := s.AddDaemon(c, true, true) - - testName := "test_secret" - id := d.CreateSecret(c, swarm.SecretSpec{ - Annotations: swarm.Annotations{ - Name: testName, - Labels: map[string]string{ - "key1": "value1", - "key2": "value2", - }, - }, - Data: []byte("TESTINGDATA"), - }) - c.Assert(id, checker.Not(checker.Equals), "", check.Commentf("secrets: %s", id)) - - secret := d.GetSecret(c, id) - c.Assert(secret.Spec.Name, checker.Equals, testName) - c.Assert(len(secret.Spec.Labels), checker.Equals, 2) - c.Assert(secret.Spec.Labels["key1"], checker.Equals, "value1") - c.Assert(secret.Spec.Labels["key2"], checker.Equals, "value2") -} - // Test case for 28884 func (s *DockerSwarmSuite) TestSecretCreateResolve(c *check.C) { d := s.AddDaemon(c, true, true) diff --git a/components/engine/integration/secret/secret_test.go b/components/engine/integration/secret/secret_test.go index 3b5e66a5bf..8a292f005b 100644 --- a/components/engine/integration/secret/secret_test.go +++ b/components/engine/integration/secret/secret_test.go @@ -129,32 +129,7 @@ func createSecret(ctx context.Context, t *testing.T, client client.APIClient, na return secret.ID } -func TestSecretsCreate(t *testing.T) { - skip.If(t, testEnv.DaemonInfo.OSType != "linux") - - defer setupTest(t)() - d := swarm.NewSwarm(t, testEnv) - defer d.Stop(t) - client, err := client.NewClientWithOpts(client.WithHost((d.Sock()))) - require.NoError(t, err) - - ctx := context.Background() - - testName := "test_secret" - createSecret(ctx, t, client, testName, []byte("TESTINGDATA"), nil) - require.NoError(t, err) - - // create an already existin secret, daemon should return a status code of 409 - _, err = client.SecretCreate(ctx, swarmtypes.SecretSpec{ - Annotations: swarmtypes.Annotations{ - Name: testName, - }, - Data: []byte("TESTINGDATA"), - }) - testutil.ErrorContains(t, err, "already exists") -} - -func TestSecretsDelete(t *testing.T) { +func TestSecretsCreateAndDelete(t *testing.T) { skip.If(t, testEnv.DaemonInfo.OSType != "linux") defer setupTest(t)() @@ -167,12 +142,17 @@ func TestSecretsDelete(t *testing.T) { testName := "test_secret" secretID := createSecret(ctx, t, client, testName, []byte("TESTINGDATA"), nil) - require.NoError(t, err) - insp, _, err := client.SecretInspectWithRaw(ctx, secretID) - require.NoError(t, err) - assert.Equal(t, insp.ID, secretID) + // create an already existin secret, daemon should return a status code of 409 + _, err = client.SecretCreate(ctx, swarmtypes.SecretSpec{ + Annotations: swarmtypes.Annotations{ + Name: testName, + }, + Data: []byte("TESTINGDATA"), + }) + testutil.ErrorContains(t, err, "already exists") + // Ported from original TestSecretsDelete err = client.SecretRemove(ctx, secretID) require.NoError(t, err) @@ -181,6 +161,20 @@ func TestSecretsDelete(t *testing.T) { err = client.SecretRemove(ctx, "non-existin") testutil.ErrorContains(t, err, "No such secret: non-existin") + + // Ported from original TestSecretsCreteaWithLabels + testName = "test_secret_with_labels" + secretID = createSecret(ctx, t, client, testName, []byte("TESTINGDATA"), map[string]string{ + "key1": "value1", + "key2": "value2", + }) + + insp, _, err := client.SecretInspectWithRaw(ctx, secretID) + require.NoError(t, err) + assert.Equal(t, insp.Spec.Name, testName) + assert.Equal(t, len(insp.Spec.Labels), 2) + assert.Equal(t, insp.Spec.Labels["key1"], "value1") + assert.Equal(t, insp.Spec.Labels["key2"], "value2") } func TestSecretsUpdate(t *testing.T) { From e4ccd4c846b6e85d2543ad6649dff52d3e303c70 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Fri, 29 Sep 2017 17:09:14 -0400 Subject: [PATCH 05/20] Updates the Dockerfile to use multi-stage Signed-off-by: Brian Goff Signed-off-by: Vincent Demeester Upstream-commit: d539038d9712daa44254412c6fcc6d78a14da691 Component: engine --- components/engine/Dockerfile | 247 +++++++++++++++++++++-------------- 1 file changed, 152 insertions(+), 95 deletions(-) diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile index fab79570c4..49a8c59967 100644 --- a/components/engine/Dockerfile +++ b/components/engine/Dockerfile @@ -32,67 +32,13 @@ # the case. Therefore, you don't have to disable it anymore. # -FROM debian:stretch - +FROM buildpack-deps:stretch AS base # allow replacing httpredir or deb mirror ARG APT_MIRROR=deb.debian.org RUN sed -ri "s/(httpredir|deb).debian.org/$APT_MIRROR/g" /etc/apt/sources.list -# Packaged dependencies -RUN apt-get update && apt-get install -y \ - apparmor \ - apt-utils \ - aufs-tools \ - automake \ - bash-completion \ - binutils-mingw-w64 \ - bsdmainutils \ - btrfs-tools \ - build-essential \ - cmake \ - createrepo \ - curl \ - dpkg-sig \ - gcc-mingw-w64 \ - git \ - iptables \ - jq \ - less \ - libapparmor-dev \ - libcap-dev \ - libdevmapper-dev \ - libnet-dev \ - libnl-3-dev \ - libprotobuf-c0-dev \ - libprotobuf-dev \ - libseccomp-dev \ - libsystemd-dev \ - libtool \ - libudev-dev \ - mercurial \ - net-tools \ - pigz \ - pkg-config \ - protobuf-compiler \ - protobuf-c-compiler \ - python-backports.ssl-match-hostname \ - python-dev \ - python-mock \ - python-pip \ - python-requests \ - python-setuptools \ - python-websocket \ - python-wheel \ - tar \ - thin-provisioning-tools \ - vim \ - vim-common \ - xfsprogs \ - zip \ - --no-install-recommends \ - && pip install awscli==1.10.15 -# Install Go +FROM base AS golang # IMPORTANT: If the version of Go is updated, the Windows to Linux CI machines # will need updating, to avoid errors. Ping #docker-maintainers on IRC # with a heads-up. @@ -100,18 +46,30 @@ RUN apt-get update && apt-get install -y \ ENV GO_VERSION 1.9.4 RUN curl -fsSL "https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz" \ | tar -xzC /usr/local +ENV PATH=/usr/local/go/bin:/go/bin:$PATH GOPATH=/go -ENV PATH /go/bin:/usr/local/go/bin:$PATH -ENV GOPATH /go +FROM base AS criu # Install CRIU for checkpoint/restore support ENV CRIU_VERSION 3.6 -RUN mkdir -p /usr/src/criu \ +# Install dependancy packages specific to criu +RUN apt-get update && apt-get install -y \ + libnet-dev \ + libprotobuf-c0-dev \ + libprotobuf-dev \ + libnl-3-dev \ + libcap-dev \ + protobuf-compiler \ + protobuf-c-compiler \ + python-protobuf \ + && mkdir -p /usr/src/criu \ && curl -sSL https://github.com/checkpoint-restore/criu/archive/v${CRIU_VERSION}.tar.gz | tar -C /usr/src/criu/ -xz --strip-components=1 \ && cd /usr/src/criu \ && make \ - && make install-criu + && make PREFIX=/opt/criu install-criu + +FROM golang AS registry # Install two versions of the registry. The first is an older version that # only supports schema1 manifests. The second is a newer version that supports # both. This allows integration-cli tests to cover push/pull with both schema1 @@ -129,6 +87,9 @@ RUN set -x \ go build -buildmode=pie -o /usr/local/bin/registry-v2-schema1 github.com/docker/distribution/cmd/registry \ && rm -rf "$GOPATH" + + +FROM golang AS notary # Install notary and notary-server ENV NOTARY_VERSION v0.5.0 RUN set -x \ @@ -141,18 +102,18 @@ RUN set -x \ go build -buildmode=pie -o /usr/local/bin/notary github.com/docker/notary/cmd/notary \ && rm -rf "$GOPATH" + + +FROM base AS docker-py # Get the "docker-py" source so we can run their integration tests ENV DOCKER_PY_COMMIT 8b246db271a85d6541dc458838627e89c683e42f -# To run integration tests docker-pycreds is required. RUN git clone https://github.com/docker/docker-py.git /docker-py \ && cd /docker-py \ - && git checkout -q $DOCKER_PY_COMMIT \ - && pip install docker-pycreds==0.2.1 \ - && pip install -r test-requirements.txt + && git checkout -q $DOCKER_PY_COMMIT -# Install yamllint for validating swagger.yaml -RUN pip install yamllint==1.5.0 + +FROM golang AS swagger # Install go-swagger for validating swagger.yaml ENV GO_SWAGGER_COMMIT c28258affb0b6251755d92489ef685af8d4ff3eb RUN set -x \ @@ -162,49 +123,145 @@ RUN set -x \ && go build -o /usr/local/bin/swagger github.com/go-swagger/go-swagger/cmd/swagger \ && rm -rf "$GOPATH" -# Set user.email so crosbymichael's in-container merge commits go smoothly -RUN git config --global user.email 'docker-dummy@example.com' - -# Add an unprivileged user to be used for tests which need it -RUN groupadd -r docker -RUN useradd --create-home --gid docker unprivilegeduser - -VOLUME /var/lib/docker -WORKDIR /go/src/github.com/docker/docker -ENV DOCKER_BUILDTAGS apparmor seccomp selinux - -# Let us use a .bashrc file -RUN ln -sfv $PWD/.bashrc ~/.bashrc -# Add integration helps to bashrc -RUN echo "source $PWD/hack/make/.integration-test-helpers" >> /etc/bash.bashrc +FROM golang AS frozen-images +RUN apt-get update && apt-get install -y jq ca-certificates --no-install-recommends # Get useful and necessary Hub images so we can "docker load" locally instead of pulling -COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/ -RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ +COPY contrib/download-frozen-image-v2.sh / +RUN /download-frozen-image-v2.sh /docker-frozen-images \ buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \ debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) -# Install tomlv, vndr, runc, containerd, tini, proxy dockercli -# Please edit hack/dockerfile/install/.installer to update them. -COPY hack/dockerfile/install hack/dockerfile/install -RUN for i in tomlv vndr tini gometalinter proxy dockercli runc containerd; \ - do hack/dockerfile/install/install.sh $i; \ - done -ENV PATH=/usr/local/cli:$PATH +# Just a little hack so we don't have to install these deps twice, once for runc and once for dockerd +FROM golang AS runtime-dev +RUN apt-get update && apt-get install -y \ + libapparmor-dev \ + libseccomp-dev + +FROM golang AS tomlv +ENV INSTALL_BINARY_NAME=tomlv +COPY hack/dockerfile/install/install.sh ./install.sh +COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./ +RUN PREFIX=/opt/$INSTALL_BINARY_NAME ./install.sh $INSTALL_BINARY_NAME + +FROM golang AS vndr +ENV INSTALL_BINARY_NAME=vndr +COPY hack/dockerfile/install/install.sh ./install.sh +COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./ +RUN PREFIX=/opt/$INSTALL_BINARY_NAME ./install.sh $INSTALL_BINARY_NAME + +FROM golang AS containerd +RUN apt-get update && apt-get install -y btrfs-tools +ENV INSTALL_BINARY_NAME=containerd +COPY hack/dockerfile/install/install.sh ./install.sh +COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./ +RUN PREFIX=/opt/$INSTALL_BINARY_NAME ./install.sh $INSTALL_BINARY_NAME + +FROM golang AS proxy +ENV INSTALL_BINARY_NAME=proxy +COPY hack/dockerfile/install/install.sh ./install.sh +COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./ +RUN PREFIX=/opt/$INSTALL_BINARY_NAME ./install.sh $INSTALL_BINARY_NAME + +FROM golang AS gometalinter +ENV INSTALL_BINARY_NAME=gometalinter +COPY hack/dockerfile/install/install.sh ./install.sh +COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./ +RUN PREFIX=/opt/$INSTALL_BINARY_NAME ./install.sh $INSTALL_BINARY_NAME + +FROM golang AS dockercli +ENV INSTALL_BINARY_NAME=dockercli +COPY hack/dockerfile/install/install.sh ./install.sh +COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./ +RUN PREFIX=/opt/$INSTALL_BINARY_NAME ./install.sh $INSTALL_BINARY_NAME + +FROM runtime-dev AS runc +ENV INSTALL_BINARY_NAME=runc +COPY hack/dockerfile/install/install.sh ./install.sh +COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./ +RUN PREFIX=/opt/$INSTALL_BINARY_NAME ./install.sh $INSTALL_BINARY_NAME + +FROM base AS tini +RUN apt-get update && apt-get install -y cmake vim-common +COPY hack/dockerfile/install/install.sh ./install.sh +ENV INSTALL_BINARY_NAME=tini +COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./ +RUN PREFIX=/opt/$INSTALL_BINARY_NAME ./install.sh $INSTALL_BINARY_NAME + + + +# TODO: Some of this is only really needed for testing, it would be nice to split this up +FROM runtime-dev AS dev +RUN groupadd -r docker +RUN useradd --create-home --gid docker unprivilegeduser # Activate bash completion and include Docker's completion if mounted with DOCKER_BASH_COMPLETION_PATH RUN echo "source /usr/share/bash-completion/bash_completion" >> /etc/bash.bashrc RUN ln -s /usr/local/completion/bash/docker /etc/bash_completion.d/docker +RUN ldconfig +# This should only install packages that are specifically needed for the dev environment and nothing else +# Do you really need to add another package here? Can it be done in a different build stage? +RUN apt-get update && apt-get install -y \ + apparmor \ + aufs-tools \ + bash-completion \ + btrfs-tools \ + iptables \ + jq \ + libdevmapper-dev \ + libudev-dev \ + libsystemd-dev \ + binutils-mingw-w64 \ + g++-mingw-w64-x86-64 \ + net-tools \ + pigz \ + python-backports.ssl-match-hostname \ + python-dev \ + python-mock \ + python-pip \ + python-requests \ + python-setuptools \ + python-websocket \ + python-wheel \ + thin-provisioning-tools \ + vim \ + vim-common \ + xfsprogs \ + zip \ + --no-install-recommends +COPY --from=swagger /usr/local/bin/swagger* /usr/local/bin/ +COPY --from=frozen-images /docker-frozen-images /docker-frozen-images +COPY --from=gometalinter /opt/gometalinter/ /usr/local/bin/ +COPY --from=tomlv /opt/tomlv/ /usr/local/bin/ +COPY --from=vndr /opt/vndr/ /usr/local/bin/ +COPY --from=tini /opt/tini/ /usr/local/bin/ +COPY --from=runc /opt/runc/ /usr/local/bin/ +COPY --from=containerd /opt/containerd/ /usr/local/bin/ +COPY --from=proxy /opt/proxy/ /usr/local/bin/ +COPY --from=dockercli /opt/dockercli /usr/local/cli +COPY --from=golang /usr/local/go /usr/local/go +COPY --from=registry /usr/local/bin/registry* /usr/local/bin/ +COPY --from=notary /usr/local/bin/notary* /usr/local/bin/ +COPY --from=criu /opt/criu/ /usr/local/ +COPY --from=docker-py /docker-py /docker-py +# TODO: This is for the docker-py tests, which shouldn't really be needed for +# this image, but currently CI is expecting to run this image. This should be +# split out into a separate image, including all the `python-*` deps installed +# above. +RUN cd /docker-py \ + && pip install docker-pycreds==0.2.1 \ + && pip install -r test-requirements.txt -# Wrap all commands in the "docker-in-docker" script to allow nested containers -ENTRYPOINT ["hack/dind"] - +ENV PATH=/usr/local/cli:$PATH +ENV DOCKER_BUILDTAGS apparmor seccomp selinux # Options for hack/validate/gometalinter ENV GOMETALINTER_OPTS="--deadline=2m" - +WORKDIR /go/src/github.com/docker/docker +VOLUME /var/lib/docker +# Wrap all commands in the "docker-in-docker" script to allow nested containers +ENTRYPOINT ["hack/dind"] # Upload docker source COPY . /go/src/github.com/docker/docker - From c67f00aca0c87d9fbef0fccc986c258b2c40bc94 Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Fri, 23 Feb 2018 15:56:03 +0100 Subject: [PATCH 06/20] Update e2e script Signed-off-by: Vincent Demeester Upstream-commit: 4bb0f24716f45ac520e73a52d7d7ca2752cabd31 Component: engine --- components/engine/hack/test/e2e-run.sh | 80 ++++++++++++++++++-------- 1 file changed, 55 insertions(+), 25 deletions(-) diff --git a/components/engine/hack/test/e2e-run.sh b/components/engine/hack/test/e2e-run.sh index 448f120895..b80f7fc312 100755 --- a/components/engine/hack/test/e2e-run.sh +++ b/components/engine/hack/test/e2e-run.sh @@ -1,41 +1,71 @@ #!/usr/bin/env bash -set -e +set -e -u -o pipefail -TESTFLAGS=${TESTFLAGS:-""} -# Currently only DockerSuite and DockerNetworkSuite have been adapted for E2E testing -TESTFLAGS_LEGACY=${TESTFLAGS_LEGACY:-""} -TIMEOUT=${TIMEOUT:-60m} +ARCH=$(uname -m) +if [ "$ARCH" == "x86_64" ]; then + ARCH="amd64" +fi -SCRIPTDIR="$(dirname ${BASH_SOURCE[0]})" +export DOCKER_ENGINE_GOARCH=${DOCKER_ENGINE_GOARCH:-${ARCH}} -export DOCKER_ENGINE_GOARCH=${DOCKER_ENGINE_GOARCH:-amd64} +# Set defaults +: ${TESTFLAGS:=} +: ${TESTDEBUG:=} + +integration_api_dirs=${TEST_INTEGRATION_DIR:-"$( + find ./integration -type d | + grep -vE '(^./integration($|/internal)|/testdata)')"} run_test_integration() { - run_test_integration_suites - run_test_integration_legacy_suites + [[ "$TESTFLAGS" != *-check.f* ]] && run_test_integration_suites + run_test_integration_legacy_suites } run_test_integration_suites() { - local flags="-test.timeout=${TIMEOUT} $TESTFLAGS" - for dir in /tests/integration/*; do - if ! ( - cd $dir - echo "Running $PWD" - ./test.main $flags - ); then exit 1; fi - done + local flags="-test.v -test.timeout=${TIMEOUT:=10m} $TESTFLAGS" + for dir in $integration_api_dirs; do + if ! ( + cd $dir + echo "Running $PWD" + test_env ./test.main $flags + ); then exit 1; fi + done } run_test_integration_legacy_suites() { - ( - flags="-check.timeout=${TIMEOUT} -test.timeout=360m $TESTFLAGS_LEGACY" - cd /tests/integration-cli - echo "Running $PWD" - ./test.main $flags - ) + ( + flags="-check.v -check.timeout=${TIMEOUT} -test.timeout=360m $TESTFLAGS" + cd test/integration-cli + echo "Running $PWD" + test_env ./test.main $flags + ) } -bash $SCRIPTDIR/ensure-emptyfs.sh +# use "env -i" to tightly control the environment variables that bleed into the tests +test_env() { + ( + set -e +u + [ -n "$TESTDEBUG" ] && set -x + env -i \ + DOCKER_API_VERSION="$DOCKER_API_VERSION" \ + DOCKER_INTEGRATION_DAEMON_DEST="$DOCKER_INTEGRATION_DAEMON_DEST" \ + DOCKER_TLS_VERIFY="$DOCKER_TEST_TLS_VERIFY" \ + DOCKER_CERT_PATH="$DOCKER_TEST_CERT_PATH" \ + DOCKER_ENGINE_GOARCH="$DOCKER_ENGINE_GOARCH" \ + DOCKER_GRAPHDRIVER="$DOCKER_GRAPHDRIVER" \ + DOCKER_USERLANDPROXY="$DOCKER_USERLANDPROXY" \ + DOCKER_HOST="$DOCKER_HOST" \ + DOCKER_REMAP_ROOT="$DOCKER_REMAP_ROOT" \ + DOCKER_REMOTE_DAEMON="$DOCKER_REMOTE_DAEMON" \ + DOCKERFILE="$DOCKERFILE" \ + GOPATH="$GOPATH" \ + GOTRACEBACK=all \ + HOME="$ABS_DEST/fake-HOME" \ + PATH="$PATH" \ + TEMP="$TEMP" \ + TEST_CLIENT_BINARY="$TEST_CLIENT_BINARY" \ + "$@" + ) +} -echo "Run integration tests" run_test_integration From 7df5ef918eb9ff0710e009e919f126208f3eb7fc Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 2 Feb 2018 14:18:46 -0800 Subject: [PATCH 07/20] Move all daemon image methods into imageService imageService provides the backend for the image API and handles the imageStore, and referenceStore. Signed-off-by: Daniel Nephin Upstream-commit: 0dab53ff3cb0aae91aae068a3f1f2fd32339e23b Component: engine --- components/engine/cmd/dockerd/daemon.go | 9 +- components/engine/daemon/cache.go | 8 +- components/engine/daemon/cluster/cluster.go | 1 + .../engine/daemon/cluster/executor/backend.go | 9 +- .../cluster/executor/container/adapter.go | 8 +- .../cluster/executor/container/attachment.go | 4 +- .../cluster/executor/container/controller.go | 4 +- .../cluster/executor/container/executor.go | 8 +- .../cluster/executor/container/health_test.go | 2 +- .../executor/container/validate_test.go | 2 +- .../engine/daemon/cluster/noderunner.go | 15 +- components/engine/daemon/cluster/services.go | 2 +- components/engine/daemon/commit.go | 4 +- components/engine/daemon/container.go | 2 +- components/engine/daemon/create.go | 36 +-- components/engine/daemon/daemon.go | 152 +++++++------ components/engine/daemon/delete.go | 12 +- components/engine/daemon/disk_usage.go | 50 +---- components/engine/daemon/export.go | 6 +- components/engine/daemon/image.go | 211 +++++++++++++++++- .../daemon/{build.go => image_builder.go} | 35 ++- components/engine/daemon/image_commit.go | 23 +- components/engine/daemon/image_delete.go | 81 +++---- components/engine/daemon/image_exporter.go | 8 +- ...{getsize_unix.go => image_getsize_unix.go} | 8 +- components/engine/daemon/image_history.go | 12 +- .../daemon/{import.go => image_import.go} | 12 +- components/engine/daemon/image_inspect.go | 14 +- components/engine/daemon/image_prune.go | 20 +- components/engine/daemon/image_pull.go | 24 +- components/engine/daemon/image_push.go | 18 +- .../daemon/{search.go => image_search.go} | 7 +- components/engine/daemon/image_tag.go | 14 +- components/engine/daemon/images.go | 42 ++-- components/engine/daemon/info.go | 5 +- components/engine/daemon/inspect.go | 2 +- components/engine/daemon/list.go | 8 +- components/engine/daemon/oci_windows.go | 2 +- components/engine/daemon/prune.go | 2 +- components/engine/daemon/reload.go | 8 +- components/engine/daemon/reload_test.go | 41 +++- components/engine/daemon/start.go | 2 +- 42 files changed, 544 insertions(+), 389 deletions(-) rename components/engine/daemon/{build.go => image_builder.go} (79%) rename components/engine/daemon/{getsize_unix.go => image_getsize_unix.go} (77%) rename components/engine/daemon/{import.go => image_import.go} (87%) rename components/engine/daemon/{search.go => image_search.go} (87%) diff --git a/components/engine/cmd/dockerd/daemon.go b/components/engine/cmd/dockerd/daemon.go index cba9a7bbfc..b8552973e8 100644 --- a/components/engine/cmd/dockerd/daemon.go +++ b/components/engine/cmd/dockerd/daemon.go @@ -253,6 +253,7 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) { Root: cli.Config.Root, Name: name, Backend: d, + ImageBackend: d.ImageService(), PluginBackend: d.PluginManager(), NetworkSubnetsProvider: d, DefaultAdvertiseAddr: cli.Config.SwarmDefaultAdvertiseAddr, @@ -345,12 +346,12 @@ func newRouterOptions(config *config.Config, daemon *daemon.Daemon) (routerOptio return opts, errors.Wrap(err, "failed to create fscache") } - manager, err := dockerfile.NewBuildManager(daemon, sm, buildCache, daemon.IDMappings()) + manager, err := dockerfile.NewBuildManager(daemon.BuilderBackend(), sm, buildCache, daemon.IDMappings()) if err != nil { return opts, err } - bb, err := buildbackend.NewBackend(daemon, manager, buildCache) + bb, err := buildbackend.NewBackend(daemon.ImageService(), manager, buildCache) if err != nil { return opts, errors.Wrap(err, "failed to create buildmanager") } @@ -507,14 +508,14 @@ func initRouter(opts routerOptions) { // we need to add the checkpoint router before the container router or the DELETE gets masked checkpointrouter.NewRouter(opts.daemon, decoder), container.NewRouter(opts.daemon, decoder), - image.NewRouter(opts.daemon), + image.NewRouter(opts.daemon.ImageService()), systemrouter.NewRouter(opts.daemon, opts.cluster, opts.buildCache), volume.NewRouter(opts.daemon), build.NewRouter(opts.buildBackend, opts.daemon), sessionrouter.NewRouter(opts.sessionManager), swarmrouter.NewRouter(opts.cluster), pluginrouter.NewRouter(opts.daemon.PluginManager()), - distributionrouter.NewRouter(opts.daemon), + distributionrouter.NewRouter(opts.daemon.ImageService()), } if opts.daemon.NetworkControllerEnabled() { diff --git a/components/engine/daemon/cache.go b/components/engine/daemon/cache.go index 2a42011eff..13ddfbe41f 100644 --- a/components/engine/daemon/cache.go +++ b/components/engine/daemon/cache.go @@ -7,15 +7,15 @@ import ( ) // MakeImageCache creates a stateful image cache. -func (daemon *Daemon) MakeImageCache(sourceRefs []string) builder.ImageCache { +func (i *imageService) MakeImageCache(sourceRefs []string) builder.ImageCache { if len(sourceRefs) == 0 { - return cache.NewLocal(daemon.imageStore) + return cache.NewLocal(i.imageStore) } - cache := cache.New(daemon.imageStore) + cache := cache.New(i.imageStore) for _, ref := range sourceRefs { - img, err := daemon.GetImage(ref) + img, err := i.GetImage(ref) if err != nil { logrus.Warnf("Could not look up %s for cache resolution, skipping: %+v", ref, err) continue diff --git a/components/engine/daemon/cluster/cluster.go b/components/engine/daemon/cluster/cluster.go index 7fdf633bd8..cdd47ec0e7 100644 --- a/components/engine/daemon/cluster/cluster.go +++ b/components/engine/daemon/cluster/cluster.go @@ -83,6 +83,7 @@ type Config struct { Root string Name string Backend executorpkg.Backend + ImageBackend executorpkg.ImageBackend PluginBackend plugin.Backend NetworkSubnetsProvider NetworkSubnetsProvider diff --git a/components/engine/daemon/cluster/executor/backend.go b/components/engine/daemon/cluster/executor/backend.go index a0b56a97bc..81ba79e91e 100644 --- a/components/engine/daemon/cluster/executor/backend.go +++ b/components/engine/daemon/cluster/executor/backend.go @@ -31,7 +31,6 @@ type Backend interface { FindNetwork(idName string) (libnetwork.Network, error) SetupIngress(clustertypes.NetworkCreateRequest, string) (<-chan struct{}, error) ReleaseIngress() (<-chan struct{}, error) - PullImage(ctx context.Context, image, tag, platform string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error CreateManagedContainer(config types.ContainerCreateConfig) (container.ContainerCreateCreatedBody, error) ContainerStart(name string, hostConfig *container.HostConfig, checkpoint string, checkpointDir string) error ContainerStop(name string, seconds *int) error @@ -58,9 +57,13 @@ type Backend interface { UnsubscribeFromEvents(listener chan interface{}) UpdateAttachment(string, string, string, *network.NetworkingConfig) error WaitForDetachment(context.Context, string, string, string, string) error - GetRepository(context.Context, reference.Named, *types.AuthConfig) (distribution.Repository, bool, error) - LookupImage(name string) (*types.ImageInspect, error) PluginManager() *plugin.Manager PluginGetter() *plugin.Store GetAttachmentStore() *networkSettings.AttachmentStore } + +type ImageBackend interface { + PullImage(ctx context.Context, image, tag, platform string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error + GetRepository(context.Context, reference.Named, *types.AuthConfig) (distribution.Repository, bool, error) + LookupImage(name string) (*types.ImageInspect, error) +} diff --git a/components/engine/daemon/cluster/executor/container/adapter.go b/components/engine/daemon/cluster/executor/container/adapter.go index 0520fa0bc1..7395036cc8 100644 --- a/components/engine/daemon/cluster/executor/container/adapter.go +++ b/components/engine/daemon/cluster/executor/container/adapter.go @@ -36,11 +36,12 @@ import ( // containerConfig. type containerAdapter struct { backend executorpkg.Backend + imageBackend executorpkg.ImageBackend container *containerConfig dependencies exec.DependencyGetter } -func newContainerAdapter(b executorpkg.Backend, task *api.Task, node *api.NodeDescription, dependencies exec.DependencyGetter) (*containerAdapter, error) { +func newContainerAdapter(b executorpkg.Backend, i executorpkg.ImageBackend, task *api.Task, node *api.NodeDescription, dependencies exec.DependencyGetter) (*containerAdapter, error) { ctnr, err := newContainerConfig(task, node) if err != nil { return nil, err @@ -49,6 +50,7 @@ func newContainerAdapter(b executorpkg.Backend, task *api.Task, node *api.NodeDe return &containerAdapter{ container: ctnr, backend: b, + imageBackend: i, dependencies: dependencies, }, nil } @@ -66,7 +68,7 @@ func (c *containerAdapter) pullImage(ctx context.Context) error { named, err := reference.ParseNormalizedNamed(spec.Image) if err == nil { if _, ok := named.(reference.Canonical); ok { - _, err := c.backend.LookupImage(spec.Image) + _, err := c.imageBackend.LookupImage(spec.Image) if err == nil { return nil } @@ -92,7 +94,7 @@ func (c *containerAdapter) pullImage(ctx context.Context) error { // TODO @jhowardmsft LCOW Support: This will need revisiting as // the stack is built up to include LCOW support for swarm. platform := runtime.GOOS - err := c.backend.PullImage(ctx, c.container.image(), "", platform, metaHeaders, authConfig, pw) + err := c.imageBackend.PullImage(ctx, c.container.image(), "", platform, metaHeaders, authConfig, pw) pw.CloseWithError(err) }() diff --git a/components/engine/daemon/cluster/executor/container/attachment.go b/components/engine/daemon/cluster/executor/container/attachment.go index 0c630745ba..4d6e5c2983 100644 --- a/components/engine/daemon/cluster/executor/container/attachment.go +++ b/components/engine/daemon/cluster/executor/container/attachment.go @@ -20,8 +20,8 @@ type networkAttacherController struct { closed chan struct{} } -func newNetworkAttacherController(b executorpkg.Backend, task *api.Task, node *api.NodeDescription, dependencies exec.DependencyGetter) (*networkAttacherController, error) { - adapter, err := newContainerAdapter(b, task, node, dependencies) +func newNetworkAttacherController(b executorpkg.Backend, i executorpkg.ImageBackend, task *api.Task, node *api.NodeDescription, dependencies exec.DependencyGetter) (*networkAttacherController, error) { + adapter, err := newContainerAdapter(b, i, task, node, dependencies) if err != nil { return nil, err } diff --git a/components/engine/daemon/cluster/executor/container/controller.go b/components/engine/daemon/cluster/executor/container/controller.go index 5b7732cf3f..32bb114fde 100644 --- a/components/engine/daemon/cluster/executor/container/controller.go +++ b/components/engine/daemon/cluster/executor/container/controller.go @@ -40,8 +40,8 @@ type controller struct { var _ exec.Controller = &controller{} // NewController returns a docker exec runner for the provided task. -func newController(b executorpkg.Backend, task *api.Task, node *api.NodeDescription, dependencies exec.DependencyGetter) (*controller, error) { - adapter, err := newContainerAdapter(b, task, node, dependencies) +func newController(b executorpkg.Backend, i executorpkg.ImageBackend, task *api.Task, node *api.NodeDescription, dependencies exec.DependencyGetter) (*controller, error) { + adapter, err := newContainerAdapter(b, i, task, node, dependencies) if err != nil { return nil, err } diff --git a/components/engine/daemon/cluster/executor/container/executor.go b/components/engine/daemon/cluster/executor/container/executor.go index 2c4f619cf0..713933fb6f 100644 --- a/components/engine/daemon/cluster/executor/container/executor.go +++ b/components/engine/daemon/cluster/executor/container/executor.go @@ -26,6 +26,7 @@ import ( type executor struct { backend executorpkg.Backend + imageBackend executorpkg.ImageBackend pluginBackend plugin.Backend dependencies exec.DependencyManager mutex sync.Mutex // This mutex protects the following node field @@ -33,10 +34,11 @@ type executor struct { } // NewExecutor returns an executor from the docker client. -func NewExecutor(b executorpkg.Backend, p plugin.Backend) exec.Executor { +func NewExecutor(b executorpkg.Backend, p plugin.Backend, i executorpkg.ImageBackend) exec.Executor { return &executor{ backend: b, pluginBackend: p, + imageBackend: i, dependencies: agent.NewDependencyManager(), } } @@ -200,7 +202,7 @@ func (e *executor) Controller(t *api.Task) (exec.Controller, error) { e.mutex.Unlock() if t.Spec.GetAttachment() != nil { - return newNetworkAttacherController(e.backend, t, nodeDescription, dependencyGetter) + return newNetworkAttacherController(e.backend, e.imageBackend, t, nodeDescription, dependencyGetter) } var ctlr exec.Controller @@ -229,7 +231,7 @@ func (e *executor) Controller(t *api.Task) (exec.Controller, error) { return ctlr, fmt.Errorf("unsupported runtime type: %q", runtimeKind) } case *api.TaskSpec_Container: - c, err := newController(e.backend, t, nodeDescription, dependencyGetter) + c, err := newController(e.backend, e.imageBackend, t, nodeDescription, dependencyGetter) if err != nil { return ctlr, err } diff --git a/components/engine/daemon/cluster/executor/container/health_test.go b/components/engine/daemon/cluster/executor/container/health_test.go index 59633dd127..2054286417 100644 --- a/components/engine/daemon/cluster/executor/container/health_test.go +++ b/components/engine/daemon/cluster/executor/container/health_test.go @@ -52,7 +52,7 @@ func TestHealthStates(t *testing.T) { EventsService: e, } - controller, err := newController(daemon, task, nil, nil) + controller, err := newController(daemon, nil, task, nil, nil) if err != nil { t.Fatalf("create controller fail %v", err) } diff --git a/components/engine/daemon/cluster/executor/container/validate_test.go b/components/engine/daemon/cluster/executor/container/validate_test.go index d843431f4f..9b6cc8ee22 100644 --- a/components/engine/daemon/cluster/executor/container/validate_test.go +++ b/components/engine/daemon/cluster/executor/container/validate_test.go @@ -12,7 +12,7 @@ import ( ) func newTestControllerWithMount(m api.Mount) (*controller, error) { - return newController(&daemon.Daemon{}, &api.Task{ + return newController(&daemon.Daemon{}, nil, &api.Task{ ID: stringid.GenerateRandomID(), ServiceID: stringid.GenerateRandomID(), Spec: api.TaskSpec{ diff --git a/components/engine/daemon/cluster/noderunner.go b/components/engine/daemon/cluster/noderunner.go index f832aca145..b538e844e7 100644 --- a/components/engine/daemon/cluster/noderunner.go +++ b/components/engine/daemon/cluster/noderunner.go @@ -120,12 +120,15 @@ func (n *nodeRunner) start(conf nodeStartConfig) error { JoinAddr: joinAddr, StateDir: n.cluster.root, JoinToken: conf.joinToken, - Executor: container.NewExecutor(n.cluster.config.Backend, n.cluster.config.PluginBackend), - HeartbeatTick: 1, - ElectionTick: 3, - UnlockKey: conf.lockKey, - AutoLockManagers: conf.autolock, - PluginGetter: n.cluster.config.Backend.PluginGetter(), + Executor: container.NewExecutor( + n.cluster.config.Backend, + n.cluster.config.PluginBackend, + n.cluster.config.ImageBackend), + HeartbeatTick: 1, + ElectionTick: 3, + UnlockKey: conf.lockKey, + AutoLockManagers: conf.autolock, + PluginGetter: n.cluster.config.Backend.PluginGetter(), } if conf.availability != "" { avail, ok := swarmapi.NodeSpec_Availability_value[strings.ToUpper(string(conf.availability))] diff --git a/components/engine/daemon/cluster/services.go b/components/engine/daemon/cluster/services.go index 5cc9566fa0..3923ced6c0 100644 --- a/components/engine/daemon/cluster/services.go +++ b/components/engine/daemon/cluster/services.go @@ -570,7 +570,7 @@ func (c *Cluster) imageWithDigestString(ctx context.Context, image string, authC return "", errors.Errorf("image reference not tagged: %s", image) } - repo, _, err := c.config.Backend.GetRepository(ctx, taggedRef, authConfig) + repo, _, err := c.config.ImageBackend.GetRepository(ctx, taggedRef, authConfig) if err != nil { return "", err } diff --git a/components/engine/daemon/commit.go b/components/engine/daemon/commit.go index 6af494da74..0f6f440514 100644 --- a/components/engine/daemon/commit.go +++ b/components/engine/daemon/commit.go @@ -155,7 +155,7 @@ func (daemon *Daemon) CreateImageFromContainer(name string, c *backend.CreateIma return "", err } - id, err := daemon.commitImage(backend.CommitConfig{ + id, err := daemon.imageService.CommitImage(backend.CommitConfig{ Author: c.Author, Comment: c.Comment, Config: newConfig, @@ -171,7 +171,7 @@ func (daemon *Daemon) CreateImageFromContainer(name string, c *backend.CreateIma var imageRef string if c.Repo != "" { - imageRef, err = daemon.TagImage(string(id), c.Repo, c.Tag) + imageRef, err = daemon.imageService.TagImage(string(id), c.Repo, c.Tag) if err != nil { return "", err } diff --git a/components/engine/daemon/container.go b/components/engine/daemon/container.go index a16abf2144..b1ae3daa94 100644 --- a/components/engine/daemon/container.go +++ b/components/engine/daemon/container.go @@ -158,7 +158,7 @@ func (daemon *Daemon) newContainer(name string, operatingSystem string, config * base.ImageID = imgID base.NetworkSettings = &network.Settings{IsAnonymousEndpoint: noExplicitName} base.Name = name - base.Driver = daemon.GraphDriverName(operatingSystem) + base.Driver = daemon.imageService.GraphDriverForOS(operatingSystem) base.OS = operatingSystem return base, err } diff --git a/components/engine/daemon/create.go b/components/engine/daemon/create.go index 7e6b6519fe..386d346f73 100644 --- a/components/engine/daemon/create.go +++ b/components/engine/daemon/create.go @@ -15,7 +15,6 @@ import ( "github.com/docker/docker/container" "github.com/docker/docker/errdefs" "github.com/docker/docker/image" - "github.com/docker/docker/layer" "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/pkg/system" @@ -42,7 +41,7 @@ func (daemon *Daemon) containerCreate(params types.ContainerCreateConfig, manage os := runtime.GOOS if params.Config.Image != "" { - img, err := daemon.GetImage(params.Config.Image) + img, err := daemon.imageService.GetImage(params.Config.Image) if err == nil { os = img.OS } @@ -92,7 +91,7 @@ func (daemon *Daemon) create(params types.ContainerCreateConfig, managed bool) ( os := runtime.GOOS if params.Config.Image != "" { - img, err = daemon.GetImage(params.Config.Image) + img, err = daemon.imageService.GetImage(params.Config.Image) if err != nil { return nil, err } @@ -158,9 +157,11 @@ func (daemon *Daemon) create(params types.ContainerCreateConfig, managed bool) ( } // Set RWLayer for container after mount labels have been set - if err := daemon.setRWLayer(container); err != nil { + rwLayer, err := daemon.imageService.GetRWLayer(container, setupInitLayer(daemon.idMappings)) + if err != nil { return nil, errdefs.System(err) } + container.RWLayer = rwLayer rootIDs := daemon.idMappings.RootPair() if err := idtools.MkdirAndChown(container.Root, 0700, rootIDs); err != nil { @@ -254,33 +255,6 @@ func (daemon *Daemon) generateSecurityOpt(hostConfig *containertypes.HostConfig) return nil, nil } -func (daemon *Daemon) setRWLayer(container *container.Container) error { - var layerID layer.ChainID - if container.ImageID != "" { - img, err := daemon.imageStore.Get(container.ImageID) - if err != nil { - return err - } - layerID = img.RootFS.ChainID() - } - - rwLayerOpts := &layer.CreateRWLayerOpts{ - MountLabel: container.MountLabel, - InitFunc: setupInitLayer(daemon.idMappings), - StorageOpt: container.HostConfig.StorageOpt, - } - - // Indexing by OS is safe here as validation of OS has already been performed in create() (the only - // caller), and guaranteed non-nil - rwLayer, err := daemon.layerStores[container.OS].CreateRWLayer(container.ID, layerID, rwLayerOpts) - if err != nil { - return err - } - container.RWLayer = rwLayer - - return nil -} - // VolumeCreate creates a volume with the specified name, driver, and opts // This is called directly from the Engine API func (daemon *Daemon) VolumeCreate(name, driverName string, opts, labels map[string]string) (*types.Volume, error) { diff --git a/components/engine/daemon/daemon.go b/components/engine/daemon/daemon.go index 335e6442f7..4a41e504ca 100644 --- a/components/engine/daemon/daemon.go +++ b/components/engine/daemon/daemon.go @@ -31,6 +31,7 @@ import ( "github.com/docker/docker/errdefs" "github.com/sirupsen/logrus" // register graph drivers + "github.com/docker/docker/builder" _ "github.com/docker/docker/daemon/graphdriver/register" "github.com/docker/docker/daemon/stats" dmetadata "github.com/docker/docker/distribution/metadata" @@ -57,7 +58,6 @@ import ( "github.com/docker/libnetwork" "github.com/docker/libnetwork/cluster" nwconfig "github.com/docker/libnetwork/config" - "github.com/docker/libtrust" "github.com/pkg/errors" ) @@ -70,44 +70,40 @@ var ( // Daemon holds information about the Docker daemon. type Daemon struct { - ID string - repository string - containers container.Store - containersReplica container.ViewDB - execCommands *exec.Store - downloadManager *xfer.LayerDownloadManager - uploadManager *xfer.LayerUploadManager - trustKey libtrust.PrivateKey - idIndex *truncindex.TruncIndex - configStore *config.Config - statsCollector *stats.Collector - defaultLogConfig containertypes.LogConfig - RegistryService registry.Service - EventsService *events.Events - netController libnetwork.NetworkController - volumes *store.VolumeStore - discoveryWatcher discovery.Reloader - root string - seccompEnabled bool - apparmorEnabled bool - shutdown bool - idMappings *idtools.IDMappings - graphDrivers map[string]string // By operating system - referenceStore refstore.Store - imageStore image.Store - imageRoot string - layerStores map[string]layer.Store // By operating system - distributionMetadataStore dmetadata.Store - PluginStore *plugin.Store // todo: remove - pluginManager *plugin.Manager - linkIndex *linkIndex - containerd libcontainerd.Client - containerdRemote libcontainerd.Remote - defaultIsolation containertypes.Isolation // Default isolation mode on Windows - clusterProvider cluster.Provider - cluster Cluster - genericResources []swarm.GenericResource - metricsPluginListener net.Listener + ID string + repository string + containers container.Store + containersReplica container.ViewDB + execCommands *exec.Store + + imageService *imageService + + idIndex *truncindex.TruncIndex + configStore *config.Config + statsCollector *stats.Collector + defaultLogConfig containertypes.LogConfig + RegistryService registry.Service + EventsService *events.Events + netController libnetwork.NetworkController + volumes *store.VolumeStore + discoveryWatcher discovery.Reloader + root string + seccompEnabled bool + apparmorEnabled bool + shutdown bool + idMappings *idtools.IDMappings + // TODO: move graphDrivers field to an InfoService + graphDrivers map[string]string // By operating system + + PluginStore *plugin.Store // todo: remove + pluginManager *plugin.Manager + linkIndex *linkIndex + containerd libcontainerd.Client + defaultIsolation containertypes.Isolation // Default isolation mode on Windows + clusterProvider cluster.Provider + cluster Cluster + genericResources []swarm.GenericResource + metricsPluginListener net.Listener machineMemory uint64 @@ -162,7 +158,7 @@ func (daemon *Daemon) restore() error { // Ignore the container if it does not support the current driver being used by the graph currentDriverForContainerOS := daemon.graphDrivers[container.OS] if (container.Driver == "" && currentDriverForContainerOS == "aufs") || container.Driver == currentDriverForContainerOS { - rwlayer, err := daemon.layerStores[container.OS].GetRWLayer(container.ID) + rwlayer, err := daemon.imageService.GetRWLayerByID(container.ID, container.OS) if err != nil { logrus.Errorf("Failed to load container mount %v: %v", id, err) continue @@ -705,7 +701,7 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe // be set through an environment variable, a daemon start parameter, or chosen through // initialization of the layerstore through driver priority order for example. d.graphDrivers = make(map[string]string) - d.layerStores = make(map[string]layer.Store) + layerStores := make(map[string]layer.Store) if runtime.GOOS == "windows" { d.graphDrivers[runtime.GOOS] = "windowsfilter" if system.LCOWSupported() { @@ -754,7 +750,7 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe } for operatingSystem, gd := range d.graphDrivers { - d.layerStores[operatingSystem], err = layer.NewStoreFromOptions(layer.StoreOptions{ + layerStores[operatingSystem], err = layer.NewStoreFromOptions(layer.StoreOptions{ Root: config.Root, MetadataStorePathTemplate: filepath.Join(config.Root, "image", "%s", "layerdb"), GraphDriver: gd, @@ -771,7 +767,7 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe // As layerstore initialization may set the driver for os := range d.graphDrivers { - d.graphDrivers[os] = d.layerStores[os].DriverName() + d.graphDrivers[os] = layerStores[os].DriverName() } // Configure and validate the kernels security support. Note this is a Linux/FreeBSD @@ -780,22 +776,17 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe return nil, err } - logrus.Debugf("Max Concurrent Downloads: %d", *config.MaxConcurrentDownloads) - d.downloadManager = xfer.NewLayerDownloadManager(d.layerStores, *config.MaxConcurrentDownloads) - logrus.Debugf("Max Concurrent Uploads: %d", *config.MaxConcurrentUploads) - d.uploadManager = xfer.NewLayerUploadManager(*config.MaxConcurrentUploads) - - d.imageRoot = filepath.Join(config.Root, "image", d.graphDrivers[runtime.GOOS]) - ifs, err := image.NewFSStoreBackend(filepath.Join(d.imageRoot, "imagedb")) + imageRoot := filepath.Join(config.Root, "image", d.graphDrivers[runtime.GOOS]) + ifs, err := image.NewFSStoreBackend(filepath.Join(imageRoot, "imagedb")) if err != nil { return nil, err } lgrMap := make(map[string]image.LayerGetReleaser) - for os, ls := range d.layerStores { + for os, ls := range layerStores { lgrMap[os] = ls } - d.imageStore, err = image.NewImageStore(ifs, lgrMap) + imageStore, err := image.NewImageStore(ifs, lgrMap) if err != nil { return nil, err } @@ -829,14 +820,13 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe // operating systems, the list of graphdrivers available isn't user configurable. // For backwards compatibility, we just put it under the windowsfilter // directory regardless. - refStoreLocation := filepath.Join(d.imageRoot, `repositories.json`) + refStoreLocation := filepath.Join(imageRoot, `repositories.json`) rs, err := refstore.NewReferenceStore(refStoreLocation) if err != nil { return nil, fmt.Errorf("Couldn't create reference store repository: %s", err) } - d.referenceStore = rs - d.distributionMetadataStore, err = dmetadata.NewFSMetadataStore(filepath.Join(d.imageRoot, "distribution")) + distributionMetadataStore, err := dmetadata.NewFSMetadataStore(filepath.Join(imageRoot, "distribution")) if err != nil { return nil, err } @@ -844,7 +834,7 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe // No content-addressability migration on Windows as it never supported pre-CA if runtime.GOOS != "windows" { migrationStart := time.Now() - if err := v1.Migrate(config.Root, d.graphDrivers[runtime.GOOS], d.layerStores[runtime.GOOS], d.imageStore, rs, d.distributionMetadataStore); err != nil { + if err := v1.Migrate(config.Root, d.graphDrivers[runtime.GOOS], layerStores[runtime.GOOS], imageStore, rs, distributionMetadataStore); err != nil { logrus.Errorf("Graph migration failed: %q. Your old graph data was found to be too inconsistent for upgrading to content-addressable storage. Some of the old data was probably not upgraded. We recommend starting over with a clean storage directory if possible.", err) } logrus.Infof("Graph migration to content-addressability took %.2f seconds", time.Since(migrationStart).Seconds()) @@ -870,7 +860,6 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe return nil, err } d.execCommands = exec.NewStore() - d.trustKey = trustKey d.idIndex = truncindex.NewTruncIndex([]string{}) d.statsCollector = d.newStatsCollector(1 * time.Second) @@ -880,10 +869,23 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe d.idMappings = idMappings d.seccompEnabled = sysInfo.Seccomp d.apparmorEnabled = sysInfo.AppArmor - d.containerdRemote = containerdRemote d.linkIndex = newLinkIndex() + logrus.Debugf("Max Concurrent Downloads: %d", *config.MaxConcurrentDownloads) + logrus.Debugf("Max Concurrent Uploads: %d", *config.MaxConcurrentUploads) + d.imageService = &imageService{ + trustKey: trustKey, + uploadManager: xfer.NewLayerUploadManager(*config.MaxConcurrentUploads), + downloadManager: xfer.NewLayerDownloadManager(layerStores, *config.MaxConcurrentDownloads), + registryService: registryService, + referenceStore: rs, + distributionMetadataStore: distributionMetadataStore, + imageStore: imageStore, + eventsService: eventsService, + containers: d.containers, + } + go d.execCommandGC() d.containerd, err = containerdRemote.NewClient(ContainersNamespace, d) @@ -1005,7 +1007,7 @@ func (daemon *Daemon) Shutdown() error { logrus.Errorf("Stop container error: %v", err) return } - if mountid, err := daemon.layerStores[c.OS].GetMountID(c.ID); err == nil { + if mountid, err := daemon.imageService.GetContainerMountID(c.ID, c.OS); err == nil { daemon.cleanupMountsByID(mountid) } logrus.Debugf("container stopped %s", c.ID) @@ -1018,13 +1020,7 @@ func (daemon *Daemon) Shutdown() error { } } - for os, ls := range daemon.layerStores { - if ls != nil { - if err := ls.Cleanup(); err != nil { - logrus.Errorf("Error during layer Store.Cleanup(): %v %s", err, os) - } - } - } + daemon.imageService.Cleanup() // If we are part of a cluster, clean up cluster's stuff if daemon.clusterProvider != nil { @@ -1064,7 +1060,7 @@ func (daemon *Daemon) Mount(container *container.Container) error { if runtime.GOOS != "windows" { daemon.Unmount(container) return fmt.Errorf("Error: driver %s is returning inconsistent paths for container %s ('%s' then '%s')", - daemon.GraphDriverName(container.OS), container.ID, container.BaseFS, dir) + daemon.imageService.GraphDriverForOS(container.OS), container.ID, container.BaseFS, dir) } } container.BaseFS = dir // TODO: combine these fields @@ -1108,11 +1104,6 @@ func (daemon *Daemon) Subnets() ([]net.IPNet, []net.IPNet) { return v4Subnets, v6Subnets } -// GraphDriverName returns the name of the graph driver used by the layer.Store -func (daemon *Daemon) GraphDriverName(os string) string { - return daemon.layerStores[os].DriverName() -} - // prepareTempDir prepares and returns the default directory to use // for temporary files. // If it doesn't exist, it is created. If it exists, its content is removed. @@ -1323,3 +1314,20 @@ func fixMemorySwappiness(resources *containertypes.Resources) { func (daemon *Daemon) GetAttachmentStore() *network.AttachmentStore { return &daemon.attachmentStore } + +// IDMappings returns uid/gid mappings for the builder +func (daemon *Daemon) IDMappings() *idtools.IDMappings { + return daemon.idMappings +} + +func (daemon *Daemon) ImageService() *imageService { + return daemon.imageService +} + +// TODO: tmp hack to merge interfaces +func (daemon *Daemon) BuilderBackend() builder.Backend { + return struct { + *Daemon + *imageService + }{daemon, daemon.imageService} +} diff --git a/components/engine/daemon/delete.go b/components/engine/daemon/delete.go index b0ea416f9f..749afeb812 100644 --- a/components/engine/daemon/delete.go +++ b/components/engine/daemon/delete.go @@ -10,7 +10,6 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/container" "github.com/docker/docker/errdefs" - "github.com/docker/docker/layer" "github.com/docker/docker/pkg/system" "github.com/docker/docker/volume" volumestore "github.com/docker/docker/volume/store" @@ -121,12 +120,11 @@ func (daemon *Daemon) cleanupContainer(container *container.Container, forceRemo // When container creation fails and `RWLayer` has not been created yet, we // do not call `ReleaseRWLayer` if container.RWLayer != nil { - metadata, err := daemon.layerStores[container.OS].ReleaseRWLayer(container.RWLayer) - layer.LogReleaseMetadata(metadata) - if err != nil && err != layer.ErrMountDoesNotExist && !os.IsNotExist(errors.Cause(err)) { - e := errors.Wrapf(err, "driver %q failed to remove root filesystem for %s", daemon.GraphDriverName(container.OS), container.ID) - container.SetRemovalError(e) - return e + err := daemon.imageService.ReleaseContainerLayer(container.RWLayer, container.OS) + if err != nil { + err = errors.Wrapf(err, "container %s", container.ID) + container.SetRemovalError(err) + return err } container.RWLayer = nil } diff --git a/components/engine/daemon/disk_usage.go b/components/engine/daemon/disk_usage.go index 2ccafbbbf1..2f6132eb39 100644 --- a/components/engine/daemon/disk_usage.go +++ b/components/engine/daemon/disk_usage.go @@ -8,34 +8,11 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" - "github.com/docker/docker/layer" "github.com/docker/docker/pkg/directory" "github.com/docker/docker/volume" - "github.com/opencontainers/go-digest" "github.com/sirupsen/logrus" ) -func (daemon *Daemon) getLayerRefs() map[layer.ChainID]int { - tmpImages := daemon.imageStore.Map() - layerRefs := map[layer.ChainID]int{} - for id, img := range tmpImages { - dgst := digest.Digest(id) - if len(daemon.referenceStore.References(dgst)) == 0 && len(daemon.imageStore.Children(id)) != 0 { - continue - } - - rootFS := *img.RootFS - rootFS.DiffIDs = nil - for _, id := range img.RootFS.DiffIDs { - rootFS.Append(id) - chid := rootFS.ChainID() - layerRefs[chid]++ - } - } - - return layerRefs -} - // SystemDiskUsage returns information about the daemon data disk usage func (daemon *Daemon) SystemDiskUsage(ctx context.Context) (*types.DiskUsage, error) { if !atomic.CompareAndSwapInt32(&daemon.diskUsageRunning, 0, 1) { @@ -53,7 +30,7 @@ func (daemon *Daemon) SystemDiskUsage(ctx context.Context) (*types.DiskUsage, er } // Get all top images with extra attributes - allImages, err := daemon.Images(filters.NewArgs(), false, true) + allImages, err := daemon.imageService.Images(filters.NewArgs(), false, true) if err != nil { return nil, fmt.Errorf("failed to retrieve image list: %v", err) } @@ -93,28 +70,9 @@ func (daemon *Daemon) SystemDiskUsage(ctx context.Context) (*types.DiskUsage, er return nil, err } - // Get total layers size on disk - var allLayersSize int64 - layerRefs := daemon.getLayerRefs() - for _, ls := range daemon.layerStores { - allLayers := ls.Map() - for _, l := range allLayers { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - size, err := l.DiffSize() - if err == nil { - if _, ok := layerRefs[l.ChainID()]; ok { - allLayersSize += size - } else { - logrus.Warnf("found leaked image layer %v", l.ChainID()) - } - } else { - logrus.Warnf("failed to get diff size for layer %v", l.ChainID()) - } - } - } + allLayersSize, err := daemon.imageService.LayerDiskUsage(ctx) + if err != nil { + return nil, err } return &types.DiskUsage{ diff --git a/components/engine/daemon/export.go b/components/engine/daemon/export.go index 1f788ad9a8..27181c9176 100644 --- a/components/engine/daemon/export.go +++ b/components/engine/daemon/export.go @@ -51,13 +51,13 @@ func (daemon *Daemon) containerExport(container *container.Container) (arch io.R if !system.IsOSSupported(container.OS) { return nil, fmt.Errorf("cannot export %s: %s ", container.ID, system.ErrNotSupportedOperatingSystem) } - rwlayer, err := daemon.layerStores[container.OS].GetRWLayer(container.ID) + rwlayer, err := daemon.imageService.GetRWLayerByID(container.ID, container.OS) if err != nil { return nil, err } defer func() { if err != nil { - daemon.layerStores[container.OS].ReleaseRWLayer(rwlayer) + daemon.imageService.ReleaseContainerLayer(rwlayer, container.OS) } }() @@ -78,7 +78,7 @@ func (daemon *Daemon) containerExport(container *container.Container) (arch io.R arch = ioutils.NewReadCloserWrapper(archive, func() error { err := archive.Close() rwlayer.Unmount() - daemon.layerStores[container.OS].ReleaseRWLayer(rwlayer) + daemon.imageService.ReleaseContainerLayer(rwlayer, container.OS) return err }) daemon.LogContainerEvent(container, "export") diff --git a/components/engine/daemon/image.go b/components/engine/daemon/image.go index 53df7417d0..366dff44fc 100644 --- a/components/engine/daemon/image.go +++ b/components/engine/daemon/image.go @@ -1,11 +1,25 @@ package daemon // import "github.com/docker/docker/daemon" import ( + "context" "fmt" + "os" "github.com/docker/distribution/reference" + "github.com/docker/docker/api/types/events" + "github.com/docker/docker/container" + daemonevents "github.com/docker/docker/daemon/events" + "github.com/docker/docker/distribution/metadata" + "github.com/docker/docker/distribution/xfer" "github.com/docker/docker/errdefs" "github.com/docker/docker/image" + "github.com/docker/docker/layer" + dockerreference "github.com/docker/docker/reference" + "github.com/docker/docker/registry" + "github.com/docker/libtrust" + "github.com/opencontainers/go-digest" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) // errImageDoesNotExist is error returned when no image can be found for a reference. @@ -25,7 +39,8 @@ func (e errImageDoesNotExist) NotFound() {} // GetImageIDAndOS returns an image ID and operating system corresponding to the image referred to by // refOrID. -func (daemon *Daemon) GetImageIDAndOS(refOrID string) (image.ID, string, error) { +// called from list.go foldFilter() +func (i imageService) GetImageIDAndOS(refOrID string) (image.ID, string, error) { ref, err := reference.ParseAnyReference(refOrID) if err != nil { return "", "", errdefs.InvalidParameter(err) @@ -37,23 +52,23 @@ func (daemon *Daemon) GetImageIDAndOS(refOrID string) (image.ID, string, error) return "", "", errImageDoesNotExist{ref} } id := image.IDFromDigest(digested.Digest()) - if img, err := daemon.imageStore.Get(id); err == nil { + if img, err := i.imageStore.Get(id); err == nil { return id, img.OperatingSystem(), nil } return "", "", errImageDoesNotExist{ref} } - if digest, err := daemon.referenceStore.Get(namedRef); err == nil { + if digest, err := i.referenceStore.Get(namedRef); err == nil { // Search the image stores to get the operating system, defaulting to host OS. id := image.IDFromDigest(digest) - if img, err := daemon.imageStore.Get(id); err == nil { + if img, err := i.imageStore.Get(id); err == nil { return id, img.OperatingSystem(), nil } } // Search based on ID - if id, err := daemon.imageStore.Search(refOrID); err == nil { - img, err := daemon.imageStore.Get(id) + if id, err := i.imageStore.Search(refOrID); err == nil { + img, err := i.imageStore.Get(id) if err != nil { return "", "", errImageDoesNotExist{ref} } @@ -64,10 +79,188 @@ func (daemon *Daemon) GetImageIDAndOS(refOrID string) (image.ID, string, error) } // GetImage returns an image corresponding to the image referred to by refOrID. -func (daemon *Daemon) GetImage(refOrID string) (*image.Image, error) { - imgID, _, err := daemon.GetImageIDAndOS(refOrID) +func (i *imageService) GetImage(refOrID string) (*image.Image, error) { + imgID, _, err := i.GetImageIDAndOS(refOrID) if err != nil { return nil, err } - return daemon.imageStore.Get(imgID) + return i.imageStore.Get(imgID) +} + +type containerStore interface { + // used by image delete + First(container.StoreFilter) *container.Container + // used by image prune, and image list + List() []*container.Container + // TODO: remove, only used for CommitBuildStep + Get(string) *container.Container +} + +type imageService struct { + eventsService *daemonevents.Events + containers containerStore + downloadManager *xfer.LayerDownloadManager + uploadManager *xfer.LayerUploadManager + + // TODO: should accept a trust service instead of a key + trustKey libtrust.PrivateKey + + registryService registry.Service + referenceStore dockerreference.Store + distributionMetadataStore metadata.Store + imageStore image.Store + layerStores map[string]layer.Store // By operating system + + pruneRunning int32 +} + +// called from info.go +func (i *imageService) CountImages() int { + return len(i.imageStore.Map()) +} + +// called from list.go to filter containers +func (i *imageService) Children(id image.ID) []image.ID { + return i.imageStore.Children(id) +} + +// TODO: accept an opt struct instead of container? +// called from create.go +func (i *imageService) GetRWLayer(container *container.Container, initFunc layer.MountInit) (layer.RWLayer, error) { + var layerID layer.ChainID + if container.ImageID != "" { + img, err := i.imageStore.Get(container.ImageID) + if err != nil { + return nil, err + } + layerID = img.RootFS.ChainID() + } + + rwLayerOpts := &layer.CreateRWLayerOpts{ + MountLabel: container.MountLabel, + InitFunc: initFunc, + StorageOpt: container.HostConfig.StorageOpt, + } + + // Indexing by OS is safe here as validation of OS has already been performed in create() (the only + // caller), and guaranteed non-nil + return i.layerStores[container.OS].CreateRWLayer(container.ID, layerID, rwLayerOpts) +} + +// called from daemon.go Daemon.restore(), and Daemon.containerExport() +func (i *imageService) GetRWLayerByID(cid string, os string) (layer.RWLayer, error) { + return i.layerStores[os].GetRWLayer(cid) +} + +// called from info.go +func (i *imageService) GraphDriverStatuses() map[string][][2]string { + result := make(map[string][][2]string) + for os, store := range i.layerStores { + result[os] = store.DriverStatus() + } + return result +} + +// called from daemon.go Daemon.Shutdown(), and Daemon.Cleanup() (cleanup is actually continerCleanup) +func (i *imageService) GetContainerMountID(cid string, os string) (string, error) { + return i.layerStores[os].GetMountID(cid) +} + +// called from daemon.go Daemon.Shutdown() +func (i *imageService) Cleanup() { + for os, ls := range i.layerStores { + if ls != nil { + if err := ls.Cleanup(); err != nil { + logrus.Errorf("Error during layer Store.Cleanup(): %v %s", err, os) + } + } + } +} + +// moved from Daemon.GraphDriverName, multiple calls +func (i *imageService) GraphDriverForOS(os string) string { + return i.layerStores[os].DriverName() +} + +// called from delete.go Daemon.cleanupContainer(), and Daemon.containerExport() +func (i *imageService) ReleaseContainerLayer(rwlayer layer.RWLayer, containerOS string) error { + metadata, err := i.layerStores[containerOS].ReleaseRWLayer(rwlayer) + layer.LogReleaseMetadata(metadata) + if err != nil && err != layer.ErrMountDoesNotExist && !os.IsNotExist(errors.Cause(err)) { + return errors.Wrapf(err, "driver %q failed to remove root filesystem", + i.layerStores[containerOS].DriverName()) + } + return nil +} + +// called from disk_usage.go +func (i *imageService) LayerDiskUsage(ctx context.Context) (int64, error) { + var allLayersSize int64 + layerRefs := i.getLayerRefs() + for _, ls := range i.layerStores { + allLayers := ls.Map() + for _, l := range allLayers { + select { + case <-ctx.Done(): + return allLayersSize, ctx.Err() + default: + size, err := l.DiffSize() + if err == nil { + if _, ok := layerRefs[l.ChainID()]; ok { + allLayersSize += size + } else { + logrus.Warnf("found leaked image layer %v", l.ChainID()) + } + } else { + logrus.Warnf("failed to get diff size for layer %v", l.ChainID()) + } + } + } + } + return allLayersSize, nil +} + +func (i *imageService) getLayerRefs() map[layer.ChainID]int { + tmpImages := i.imageStore.Map() + layerRefs := map[layer.ChainID]int{} + for id, img := range tmpImages { + dgst := digest.Digest(id) + if len(i.referenceStore.References(dgst)) == 0 && len(i.imageStore.Children(id)) != 0 { + continue + } + + rootFS := *img.RootFS + rootFS.DiffIDs = nil + for _, id := range img.RootFS.DiffIDs { + rootFS.Append(id) + chid := rootFS.ChainID() + layerRefs[chid]++ + } + } + + return layerRefs +} + +// LogImageEvent generates an event related to an image with only the default attributes. +func (i *imageService) LogImageEvent(imageID, refName, action string) { + i.LogImageEventWithAttributes(imageID, refName, action, map[string]string{}) +} + +// LogImageEventWithAttributes generates an event related to an image with specific given attributes. +func (i *imageService) LogImageEventWithAttributes(imageID, refName, action string, attributes map[string]string) { + img, err := i.GetImage(imageID) + if err == nil && img.Config != nil { + // image has not been removed yet. + // it could be missing if the event is `delete`. + copyAttributes(attributes, img.Config.Labels) + } + if refName != "" { + attributes["name"] = refName + } + actor := events.Actor{ + ID: imageID, + Attributes: attributes, + } + + i.eventsService.Log(action, events.ImageEventType, actor) } diff --git a/components/engine/daemon/build.go b/components/engine/daemon/image_builder.go similarity index 79% rename from components/engine/daemon/build.go rename to components/engine/daemon/image_builder.go index d1396fb50e..50a5212884 100644 --- a/components/engine/daemon/build.go +++ b/components/engine/daemon/image_builder.go @@ -8,9 +8,9 @@ import ( "github.com/docker/docker/api/types/backend" "github.com/docker/docker/builder" "github.com/docker/docker/image" + "github.com/docker/docker/image/cache" "github.com/docker/docker/layer" "github.com/docker/docker/pkg/containerfs" - "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/pkg/system" "github.com/docker/docker/registry" @@ -138,7 +138,7 @@ func newROLayerForImage(img *image.Image, layerStore layer.Store) (builder.ROLay } // TODO: could this use the regular daemon PullImage ? -func (daemon *Daemon) pullForBuilder(ctx context.Context, name string, authConfigs map[string]types.AuthConfig, output io.Writer, os string) (*image.Image, error) { +func (i *imageService) pullForBuilder(ctx context.Context, name string, authConfigs map[string]types.AuthConfig, output io.Writer, os string) (*image.Image, error) { ref, err := reference.ParseNormalizedNamed(name) if err != nil { return nil, err @@ -148,7 +148,7 @@ func (daemon *Daemon) pullForBuilder(ctx context.Context, name string, authConfi pullRegistryAuth := &types.AuthConfig{} if len(authConfigs) > 0 { // The request came with a full auth config, use it - repoInfo, err := daemon.RegistryService.ResolveRepository(ref) + repoInfo, err := i.registryService.ResolveRepository(ref) if err != nil { return nil, err } @@ -157,26 +157,26 @@ func (daemon *Daemon) pullForBuilder(ctx context.Context, name string, authConfi pullRegistryAuth = &resolvedConfig } - if err := daemon.pullImageWithReference(ctx, ref, os, nil, pullRegistryAuth, output); err != nil { + if err := i.pullImageWithReference(ctx, ref, os, nil, pullRegistryAuth, output); err != nil { return nil, err } - return daemon.GetImage(name) + return i.GetImage(name) } // GetImageAndReleasableLayer returns an image and releaseable layer for a reference or ID. // Every call to GetImageAndReleasableLayer MUST call releasableLayer.Release() to prevent // leaking of layers. -func (daemon *Daemon) GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (builder.Image, builder.ROLayer, error) { +func (i *imageService) GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (builder.Image, builder.ROLayer, error) { if refOrID == "" { if !system.IsOSSupported(opts.OS) { return nil, nil, system.ErrNotSupportedOperatingSystem } - layer, err := newROLayerForImage(nil, daemon.layerStores[opts.OS]) + layer, err := newROLayerForImage(nil, i.layerStores[opts.OS]) return nil, layer, err } if opts.PullOption != backend.PullOptionForcePull { - image, err := daemon.GetImage(refOrID) + image, err := i.GetImage(refOrID) if err != nil && opts.PullOption == backend.PullOptionNoPull { return nil, nil, err } @@ -185,41 +185,36 @@ func (daemon *Daemon) GetImageAndReleasableLayer(ctx context.Context, refOrID st if !system.IsOSSupported(image.OperatingSystem()) { return nil, nil, system.ErrNotSupportedOperatingSystem } - layer, err := newROLayerForImage(image, daemon.layerStores[image.OperatingSystem()]) + layer, err := newROLayerForImage(image, i.layerStores[image.OperatingSystem()]) return image, layer, err } } - image, err := daemon.pullForBuilder(ctx, refOrID, opts.AuthConfig, opts.Output, opts.OS) + image, err := i.pullForBuilder(ctx, refOrID, opts.AuthConfig, opts.Output, opts.OS) if err != nil { return nil, nil, err } if !system.IsOSSupported(image.OperatingSystem()) { return nil, nil, system.ErrNotSupportedOperatingSystem } - layer, err := newROLayerForImage(image, daemon.layerStores[image.OperatingSystem()]) + layer, err := newROLayerForImage(image, i.layerStores[image.OperatingSystem()]) return image, layer, err } // CreateImage creates a new image by adding a config and ID to the image store. // This is similar to LoadImage() except that it receives JSON encoded bytes of // an image instead of a tar archive. -func (daemon *Daemon) CreateImage(config []byte, parent string) (builder.Image, error) { - id, err := daemon.imageStore.Create(config) +func (i *imageService) CreateImage(config []byte, parent string) (builder.Image, error) { + id, err := i.imageStore.Create(config) if err != nil { return nil, errors.Wrapf(err, "failed to create image") } if parent != "" { - if err := daemon.imageStore.SetParent(id, image.ID(parent)); err != nil { + if err := i.imageStore.SetParent(id, image.ID(parent)); err != nil { return nil, errors.Wrapf(err, "failed to set parent %s", parent) } } - return daemon.imageStore.Get(id) -} - -// IDMappings returns uid/gid mappings for the builder -func (daemon *Daemon) IDMappings() *idtools.IDMappings { - return daemon.idMappings + return i.imageStore.Get(id) } diff --git a/components/engine/daemon/image_commit.go b/components/engine/daemon/image_commit.go index 48f890b80d..6188b52577 100644 --- a/components/engine/daemon/image_commit.go +++ b/components/engine/daemon/image_commit.go @@ -9,10 +9,12 @@ import ( "github.com/docker/docker/layer" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/system" + "github.com/pkg/errors" ) -func (daemon *Daemon) commitImage(c backend.CommitConfig) (image.ID, error) { - layerStore, ok := daemon.layerStores[c.ContainerOS] +// CommitImage creates a new image from a commit config +func (i *imageService) CommitImage(c backend.CommitConfig) (image.ID, error) { + layerStore, ok := i.layerStores[c.ContainerOS] if !ok { return "", system.ErrNotSupportedOperatingSystem } @@ -31,7 +33,7 @@ func (daemon *Daemon) commitImage(c backend.CommitConfig) (image.ID, error) { parent = new(image.Image) parent.RootFS = image.NewRootFS() } else { - parent, err = daemon.imageStore.Get(image.ID(c.ParentImageID)) + parent, err = i.imageStore.Get(image.ID(c.ParentImageID)) if err != nil { return "", err } @@ -56,13 +58,13 @@ func (daemon *Daemon) commitImage(c backend.CommitConfig) (image.ID, error) { return "", err } - id, err := daemon.imageStore.Create(config) + id, err := i.imageStore.Create(config) if err != nil { return "", err } if c.ParentImageID != "" { - if err := daemon.imageStore.SetParent(id, image.ID(c.ParentImageID)); err != nil { + if err := i.imageStore.SetParent(id, image.ID(c.ParentImageID)); err != nil { return "", err } } @@ -112,13 +114,14 @@ func exportContainerRw(layerStore layer.Store, id, mountLabel string) (arch io.R // * it doesn't log a container commit event // // This is a temporary shim. Should be removed when builder stops using commit. -func (daemon *Daemon) CommitBuildStep(c backend.CommitConfig) (image.ID, error) { - container, err := daemon.GetContainer(c.ContainerID) - if err != nil { - return "", err +func (i *imageService) CommitBuildStep(c backend.CommitConfig) (image.ID, error) { + container := i.containers.Get(c.ContainerID) + if container == nil { + // TODO: use typed error + return "", errors.Errorf("container not found: %s", c.ContainerID) } c.ContainerMountLabel = container.MountLabel c.ContainerOS = container.OS c.ParentImageID = string(container.ImageID) - return daemon.commitImage(c) + return i.CommitImage(c) } diff --git a/components/engine/daemon/image_delete.go b/components/engine/daemon/image_delete.go index c591390b4f..2615f05ca7 100644 --- a/components/engine/daemon/image_delete.go +++ b/components/engine/daemon/image_delete.go @@ -60,14 +60,11 @@ const ( // meaning any delete conflicts will cause the image to not be deleted and the // conflict will not be reported. // -// FIXME: remove ImageDelete's dependency on Daemon, then move to the graph -// package. This would require that we no longer need the daemon to determine -// whether images are being used by a stopped or running container. -func (daemon *Daemon) ImageDelete(imageRef string, force, prune bool) ([]types.ImageDeleteResponseItem, error) { +func (i *imageService) ImageDelete(imageRef string, force, prune bool) ([]types.ImageDeleteResponseItem, error) { start := time.Now() records := []types.ImageDeleteResponseItem{} - imgID, operatingSystem, err := daemon.GetImageIDAndOS(imageRef) + imgID, operatingSystem, err := i.GetImageIDAndOS(imageRef) if err != nil { return nil, err } @@ -75,7 +72,11 @@ func (daemon *Daemon) ImageDelete(imageRef string, force, prune bool) ([]types.I return nil, errors.Errorf("unable to delete image: %q", system.ErrNotSupportedOperatingSystem) } - repoRefs := daemon.referenceStore.References(imgID.Digest()) + repoRefs := i.referenceStore.References(imgID.Digest()) + + using := func(c *container.Container) bool { + return c.ImageID == imgID + } var removedRepositoryRef bool if !isImageIDPrefix(imgID.String(), imageRef) { @@ -84,7 +85,7 @@ func (daemon *Daemon) ImageDelete(imageRef string, force, prune bool) ([]types.I // true, there are multiple repository references to this // image, or there are no containers using the given reference. if !force && isSingleReference(repoRefs) { - if container := daemon.getContainerUsingImage(imgID); container != nil { + if container := i.containers.First(using); container != nil { // If we removed the repository reference then // this image would remain "dangling" and since // we really want to avoid that the client must @@ -99,17 +100,17 @@ func (daemon *Daemon) ImageDelete(imageRef string, force, prune bool) ([]types.I return nil, err } - parsedRef, err = daemon.removeImageRef(parsedRef) + parsedRef, err = i.removeImageRef(parsedRef) if err != nil { return nil, err } untaggedRecord := types.ImageDeleteResponseItem{Untagged: reference.FamiliarString(parsedRef)} - daemon.LogImageEvent(imgID.String(), imgID.String(), "untag") + i.LogImageEvent(imgID.String(), imgID.String(), "untag") records = append(records, untaggedRecord) - repoRefs = daemon.referenceStore.References(imgID.Digest()) + repoRefs = i.referenceStore.References(imgID.Digest()) // If a tag reference was removed and the only remaining // references to the same repository are digest references, @@ -127,7 +128,7 @@ func (daemon *Daemon) ImageDelete(imageRef string, force, prune bool) ([]types.I remainingRefs := []reference.Named{} for _, repoRef := range repoRefs { if _, repoRefIsCanonical := repoRef.(reference.Canonical); repoRefIsCanonical && parsedRef.Name() == repoRef.Name() { - if _, err := daemon.removeImageRef(repoRef); err != nil { + if _, err := i.removeImageRef(repoRef); err != nil { return records, err } @@ -157,25 +158,25 @@ func (daemon *Daemon) ImageDelete(imageRef string, force, prune bool) ([]types.I if !force { c |= conflictSoft &^ conflictActiveReference } - if conflict := daemon.checkImageDeleteConflict(imgID, c); conflict != nil { + if conflict := i.checkImageDeleteConflict(imgID, c); conflict != nil { return nil, conflict } for _, repoRef := range repoRefs { - parsedRef, err := daemon.removeImageRef(repoRef) + parsedRef, err := i.removeImageRef(repoRef) if err != nil { return nil, err } untaggedRecord := types.ImageDeleteResponseItem{Untagged: reference.FamiliarString(parsedRef)} - daemon.LogImageEvent(imgID.String(), imgID.String(), "untag") + i.LogImageEvent(imgID.String(), imgID.String(), "untag") records = append(records, untaggedRecord) } } } - if err := daemon.imageDeleteHelper(imgID, &records, force, prune, removedRepositoryRef); err != nil { + if err := i.imageDeleteHelper(imgID, &records, force, prune, removedRepositoryRef); err != nil { return nil, err } @@ -223,26 +224,18 @@ func isImageIDPrefix(imageID, possiblePrefix string) bool { return false } -// getContainerUsingImage returns a container that was created using the given -// imageID. Returns nil if there is no such container. -func (daemon *Daemon) getContainerUsingImage(imageID image.ID) *container.Container { - return daemon.containers.First(func(c *container.Container) bool { - return c.ImageID == imageID - }) -} - // removeImageRef attempts to parse and remove the given image reference from // this daemon's store of repository tag/digest references. The given // repositoryRef must not be an image ID but a repository name followed by an // optional tag or digest reference. If tag or digest is omitted, the default // tag is used. Returns the resolved image reference and an error. -func (daemon *Daemon) removeImageRef(ref reference.Named) (reference.Named, error) { +func (i *imageService) removeImageRef(ref reference.Named) (reference.Named, error) { ref = reference.TagNameOnly(ref) // Ignore the boolean value returned, as far as we're concerned, this // is an idempotent operation and it's okay if the reference didn't // exist in the first place. - _, err := daemon.referenceStore.Delete(ref) + _, err := i.referenceStore.Delete(ref) return ref, err } @@ -252,18 +245,18 @@ func (daemon *Daemon) removeImageRef(ref reference.Named) (reference.Named, erro // on the first encountered error. Removed references are logged to this // daemon's event service. An "Untagged" types.ImageDeleteResponseItem is added to the // given list of records. -func (daemon *Daemon) removeAllReferencesToImageID(imgID image.ID, records *[]types.ImageDeleteResponseItem) error { - imageRefs := daemon.referenceStore.References(imgID.Digest()) +func (i *imageService) removeAllReferencesToImageID(imgID image.ID, records *[]types.ImageDeleteResponseItem) error { + imageRefs := i.referenceStore.References(imgID.Digest()) for _, imageRef := range imageRefs { - parsedRef, err := daemon.removeImageRef(imageRef) + parsedRef, err := i.removeImageRef(imageRef) if err != nil { return err } untaggedRecord := types.ImageDeleteResponseItem{Untagged: reference.FamiliarString(parsedRef)} - daemon.LogImageEvent(imgID.String(), imgID.String(), "untag") + i.LogImageEvent(imgID.String(), imgID.String(), "untag") *records = append(*records, untaggedRecord) } @@ -303,15 +296,15 @@ func (idc *imageDeleteConflict) Conflict() {} // conflict is encountered, it will be returned immediately without deleting // the image. If quiet is true, any encountered conflicts will be ignored and // the function will return nil immediately without deleting the image. -func (daemon *Daemon) imageDeleteHelper(imgID image.ID, records *[]types.ImageDeleteResponseItem, force, prune, quiet bool) error { +func (i *imageService) imageDeleteHelper(imgID image.ID, records *[]types.ImageDeleteResponseItem, force, prune, quiet bool) error { // First, determine if this image has any conflicts. Ignore soft conflicts // if force is true. c := conflictHard if !force { c |= conflictSoft } - if conflict := daemon.checkImageDeleteConflict(imgID, c); conflict != nil { - if quiet && (!daemon.imageIsDangling(imgID) || conflict.used) { + if conflict := i.checkImageDeleteConflict(imgID, c); conflict != nil { + if quiet && (!i.imageIsDangling(imgID) || conflict.used) { // Ignore conflicts UNLESS the image is "dangling" or not being used in // which case we want the user to know. return nil @@ -322,23 +315,23 @@ func (daemon *Daemon) imageDeleteHelper(imgID image.ID, records *[]types.ImageDe return conflict } - parent, err := daemon.imageStore.GetParent(imgID) + parent, err := i.imageStore.GetParent(imgID) if err != nil { // There may be no parent parent = "" } // Delete all repository tag/digest references to this image. - if err := daemon.removeAllReferencesToImageID(imgID, records); err != nil { + if err := i.removeAllReferencesToImageID(imgID, records); err != nil { return err } - removedLayers, err := daemon.imageStore.Delete(imgID) + removedLayers, err := i.imageStore.Delete(imgID) if err != nil { return err } - daemon.LogImageEvent(imgID.String(), imgID.String(), "delete") + i.LogImageEvent(imgID.String(), imgID.String(), "delete") *records = append(*records, types.ImageDeleteResponseItem{Deleted: imgID.String()}) for _, removedLayer := range removedLayers { *records = append(*records, types.ImageDeleteResponseItem{Deleted: removedLayer.ChainID.String()}) @@ -353,7 +346,7 @@ func (daemon *Daemon) imageDeleteHelper(imgID image.ID, records *[]types.ImageDe // either running or stopped). // Do not force prunings, but do so quietly (stopping on any encountered // conflicts). - return daemon.imageDeleteHelper(parent, records, false, true, true) + return i.imageDeleteHelper(parent, records, false, true, true) } // checkImageDeleteConflict determines whether there are any conflicts @@ -362,9 +355,9 @@ func (daemon *Daemon) imageDeleteHelper(imgID image.ID, records *[]types.ImageDe // using the image. A soft conflict is any tags/digest referencing the given // image or any stopped container using the image. If ignoreSoftConflicts is // true, this function will not check for soft conflict conditions. -func (daemon *Daemon) checkImageDeleteConflict(imgID image.ID, mask conflictType) *imageDeleteConflict { +func (i *imageService) checkImageDeleteConflict(imgID image.ID, mask conflictType) *imageDeleteConflict { // Check if the image has any descendant images. - if mask&conflictDependentChild != 0 && len(daemon.imageStore.Children(imgID)) > 0 { + if mask&conflictDependentChild != 0 && len(i.imageStore.Children(imgID)) > 0 { return &imageDeleteConflict{ hard: true, imgID: imgID, @@ -377,7 +370,7 @@ func (daemon *Daemon) checkImageDeleteConflict(imgID image.ID, mask conflictType running := func(c *container.Container) bool { return c.IsRunning() && c.ImageID == imgID } - if container := daemon.containers.First(running); container != nil { + if container := i.containers.First(running); container != nil { return &imageDeleteConflict{ imgID: imgID, hard: true, @@ -388,7 +381,7 @@ func (daemon *Daemon) checkImageDeleteConflict(imgID image.ID, mask conflictType } // Check if any repository tags/digest reference this image. - if mask&conflictActiveReference != 0 && len(daemon.referenceStore.References(imgID.Digest())) > 0 { + if mask&conflictActiveReference != 0 && len(i.referenceStore.References(imgID.Digest())) > 0 { return &imageDeleteConflict{ imgID: imgID, message: "image is referenced in multiple repositories", @@ -400,7 +393,7 @@ func (daemon *Daemon) checkImageDeleteConflict(imgID image.ID, mask conflictType stopped := func(c *container.Container) bool { return !c.IsRunning() && c.ImageID == imgID } - if container := daemon.containers.First(stopped); container != nil { + if container := i.containers.First(stopped); container != nil { return &imageDeleteConflict{ imgID: imgID, used: true, @@ -415,6 +408,6 @@ func (daemon *Daemon) checkImageDeleteConflict(imgID image.ID, mask conflictType // imageIsDangling returns whether the given image is "dangling" which means // that there are no repository references to the given image and it has no // child images. -func (daemon *Daemon) imageIsDangling(imgID image.ID) bool { - return !(len(daemon.referenceStore.References(imgID.Digest())) > 0 || len(daemon.imageStore.Children(imgID)) > 0) +func (i *imageService) imageIsDangling(imgID image.ID) bool { + return !(len(i.referenceStore.References(imgID.Digest())) > 0 || len(i.imageStore.Children(imgID)) > 0) } diff --git a/components/engine/daemon/image_exporter.go b/components/engine/daemon/image_exporter.go index d746236af9..575565fc25 100644 --- a/components/engine/daemon/image_exporter.go +++ b/components/engine/daemon/image_exporter.go @@ -11,15 +11,15 @@ import ( // stream. All images with the given tag and all versions containing // the same tag are exported. names is the set of tags to export, and // outStream is the writer which the images are written to. -func (daemon *Daemon) ExportImage(names []string, outStream io.Writer) error { - imageExporter := tarexport.NewTarExporter(daemon.imageStore, daemon.layerStores, daemon.referenceStore, daemon) +func (i *imageService) ExportImage(names []string, outStream io.Writer) error { + imageExporter := tarexport.NewTarExporter(i.imageStore, i.layerStores, i.referenceStore, i) return imageExporter.Save(names, outStream) } // LoadImage uploads a set of images into the repository. This is the // complement of ImageExport. The input stream is an uncompressed tar // ball containing images and metadata. -func (daemon *Daemon) LoadImage(inTar io.ReadCloser, outStream io.Writer, quiet bool) error { - imageExporter := tarexport.NewTarExporter(daemon.imageStore, daemon.layerStores, daemon.referenceStore, daemon) +func (i *imageService) LoadImage(inTar io.ReadCloser, outStream io.Writer, quiet bool) error { + imageExporter := tarexport.NewTarExporter(i.imageStore, i.layerStores, i.referenceStore, i) return imageExporter.Load(inTar, outStream, quiet) } diff --git a/components/engine/daemon/getsize_unix.go b/components/engine/daemon/image_getsize_unix.go similarity index 77% rename from components/engine/daemon/getsize_unix.go rename to components/engine/daemon/image_getsize_unix.go index 292108307c..faa1edd843 100644 --- a/components/engine/daemon/getsize_unix.go +++ b/components/engine/daemon/image_getsize_unix.go @@ -9,7 +9,7 @@ import ( ) // getSize returns the real size & virtual size of the container. -func (daemon *Daemon) getSize(containerID string) (int64, int64) { +func (i *imageService) GetContainerLayerSize(containerID string) (int64, int64) { var ( sizeRw, sizeRootfs int64 err error @@ -17,17 +17,17 @@ func (daemon *Daemon) getSize(containerID string) (int64, int64) { // Safe to index by runtime.GOOS as Unix hosts don't support multiple // container operating systems. - rwlayer, err := daemon.layerStores[runtime.GOOS].GetRWLayer(containerID) + rwlayer, err := i.layerStores[runtime.GOOS].GetRWLayer(containerID) if err != nil { logrus.Errorf("Failed to compute size of container rootfs %v: %v", containerID, err) return sizeRw, sizeRootfs } - defer daemon.layerStores[runtime.GOOS].ReleaseRWLayer(rwlayer) + defer i.layerStores[runtime.GOOS].ReleaseRWLayer(rwlayer) sizeRw, err = rwlayer.Size() if err != nil { logrus.Errorf("Driver %s couldn't return diff size of container %s: %s", - daemon.GraphDriverName(runtime.GOOS), containerID, err) + i.layerStores[runtime.GOOS].DriverName(), containerID, err) // FIXME: GetSize should return an error. Not changing it now in case // there is a side-effect. sizeRw = -1 diff --git a/components/engine/daemon/image_history.go b/components/engine/daemon/image_history.go index 58dd59d3a7..621e8cec05 100644 --- a/components/engine/daemon/image_history.go +++ b/components/engine/daemon/image_history.go @@ -11,9 +11,9 @@ import ( // ImageHistory returns a slice of ImageHistory structures for the specified image // name by walking the image lineage. -func (daemon *Daemon) ImageHistory(name string) ([]*image.HistoryResponseItem, error) { +func (i *imageService) ImageHistory(name string) ([]*image.HistoryResponseItem, error) { start := time.Now() - img, err := daemon.GetImage(name) + img, err := i.GetImage(name) if err != nil { return nil, err } @@ -33,12 +33,12 @@ func (daemon *Daemon) ImageHistory(name string) ([]*image.HistoryResponseItem, e } rootFS.Append(img.RootFS.DiffIDs[layerCounter]) - l, err := daemon.layerStores[img.OperatingSystem()].Get(rootFS.ChainID()) + l, err := i.layerStores[img.OperatingSystem()].Get(rootFS.ChainID()) if err != nil { return nil, err } layerSize, err = l.DiffSize() - layer.ReleaseAndLog(daemon.layerStores[img.OperatingSystem()], l) + layer.ReleaseAndLog(i.layerStores[img.OperatingSystem()], l) if err != nil { return nil, err } @@ -62,7 +62,7 @@ func (daemon *Daemon) ImageHistory(name string) ([]*image.HistoryResponseItem, e h.ID = id.String() var tags []string - for _, r := range daemon.referenceStore.References(id.Digest()) { + for _, r := range i.referenceStore.References(id.Digest()) { if _, ok := r.(reference.NamedTagged); ok { tags = append(tags, reference.FamiliarString(r)) } @@ -74,7 +74,7 @@ func (daemon *Daemon) ImageHistory(name string) ([]*image.HistoryResponseItem, e if id == "" { break } - histImg, err = daemon.GetImage(id.String()) + histImg, err = i.GetImage(id.String()) if err != nil { break } diff --git a/components/engine/daemon/import.go b/components/engine/daemon/image_import.go similarity index 87% rename from components/engine/daemon/import.go rename to components/engine/daemon/image_import.go index 01b7906888..f9bf0b119b 100644 --- a/components/engine/daemon/import.go +++ b/components/engine/daemon/image_import.go @@ -27,7 +27,7 @@ import ( // inConfig (if src is "-"), or from a URI specified in src. Progress output is // written to outStream. Repository and tag names can optionally be given in // the repo and tag arguments, respectively. -func (daemon *Daemon) ImportImage(src string, repository, os string, tag string, msg string, inConfig io.ReadCloser, outStream io.Writer, changes []string) error { +func (i *imageService) ImportImage(src string, repository, os string, tag string, msg string, inConfig io.ReadCloser, outStream io.Writer, changes []string) error { var ( rc io.ReadCloser resp *http.Response @@ -91,11 +91,11 @@ func (daemon *Daemon) ImportImage(src string, repository, os string, tag string, if err != nil { return err } - l, err := daemon.layerStores[os].Register(inflatedLayerData, "") + l, err := i.layerStores[os].Register(inflatedLayerData, "") if err != nil { return err } - defer layer.ReleaseAndLog(daemon.layerStores[os], l) + defer layer.ReleaseAndLog(i.layerStores[os], l) created := time.Now().UTC() imgConfig, err := json.Marshal(&image.Image{ @@ -120,19 +120,19 @@ func (daemon *Daemon) ImportImage(src string, repository, os string, tag string, return err } - id, err := daemon.imageStore.Create(imgConfig) + id, err := i.imageStore.Create(imgConfig) if err != nil { return err } // FIXME: connect with commit code and call refstore directly if newRef != nil { - if err := daemon.TagImageWithReference(id, newRef); err != nil { + if err := i.TagImageWithReference(id, newRef); err != nil { return err } } - daemon.LogImageEvent(id.String(), id.String(), "import") + i.LogImageEvent(id.String(), id.String(), "import") outStream.Write(streamformatter.FormatStatus("", id.String())) return nil } diff --git a/components/engine/daemon/image_inspect.go b/components/engine/daemon/image_inspect.go index 731057cf34..c00ceb343c 100644 --- a/components/engine/daemon/image_inspect.go +++ b/components/engine/daemon/image_inspect.go @@ -13,15 +13,15 @@ import ( // LookupImage looks up an image by name and returns it as an ImageInspect // structure. -func (daemon *Daemon) LookupImage(name string) (*types.ImageInspect, error) { - img, err := daemon.GetImage(name) +func (i *imageService) LookupImage(name string) (*types.ImageInspect, error) { + img, err := i.GetImage(name) if err != nil { return nil, errors.Wrapf(err, "no such image: %s", name) } if !system.IsOSSupported(img.OperatingSystem()) { return nil, system.ErrNotSupportedOperatingSystem } - refs := daemon.referenceStore.References(img.ID().Digest()) + refs := i.referenceStore.References(img.ID().Digest()) repoTags := []string{} repoDigests := []string{} for _, ref := range refs { @@ -37,11 +37,11 @@ func (daemon *Daemon) LookupImage(name string) (*types.ImageInspect, error) { var layerMetadata map[string]string layerID := img.RootFS.ChainID() if layerID != "" { - l, err := daemon.layerStores[img.OperatingSystem()].Get(layerID) + l, err := i.layerStores[img.OperatingSystem()].Get(layerID) if err != nil { return nil, err } - defer layer.ReleaseAndLog(daemon.layerStores[img.OperatingSystem()], l) + defer layer.ReleaseAndLog(i.layerStores[img.OperatingSystem()], l) size, err = l.Size() if err != nil { return nil, err @@ -58,7 +58,7 @@ func (daemon *Daemon) LookupImage(name string) (*types.ImageInspect, error) { comment = img.History[len(img.History)-1].Comment } - lastUpdated, err := daemon.imageStore.GetLastUpdated(img.ID()) + lastUpdated, err := i.imageStore.GetLastUpdated(img.ID()) if err != nil { return nil, err } @@ -86,7 +86,7 @@ func (daemon *Daemon) LookupImage(name string) (*types.ImageInspect, error) { }, } - imageInspect.GraphDriver.Name = daemon.GraphDriverName(img.OperatingSystem()) + imageInspect.GraphDriver.Name = i.GraphDriverForOS(img.OperatingSystem()) imageInspect.GraphDriver.Data = layerMetadata return imageInspect, nil diff --git a/components/engine/daemon/image_prune.go b/components/engine/daemon/image_prune.go index 193c3aa0ac..92399a38be 100644 --- a/components/engine/daemon/image_prune.go +++ b/components/engine/daemon/image_prune.go @@ -22,11 +22,11 @@ var imagesAcceptedFilters = map[string]bool{ } // ImagesPrune removes unused images -func (daemon *Daemon) ImagesPrune(ctx context.Context, pruneFilters filters.Args) (*types.ImagesPruneReport, error) { - if !atomic.CompareAndSwapInt32(&daemon.pruneRunning, 0, 1) { +func (i *imageService) ImagesPrune(ctx context.Context, pruneFilters filters.Args) (*types.ImagesPruneReport, error) { + if !atomic.CompareAndSwapInt32(&i.pruneRunning, 0, 1) { return nil, errPruneRunning } - defer atomic.StoreInt32(&daemon.pruneRunning, 0) + defer atomic.StoreInt32(&i.pruneRunning, 0) // make sure that only accepted filters have been received err := pruneFilters.Validate(imagesAcceptedFilters) @@ -52,14 +52,14 @@ func (daemon *Daemon) ImagesPrune(ctx context.Context, pruneFilters filters.Args var allImages map[image.ID]*image.Image if danglingOnly { - allImages = daemon.imageStore.Heads() + allImages = i.imageStore.Heads() } else { - allImages = daemon.imageStore.Map() + allImages = i.imageStore.Map() } // Filter intermediary images and get their unique size allLayers := make(map[layer.ChainID]layer.Layer) - for _, ls := range daemon.layerStores { + for _, ls := range i.layerStores { for k, v := range ls.Map() { allLayers[k] = v } @@ -71,7 +71,7 @@ func (daemon *Daemon) ImagesPrune(ctx context.Context, pruneFilters filters.Args return nil, ctx.Err() default: dgst := digest.Digest(id) - if len(daemon.referenceStore.References(dgst)) == 0 && len(daemon.imageStore.Children(id)) != 0 { + if len(i.referenceStore.References(dgst)) == 0 && len(i.imageStore.Children(id)) != 0 { continue } if !until.IsZero() && img.Created.After(until) { @@ -96,7 +96,7 @@ deleteImagesLoop: } deletedImages := []types.ImageDeleteResponseItem{} - refs := daemon.referenceStore.References(id.Digest()) + refs := i.referenceStore.References(dgst) if len(refs) > 0 { shouldDelete := !danglingOnly if !shouldDelete { @@ -114,7 +114,7 @@ deleteImagesLoop: if shouldDelete { for _, ref := range refs { - imgDel, err := daemon.ImageDelete(ref.String(), false, true) + imgDel, err := i.ImageDelete(ref.String(), false, true) if imageDeleteFailed(ref.String(), err) { continue } @@ -123,7 +123,7 @@ deleteImagesLoop: } } else { hex := id.Digest().Hex() - imgDel, err := daemon.ImageDelete(hex, false, true) + imgDel, err := i.ImageDelete(hex, false, true) if imageDeleteFailed(hex, err) { continue } diff --git a/components/engine/daemon/image_pull.go b/components/engine/daemon/image_pull.go index cd3ef90b92..f6606cb38c 100644 --- a/components/engine/daemon/image_pull.go +++ b/components/engine/daemon/image_pull.go @@ -19,7 +19,7 @@ import ( // PullImage initiates a pull operation. image is the repository name to pull, and // tag may be either empty, or indicate a specific tag to pull. -func (daemon *Daemon) PullImage(ctx context.Context, image, tag, os string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error { +func (i *imageService) PullImage(ctx context.Context, image, tag, os string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error { // Special case: "pull -a" may send an image name with a // trailing :. This is ugly, but let's not break API // compatibility. @@ -44,10 +44,10 @@ func (daemon *Daemon) PullImage(ctx context.Context, image, tag, os string, meta } } - return daemon.pullImageWithReference(ctx, ref, os, metaHeaders, authConfig, outStream) + return i.pullImageWithReference(ctx, ref, os, metaHeaders, authConfig, outStream) } -func (daemon *Daemon) pullImageWithReference(ctx context.Context, ref reference.Named, os string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error { +func (i *imageService) pullImageWithReference(ctx context.Context, ref reference.Named, os string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error { // Include a buffer so that slow client connections don't affect // transfer performance. progressChan := make(chan progress.Progress, 100) @@ -71,13 +71,13 @@ func (daemon *Daemon) pullImageWithReference(ctx context.Context, ref reference. MetaHeaders: metaHeaders, AuthConfig: authConfig, ProgressOutput: progress.ChanOutput(progressChan), - RegistryService: daemon.RegistryService, - ImageEventLogger: daemon.LogImageEvent, - MetadataStore: daemon.distributionMetadataStore, - ImageStore: distribution.NewImageConfigStoreFromStore(daemon.imageStore), - ReferenceStore: daemon.referenceStore, + RegistryService: i.registryService, + ImageEventLogger: i.LogImageEvent, + MetadataStore: i.distributionMetadataStore, + ImageStore: distribution.NewImageConfigStoreFromStore(i.imageStore), + ReferenceStore: i.referenceStore, }, - DownloadManager: daemon.downloadManager, + DownloadManager: i.downloadManager, Schema2Types: distribution.ImageTypes, OS: os, } @@ -89,9 +89,9 @@ func (daemon *Daemon) pullImageWithReference(ctx context.Context, ref reference. } // GetRepository returns a repository from the registry. -func (daemon *Daemon) GetRepository(ctx context.Context, ref reference.Named, authConfig *types.AuthConfig) (dist.Repository, bool, error) { +func (i *imageService) GetRepository(ctx context.Context, ref reference.Named, authConfig *types.AuthConfig) (dist.Repository, bool, error) { // get repository info - repoInfo, err := daemon.RegistryService.ResolveRepository(ref) + repoInfo, err := i.registryService.ResolveRepository(ref) if err != nil { return nil, false, err } @@ -101,7 +101,7 @@ func (daemon *Daemon) GetRepository(ctx context.Context, ref reference.Named, au } // get endpoints - endpoints, err := daemon.RegistryService.LookupPullEndpoints(reference.Domain(repoInfo.Name)) + endpoints, err := i.registryService.LookupPullEndpoints(reference.Domain(repoInfo.Name)) if err != nil { return nil, false, err } diff --git a/components/engine/daemon/image_push.go b/components/engine/daemon/image_push.go index b5bdb74fb6..23c2d1e2eb 100644 --- a/components/engine/daemon/image_push.go +++ b/components/engine/daemon/image_push.go @@ -13,7 +13,7 @@ import ( ) // PushImage initiates a push operation on the repository named localName. -func (daemon *Daemon) PushImage(ctx context.Context, image, tag string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error { +func (i *imageService) PushImage(ctx context.Context, image, tag string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error { ref, err := reference.ParseNormalizedNamed(image) if err != nil { return err @@ -44,16 +44,16 @@ func (daemon *Daemon) PushImage(ctx context.Context, image, tag string, metaHead MetaHeaders: metaHeaders, AuthConfig: authConfig, ProgressOutput: progress.ChanOutput(progressChan), - RegistryService: daemon.RegistryService, - ImageEventLogger: daemon.LogImageEvent, - MetadataStore: daemon.distributionMetadataStore, - ImageStore: distribution.NewImageConfigStoreFromStore(daemon.imageStore), - ReferenceStore: daemon.referenceStore, + RegistryService: i.registryService, + ImageEventLogger: i.LogImageEvent, + MetadataStore: i.distributionMetadataStore, + ImageStore: distribution.NewImageConfigStoreFromStore(i.imageStore), + ReferenceStore: i.referenceStore, }, ConfigMediaType: schema2.MediaTypeImageConfig, - LayerStores: distribution.NewLayerProvidersFromStores(daemon.layerStores), - TrustKey: daemon.trustKey, - UploadManager: daemon.uploadManager, + LayerStores: distribution.NewLayerProvidersFromStores(i.layerStores), + TrustKey: i.trustKey, + UploadManager: i.uploadManager, } err = distribution.Push(ctx, ref, imagePushConfig) diff --git a/components/engine/daemon/search.go b/components/engine/daemon/image_search.go similarity index 87% rename from components/engine/daemon/search.go rename to components/engine/daemon/image_search.go index 6d137d3d65..101bd2e306 100644 --- a/components/engine/daemon/search.go +++ b/components/engine/daemon/image_search.go @@ -19,7 +19,10 @@ var acceptedSearchFilterTags = map[string]bool{ // SearchRegistryForImages queries the registry for images matching // term. authConfig is used to login. -func (daemon *Daemon) SearchRegistryForImages(ctx context.Context, filtersArgs string, term string, limit int, +// +// TODO: this could be implemented in a registry service instead of the image +// service. +func (i *imageService) SearchRegistryForImages(ctx context.Context, filtersArgs string, term string, limit int, authConfig *types.AuthConfig, headers map[string][]string) (*registrytypes.SearchResults, error) { @@ -60,7 +63,7 @@ func (daemon *Daemon) SearchRegistryForImages(ctx context.Context, filtersArgs s } } - unfilteredResult, err := daemon.RegistryService.Search(ctx, term, limit, authConfig, dockerversion.DockerUserAgent(ctx), headers) + unfilteredResult, err := i.registryService.Search(ctx, term, limit, authConfig, dockerversion.DockerUserAgent(ctx), headers) if err != nil { return nil, err } diff --git a/components/engine/daemon/image_tag.go b/components/engine/daemon/image_tag.go index 56b325f66d..6e929437f5 100644 --- a/components/engine/daemon/image_tag.go +++ b/components/engine/daemon/image_tag.go @@ -7,8 +7,8 @@ import ( // TagImage creates the tag specified by newTag, pointing to the image named // imageName (alternatively, imageName can also be an image ID). -func (daemon *Daemon) TagImage(imageName, repository, tag string) (string, error) { - imageID, _, err := daemon.GetImageIDAndOS(imageName) +func (i *imageService) TagImage(imageName, repository, tag string) (string, error) { + imageID, _, err := i.GetImageIDAndOS(imageName) if err != nil { return "", err } @@ -23,19 +23,19 @@ func (daemon *Daemon) TagImage(imageName, repository, tag string) (string, error } } - err = daemon.TagImageWithReference(imageID, newTag) + err = i.TagImageWithReference(imageID, newTag) return reference.FamiliarString(newTag), err } // TagImageWithReference adds the given reference to the image ID provided. -func (daemon *Daemon) TagImageWithReference(imageID image.ID, newTag reference.Named) error { - if err := daemon.referenceStore.AddTag(newTag, imageID.Digest(), true); err != nil { +func (i *imageService) TagImageWithReference(imageID image.ID, newTag reference.Named) error { + if err := i.referenceStore.AddTag(newTag, imageID.Digest(), true); err != nil { return err } - if err := daemon.imageStore.SetLastUpdated(imageID); err != nil { + if err := i.imageStore.SetLastUpdated(imageID); err != nil { return err } - daemon.LogImageEvent(imageID.String(), reference.FamiliarString(newTag), "tag") + i.LogImageEvent(imageID.String(), reference.FamiliarString(newTag), "tag") return nil } diff --git a/components/engine/daemon/images.go b/components/engine/daemon/images.go index 7e2a9418c7..903583bbcb 100644 --- a/components/engine/daemon/images.go +++ b/components/engine/daemon/images.go @@ -34,8 +34,8 @@ func (r byCreated) Swap(i, j int) { r[i], r[j] = r[j], r[i] } func (r byCreated) Less(i, j int) bool { return r[i].Created < r[j].Created } // Map returns a map of all images in the ImageStore -func (daemon *Daemon) Map() map[image.ID]*image.Image { - return daemon.imageStore.Map() +func (i *imageService) Map() map[image.ID]*image.Image { + return i.imageStore.Map() } // Images returns a filtered list of images. filterArgs is a JSON-encoded set @@ -43,7 +43,7 @@ func (daemon *Daemon) Map() map[image.ID]*image.Image { // filter is a shell glob string applied to repository names. The argument // named all controls whether all images in the graph are filtered, or just // the heads. -func (daemon *Daemon) Images(imageFilters filters.Args, all bool, withExtraAttrs bool) ([]*types.ImageSummary, error) { +func (i *imageService) Images(imageFilters filters.Args, all bool, withExtraAttrs bool) ([]*types.ImageSummary, error) { var ( allImages map[image.ID]*image.Image err error @@ -62,14 +62,14 @@ func (daemon *Daemon) Images(imageFilters filters.Args, all bool, withExtraAttrs } } if danglingOnly { - allImages = daemon.imageStore.Heads() + allImages = i.imageStore.Heads() } else { - allImages = daemon.imageStore.Map() + allImages = i.imageStore.Map() } var beforeFilter, sinceFilter *image.Image err = imageFilters.WalkValues("before", func(value string) error { - beforeFilter, err = daemon.GetImage(value) + beforeFilter, err = i.GetImage(value) return err }) if err != nil { @@ -77,7 +77,7 @@ func (daemon *Daemon) Images(imageFilters filters.Args, all bool, withExtraAttrs } err = imageFilters.WalkValues("since", func(value string) error { - sinceFilter, err = daemon.GetImage(value) + sinceFilter, err = i.GetImage(value) return err }) if err != nil { @@ -124,7 +124,7 @@ func (daemon *Daemon) Images(imageFilters filters.Args, all bool, withExtraAttrs layerID := img.RootFS.ChainID() var size int64 if layerID != "" { - l, err := daemon.layerStores[img.OperatingSystem()].Get(layerID) + l, err := i.layerStores[img.OperatingSystem()].Get(layerID) if err != nil { // The layer may have been deleted between the call to `Map()` or // `Heads()` and the call to `Get()`, so we just ignore this error @@ -135,7 +135,7 @@ func (daemon *Daemon) Images(imageFilters filters.Args, all bool, withExtraAttrs } size, err = l.Size() - layer.ReleaseAndLog(daemon.layerStores[img.OperatingSystem()], l) + layer.ReleaseAndLog(i.layerStores[img.OperatingSystem()], l) if err != nil { return nil, err } @@ -143,7 +143,7 @@ func (daemon *Daemon) Images(imageFilters filters.Args, all bool, withExtraAttrs newImage := newImage(img, size) - for _, ref := range daemon.referenceStore.References(id.Digest()) { + for _, ref := range i.referenceStore.References(id.Digest()) { if imageFilters.Contains("reference") { var found bool var matchErr error @@ -165,7 +165,7 @@ func (daemon *Daemon) Images(imageFilters filters.Args, all bool, withExtraAttrs } } if newImage.RepoDigests == nil && newImage.RepoTags == nil { - if all || len(daemon.imageStore.Children(id)) == 0 { + if all || len(i.imageStore.Children(id)) == 0 { if imageFilters.Contains("dangling") && !danglingOnly { //dangling=false case, so dangling image is not needed @@ -186,8 +186,8 @@ func (daemon *Daemon) Images(imageFilters filters.Args, all bool, withExtraAttrs if withExtraAttrs { // lazily init variables if imagesMap == nil { - allContainers = daemon.List() - allLayers = daemon.layerStores[img.OperatingSystem()].Map() + allContainers = i.containers.List() + allLayers = i.layerStores[img.OperatingSystem()].Map() imagesMap = make(map[*image.Image]*types.ImageSummary) layerRefs = make(map[layer.ChainID]int) } @@ -249,20 +249,20 @@ func (daemon *Daemon) Images(imageFilters filters.Args, all bool, withExtraAttrs // This new image contains only the layers from it's parent + 1 extra layer which contains the diff of all the layers in between. // The existing image(s) is not destroyed. // If no parent is specified, a new image with the diff of all the specified image's layers merged into a new layer that has no parents. -func (daemon *Daemon) SquashImage(id, parent string) (string, error) { +func (i *imageService) SquashImage(id, parent string) (string, error) { var ( img *image.Image err error ) - if img, err = daemon.imageStore.Get(image.ID(id)); err != nil { + if img, err = i.imageStore.Get(image.ID(id)); err != nil { return "", err } var parentImg *image.Image var parentChainID layer.ChainID if len(parent) != 0 { - parentImg, err = daemon.imageStore.Get(image.ID(parent)) + parentImg, err = i.imageStore.Get(image.ID(parent)) if err != nil { return "", errors.Wrap(err, "error getting specified parent layer") } @@ -272,11 +272,11 @@ func (daemon *Daemon) SquashImage(id, parent string) (string, error) { parentImg = &image.Image{RootFS: rootFS} } - l, err := daemon.layerStores[img.OperatingSystem()].Get(img.RootFS.ChainID()) + l, err := i.layerStores[img.OperatingSystem()].Get(img.RootFS.ChainID()) if err != nil { return "", errors.Wrap(err, "error getting image layer") } - defer daemon.layerStores[img.OperatingSystem()].Release(l) + defer i.layerStores[img.OperatingSystem()].Release(l) ts, err := l.TarStreamFrom(parentChainID) if err != nil { @@ -284,11 +284,11 @@ func (daemon *Daemon) SquashImage(id, parent string) (string, error) { } defer ts.Close() - newL, err := daemon.layerStores[img.OperatingSystem()].Register(ts, parentChainID) + newL, err := i.layerStores[img.OperatingSystem()].Register(ts, parentChainID) if err != nil { return "", errors.Wrap(err, "error registering layer") } - defer daemon.layerStores[img.OperatingSystem()].Release(newL) + defer i.layerStores[img.OperatingSystem()].Release(newL) newImage := *img newImage.RootFS = nil @@ -323,7 +323,7 @@ func (daemon *Daemon) SquashImage(id, parent string) (string, error) { return "", errors.Wrap(err, "error marshalling image config") } - newImgID, err := daemon.imageStore.Create(b) + newImgID, err := i.imageStore.Create(b) if err != nil { return "", errors.Wrap(err, "error creating new image after squash") } diff --git a/components/engine/daemon/info.go b/components/engine/daemon/info.go index c73734ded7..24940be16a 100644 --- a/components/engine/daemon/info.go +++ b/components/engine/daemon/info.go @@ -80,8 +80,9 @@ func (daemon *Daemon) SystemInfo() (*types.Info, error) { var ds [][2]string drivers := "" + statuses := daemon.imageService.GraphDriverStatuses() for os, gd := range daemon.graphDrivers { - ds = append(ds, daemon.layerStores[os].DriverStatus()...) + ds = append(ds, statuses[os]...) drivers += gd if len(daemon.graphDrivers) > 1 { drivers += fmt.Sprintf(" (%s) ", os) @@ -95,7 +96,7 @@ func (daemon *Daemon) SystemInfo() (*types.Info, error) { ContainersRunning: cRunning, ContainersPaused: cPaused, ContainersStopped: cStopped, - Images: len(daemon.imageStore.Map()), + Images: daemon.imageService.CountImages(), Driver: drivers, DriverStatus: ds, Plugins: daemon.showPluginsInfo(), diff --git a/components/engine/daemon/inspect.go b/components/engine/daemon/inspect.go index 164e1aa2ae..f8dceb71ab 100644 --- a/components/engine/daemon/inspect.go +++ b/components/engine/daemon/inspect.go @@ -79,7 +79,7 @@ func (daemon *Daemon) ContainerInspectCurrent(name string, size bool) (*types.Co container.Unlock() if size { - sizeRw, sizeRootFs := daemon.getSize(base.ID) + sizeRw, sizeRootFs := daemon.imageService.GetContainerLayerSize(base.ID) base.SizeRw = &sizeRw base.SizeRootFs = &sizeRootFs } diff --git a/components/engine/daemon/list.go b/components/engine/daemon/list.go index e9c50071a0..254cb7253a 100644 --- a/components/engine/daemon/list.go +++ b/components/engine/daemon/list.go @@ -239,7 +239,7 @@ func (daemon *Daemon) reducePsContainer(container *container.Snapshot, ctx *list // release lock because size calculation is slow if ctx.Size { - sizeRw, sizeRootFs := daemon.getSize(newC.ID) + sizeRw, sizeRootFs := daemon.imageService.GetContainerLayerSize(newC.ID) newC.SizeRw = sizeRw newC.SizeRootFs = sizeRootFs } @@ -323,7 +323,7 @@ func (daemon *Daemon) foldFilter(view container.View, config *types.ContainerLis if psFilters.Contains("ancestor") { ancestorFilter = true psFilters.WalkValues("ancestor", func(ancestor string) error { - id, _, err := daemon.GetImageIDAndOS(ancestor) + id, _, err := daemon.imageService.GetImageIDAndOS(ancestor) if err != nil { logrus.Warnf("Error while looking up for image %v", ancestor) return nil @@ -333,7 +333,7 @@ func (daemon *Daemon) foldFilter(view container.View, config *types.ContainerLis return nil } // Then walk down the graph and put the imageIds in imagesFilter - populateImageFilterByParents(imagesFilter, id, daemon.imageStore.Children) + populateImageFilterByParents(imagesFilter, id, daemon.imageService.Children) return nil }) } @@ -591,7 +591,7 @@ func (daemon *Daemon) refreshImage(s *container.Snapshot, ctx *listContext) (*ty c := s.Container image := s.Image // keep the original ref if still valid (hasn't changed) if image != s.ImageID { - id, _, err := daemon.GetImageIDAndOS(image) + id, _, err := daemon.imageService.GetImageIDAndOS(image) if _, isDNE := err.(errImageDoesNotExist); err != nil && !isDNE { return nil, err } diff --git a/components/engine/daemon/oci_windows.go b/components/engine/daemon/oci_windows.go index e4b536f4e4..93d4ae2297 100644 --- a/components/engine/daemon/oci_windows.go +++ b/components/engine/daemon/oci_windows.go @@ -24,7 +24,7 @@ const ( ) func (daemon *Daemon) createSpec(c *container.Container) (*specs.Spec, error) { - img, err := daemon.GetImage(string(c.ImageID)) + img, err := daemon.imageService.GetImage(string(c.ImageID)) if err != nil { return nil, err } diff --git a/components/engine/daemon/prune.go b/components/engine/daemon/prune.go index f16e448ba8..9f42e49a2b 100644 --- a/components/engine/daemon/prune.go +++ b/components/engine/daemon/prune.go @@ -75,7 +75,7 @@ func (daemon *Daemon) ContainersPrune(ctx context.Context, pruneFilters filters. if !matchLabels(pruneFilters, c.Config.Labels) { continue } - cSize, _ := daemon.getSize(c.ID) + cSize, _ := daemon.imageService.GetContainerLayerSize(c.ID) // TODO: sets RmLink to true? err := daemon.ContainerRm(c.ID, &types.ContainerRmConfig{}) if err != nil { diff --git a/components/engine/daemon/reload.go b/components/engine/daemon/reload.go index 2ded7d5bd9..5d5cba985d 100644 --- a/components/engine/daemon/reload.go +++ b/components/engine/daemon/reload.go @@ -90,8 +90,8 @@ func (daemon *Daemon) reloadMaxConcurrentDownloadsAndUploads(conf *config.Config daemon.configStore.MaxConcurrentDownloads = &maxConcurrentDownloads } logrus.Debugf("Reset Max Concurrent Downloads: %d", *daemon.configStore.MaxConcurrentDownloads) - if daemon.downloadManager != nil { - daemon.downloadManager.SetConcurrency(*daemon.configStore.MaxConcurrentDownloads) + if daemon.imageService.downloadManager != nil { + daemon.imageService.downloadManager.SetConcurrency(*daemon.configStore.MaxConcurrentDownloads) } // prepare reload event attributes with updatable configurations @@ -106,8 +106,8 @@ func (daemon *Daemon) reloadMaxConcurrentDownloadsAndUploads(conf *config.Config daemon.configStore.MaxConcurrentUploads = &maxConcurrentUploads } logrus.Debugf("Reset Max Concurrent Uploads: %d", *daemon.configStore.MaxConcurrentUploads) - if daemon.uploadManager != nil { - daemon.uploadManager.SetConcurrency(*daemon.configStore.MaxConcurrentUploads) + if daemon.imageService.uploadManager != nil { + daemon.imageService.uploadManager.SetConcurrency(*daemon.configStore.MaxConcurrentUploads) } // prepare reload event attributes with updatable configurations diff --git a/components/engine/daemon/reload_test.go b/components/engine/daemon/reload_test.go index 83e86fc5a0..3e43b6d818 100644 --- a/components/engine/daemon/reload_test.go +++ b/components/engine/daemon/reload_test.go @@ -15,11 +15,13 @@ import ( ) func TestDaemonReloadLabels(t *testing.T) { - daemon := &Daemon{} - daemon.configStore = &config.Config{ - CommonConfig: config.CommonConfig{ - Labels: []string{"foo:bar"}, + daemon := &Daemon{ + configStore: &config.Config{ + CommonConfig: config.CommonConfig{ + Labels: []string{"foo:bar"}, + }, }, + imageService: &imageService{}, } valuesSets := make(map[string]interface{}) @@ -43,7 +45,8 @@ func TestDaemonReloadLabels(t *testing.T) { func TestDaemonReloadAllowNondistributableArtifacts(t *testing.T) { daemon := &Daemon{ - configStore: &config.Config{}, + configStore: &config.Config{}, + imageService: &imageService{}, } var err error @@ -97,7 +100,9 @@ func TestDaemonReloadAllowNondistributableArtifacts(t *testing.T) { } func TestDaemonReloadMirrors(t *testing.T) { - daemon := &Daemon{} + daemon := &Daemon{ + imageService: &imageService{}, + } var err error daemon.RegistryService, err = registry.NewService(registry.ServiceOptions{ InsecureRegistries: []string{}, @@ -194,7 +199,9 @@ func TestDaemonReloadMirrors(t *testing.T) { } func TestDaemonReloadInsecureRegistries(t *testing.T) { - daemon := &Daemon{} + daemon := &Daemon{ + imageService: &imageService{}, + } var err error // initialize daemon with existing insecure registries: "127.0.0.0/8", "10.10.1.11:5000", "10.10.1.22:5000" daemon.RegistryService, err = registry.NewService(registry.ServiceOptions{ @@ -284,7 +291,9 @@ func TestDaemonReloadInsecureRegistries(t *testing.T) { } func TestDaemonReloadNotAffectOthers(t *testing.T) { - daemon := &Daemon{} + daemon := &Daemon{ + imageService: &imageService{}, + } daemon.configStore = &config.Config{ CommonConfig: config.CommonConfig{ Labels: []string{"foo:bar"}, @@ -316,7 +325,9 @@ func TestDaemonReloadNotAffectOthers(t *testing.T) { } func TestDaemonDiscoveryReload(t *testing.T) { - daemon := &Daemon{} + daemon := &Daemon{ + imageService: &imageService{}, + } daemon.configStore = &config.Config{ CommonConfig: config.CommonConfig{ ClusterStore: "memory://127.0.0.1", @@ -393,7 +404,9 @@ func TestDaemonDiscoveryReload(t *testing.T) { } func TestDaemonDiscoveryReloadFromEmptyDiscovery(t *testing.T) { - daemon := &Daemon{} + daemon := &Daemon{ + imageService: &imageService{}, + } daemon.configStore = &config.Config{} valuesSet := make(map[string]interface{}) @@ -438,7 +451,9 @@ func TestDaemonDiscoveryReloadFromEmptyDiscovery(t *testing.T) { } func TestDaemonDiscoveryReloadOnlyClusterAdvertise(t *testing.T) { - daemon := &Daemon{} + daemon := &Daemon{ + imageService: &imageService{}, + } daemon.configStore = &config.Config{ CommonConfig: config.CommonConfig{ ClusterStore: "memory://127.0.0.1", @@ -482,7 +497,9 @@ func TestDaemonDiscoveryReloadOnlyClusterAdvertise(t *testing.T) { } func TestDaemonReloadNetworkDiagnosticPort(t *testing.T) { - daemon := &Daemon{} + daemon := &Daemon{ + imageService: &imageService{}, + } daemon.configStore = &config.Config{} valuesSet := make(map[string]interface{}) diff --git a/components/engine/daemon/start.go b/components/engine/daemon/start.go index 2750df20b8..ad61747a2f 100644 --- a/components/engine/daemon/start.go +++ b/components/engine/daemon/start.go @@ -223,7 +223,7 @@ func (daemon *Daemon) Cleanup(container *container.Container) { if err := daemon.conditionalUnmountOnCleanup(container); err != nil { // FIXME: remove once reference counting for graphdrivers has been refactored // Ensure that all the mounts are gone - if mountid, err := daemon.layerStores[container.OS].GetMountID(container.ID); err == nil { + if mountid, err := daemon.imageService.GetContainerMountID(container.ID, container.OS); err == nil { daemon.cleanupMountsByID(mountid) } } From fa6faa5087c4dcbf31f3764662fd8a7b5e797f44 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 7 Feb 2018 15:52:47 -0500 Subject: [PATCH 08/20] Move ImageService to new package Signed-off-by: Daniel Nephin Upstream-commit: 2b1a2b10afce6ba251d096cfdbd642fc436120ef Component: engine --- .../engine/daemon/cluster/executor/backend.go | 1 + .../daemon/container_operations_windows.go | 6 - components/engine/daemon/create.go | 2 +- components/engine/daemon/daemon.go | 83 +++--- components/engine/daemon/delete.go | 2 +- components/engine/daemon/export.go | 6 +- components/engine/daemon/image.go | 266 ------------------ components/engine/daemon/image_events.go | 29 -- .../engine/daemon/{ => images}/cache.go | 4 +- components/engine/daemon/images/image.go | 75 +++++ .../daemon/{ => images}/image_builder.go | 9 +- .../daemon/{ => images}/image_commit.go | 6 +- .../daemon/{ => images}/image_delete.go | 14 +- .../engine/daemon/images/image_events.go | 39 +++ .../daemon/{ => images}/image_exporter.go | 6 +- .../daemon/{ => images}/image_history.go | 4 +- .../daemon/{ => images}/image_import.go | 4 +- .../daemon/{ => images}/image_inspect.go | 6 +- .../engine/daemon/{ => images}/image_prune.go | 48 +++- .../engine/daemon/{ => images}/image_pull.go | 8 +- .../engine/daemon/{ => images}/image_push.go | 4 +- .../daemon/{ => images}/image_search.go | 4 +- .../image_search_test.go} | 10 +- .../engine/daemon/{ => images}/image_tag.go | 6 +- .../image_unix.go} | 6 +- .../daemon/{ => images}/image_windows.go | 14 +- .../engine/daemon/{ => images}/images.go | 8 +- components/engine/daemon/images/locals.go | 32 +++ components/engine/daemon/images/service.go | 229 +++++++++++++++ components/engine/daemon/info.go | 2 +- components/engine/daemon/list.go | 3 +- components/engine/daemon/metrics.go | 2 - components/engine/daemon/oci_windows.go | 2 +- components/engine/daemon/reload.go | 12 +- components/engine/daemon/reload_test.go | 19 +- components/engine/daemon/start.go | 2 +- 36 files changed, 545 insertions(+), 428 deletions(-) delete mode 100644 components/engine/daemon/image.go delete mode 100644 components/engine/daemon/image_events.go rename components/engine/daemon/{ => images}/cache.go (79%) create mode 100644 components/engine/daemon/images/image.go rename components/engine/daemon/{ => images}/image_builder.go (95%) rename components/engine/daemon/{ => images}/image_commit.go (94%) rename components/engine/daemon/{ => images}/image_delete.go (96%) create mode 100644 components/engine/daemon/images/image_events.go rename components/engine/daemon/{ => images}/image_exporter.go (82%) rename components/engine/daemon/{ => images}/image_history.go (93%) rename components/engine/daemon/{ => images}/image_import.go (96%) rename components/engine/daemon/{ => images}/image_inspect.go (92%) rename components/engine/daemon/{ => images}/image_prune.go (73%) rename components/engine/daemon/{ => images}/image_pull.go (93%) rename components/engine/daemon/{ => images}/image_push.go (93%) rename components/engine/daemon/{ => images}/image_search.go (95%) rename components/engine/daemon/{search_test.go => images/image_search_test.go} (97%) rename components/engine/daemon/{ => images}/image_tag.go (84%) rename components/engine/daemon/{image_getsize_unix.go => images/image_unix.go} (83%) rename components/engine/daemon/{ => images}/image_windows.go (67%) rename components/engine/daemon/{ => images}/images.go (97%) create mode 100644 components/engine/daemon/images/locals.go create mode 100644 components/engine/daemon/images/service.go diff --git a/components/engine/daemon/cluster/executor/backend.go b/components/engine/daemon/cluster/executor/backend.go index 81ba79e91e..3a5ff479bd 100644 --- a/components/engine/daemon/cluster/executor/backend.go +++ b/components/engine/daemon/cluster/executor/backend.go @@ -62,6 +62,7 @@ type Backend interface { GetAttachmentStore() *networkSettings.AttachmentStore } +// ImageBackend is used by an executor to perform image operations type ImageBackend interface { PullImage(ctx context.Context, image, tag, platform string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error GetRepository(context.Context, reference.Named, *types.AuthConfig) (distribution.Repository, bool, error) diff --git a/components/engine/daemon/container_operations_windows.go b/components/engine/daemon/container_operations_windows.go index 0559b8ac3e..a4541b0634 100644 --- a/components/engine/daemon/container_operations_windows.go +++ b/components/engine/daemon/container_operations_windows.go @@ -64,12 +64,6 @@ func (daemon *Daemon) setupConfigDir(c *container.Container) (setupErr error) { return nil } -// getSize returns real size & virtual size -func (daemon *Daemon) getSize(containerID string) (int64, int64) { - // TODO Windows - return 0, 0 -} - func (daemon *Daemon) setupIpcDirs(container *container.Container) error { return nil } diff --git a/components/engine/daemon/create.go b/components/engine/daemon/create.go index 386d346f73..e57452b188 100644 --- a/components/engine/daemon/create.go +++ b/components/engine/daemon/create.go @@ -157,7 +157,7 @@ func (daemon *Daemon) create(params types.ContainerCreateConfig, managed bool) ( } // Set RWLayer for container after mount labels have been set - rwLayer, err := daemon.imageService.GetRWLayer(container, setupInitLayer(daemon.idMappings)) + rwLayer, err := daemon.imageService.CreateLayer(container, setupInitLayer(daemon.idMappings)) if err != nil { return nil, errdefs.System(err) } diff --git a/components/engine/daemon/daemon.go b/components/engine/daemon/daemon.go index 4a41e504ca..8c36f042ea 100644 --- a/components/engine/daemon/daemon.go +++ b/components/engine/daemon/daemon.go @@ -21,21 +21,21 @@ import ( "github.com/docker/docker/api/types" containertypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/swarm" + "github.com/docker/docker/builder" "github.com/docker/docker/container" "github.com/docker/docker/daemon/config" "github.com/docker/docker/daemon/discovery" "github.com/docker/docker/daemon/events" "github.com/docker/docker/daemon/exec" + "github.com/docker/docker/daemon/images" "github.com/docker/docker/daemon/logger" "github.com/docker/docker/daemon/network" "github.com/docker/docker/errdefs" "github.com/sirupsen/logrus" // register graph drivers - "github.com/docker/docker/builder" _ "github.com/docker/docker/daemon/graphdriver/register" "github.com/docker/docker/daemon/stats" dmetadata "github.com/docker/docker/distribution/metadata" - "github.com/docker/docker/distribution/xfer" "github.com/docker/docker/dockerversion" "github.com/docker/docker/image" "github.com/docker/docker/layer" @@ -75,23 +75,21 @@ type Daemon struct { containers container.Store containersReplica container.ViewDB execCommands *exec.Store - - imageService *imageService - - idIndex *truncindex.TruncIndex - configStore *config.Config - statsCollector *stats.Collector - defaultLogConfig containertypes.LogConfig - RegistryService registry.Service - EventsService *events.Events - netController libnetwork.NetworkController - volumes *store.VolumeStore - discoveryWatcher discovery.Reloader - root string - seccompEnabled bool - apparmorEnabled bool - shutdown bool - idMappings *idtools.IDMappings + imageService *images.ImageService + idIndex *truncindex.TruncIndex + configStore *config.Config + statsCollector *stats.Collector + defaultLogConfig containertypes.LogConfig + RegistryService registry.Service + EventsService *events.Events + netController libnetwork.NetworkController + volumes *store.VolumeStore + discoveryWatcher discovery.Reloader + root string + seccompEnabled bool + apparmorEnabled bool + shutdown bool + idMappings *idtools.IDMappings // TODO: move graphDrivers field to an InfoService graphDrivers map[string]string // By operating system @@ -158,7 +156,7 @@ func (daemon *Daemon) restore() error { // Ignore the container if it does not support the current driver being used by the graph currentDriverForContainerOS := daemon.graphDrivers[container.OS] if (container.Driver == "" && currentDriverForContainerOS == "aufs") || container.Driver == currentDriverForContainerOS { - rwlayer, err := daemon.imageService.GetRWLayerByID(container.ID, container.OS) + rwlayer, err := daemon.imageService.GetLayerByID(container.ID, container.OS) if err != nil { logrus.Errorf("Failed to load container mount %v: %v", id, err) continue @@ -808,8 +806,6 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe return nil, err } - eventsService := events.New() - // We have a single tag/reference store for the daemon globally. However, it's // stored under the graphdriver. On host platforms which only support a single // container OS, but multiple selectable graphdrivers, this means depending on which @@ -863,7 +859,7 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe d.idIndex = truncindex.NewTruncIndex([]string{}) d.statsCollector = d.newStatsCollector(1 * time.Second) - d.EventsService = eventsService + d.EventsService = events.New() d.volumes = volStore d.root = config.Root d.idMappings = idMappings @@ -872,19 +868,21 @@ func NewDaemon(config *config.Config, registryService registry.Service, containe d.linkIndex = newLinkIndex() - logrus.Debugf("Max Concurrent Downloads: %d", *config.MaxConcurrentDownloads) - logrus.Debugf("Max Concurrent Uploads: %d", *config.MaxConcurrentUploads) - d.imageService = &imageService{ - trustKey: trustKey, - uploadManager: xfer.NewLayerUploadManager(*config.MaxConcurrentUploads), - downloadManager: xfer.NewLayerDownloadManager(layerStores, *config.MaxConcurrentDownloads), - registryService: registryService, - referenceStore: rs, - distributionMetadataStore: distributionMetadataStore, - imageStore: imageStore, - eventsService: eventsService, - containers: d.containers, - } + // TODO: imageStore, distributionMetadataStore, and ReferenceStore are only + // used above to run migration. They could be initialized in ImageService + // if migration is called from daemon/images. layerStore might move as well. + d.imageService = images.NewImageService(images.ImageServiceConfig{ + ContainerStore: d.containers, + DistributionMetadataStore: distributionMetadataStore, + EventsService: d.EventsService, + ImageStore: imageStore, + LayerStores: layerStores, + MaxConcurrentDownloads: *config.MaxConcurrentDownloads, + MaxConcurrentUploads: *config.MaxConcurrentUploads, + ReferenceStore: rs, + RegistryService: registryService, + TrustKey: trustKey, + }) go d.execCommandGC() @@ -1007,7 +1005,7 @@ func (daemon *Daemon) Shutdown() error { logrus.Errorf("Stop container error: %v", err) return } - if mountid, err := daemon.imageService.GetContainerMountID(c.ID, c.OS); err == nil { + if mountid, err := daemon.imageService.GetLayerMountID(c.ID, c.OS); err == nil { daemon.cleanupMountsByID(mountid) } logrus.Debugf("container stopped %s", c.ID) @@ -1020,7 +1018,9 @@ func (daemon *Daemon) Shutdown() error { } } - daemon.imageService.Cleanup() + if daemon.imageService != nil { + daemon.imageService.Cleanup() + } // If we are part of a cluster, clean up cluster's stuff if daemon.clusterProvider != nil { @@ -1320,14 +1320,15 @@ func (daemon *Daemon) IDMappings() *idtools.IDMappings { return daemon.idMappings } -func (daemon *Daemon) ImageService() *imageService { +// ImageService returns the Daemon's ImageService +func (daemon *Daemon) ImageService() *images.ImageService { return daemon.imageService } -// TODO: tmp hack to merge interfaces +// BuilderBackend returns the backend used by builder func (daemon *Daemon) BuilderBackend() builder.Backend { return struct { *Daemon - *imageService + *images.ImageService }{daemon, daemon.imageService} } diff --git a/components/engine/daemon/delete.go b/components/engine/daemon/delete.go index 749afeb812..1eff18968f 100644 --- a/components/engine/daemon/delete.go +++ b/components/engine/daemon/delete.go @@ -120,7 +120,7 @@ func (daemon *Daemon) cleanupContainer(container *container.Container, forceRemo // When container creation fails and `RWLayer` has not been created yet, we // do not call `ReleaseRWLayer` if container.RWLayer != nil { - err := daemon.imageService.ReleaseContainerLayer(container.RWLayer, container.OS) + err := daemon.imageService.ReleaseLayer(container.RWLayer, container.OS) if err != nil { err = errors.Wrapf(err, "container %s", container.ID) container.SetRemovalError(err) diff --git a/components/engine/daemon/export.go b/components/engine/daemon/export.go index 27181c9176..52c23a3c28 100644 --- a/components/engine/daemon/export.go +++ b/components/engine/daemon/export.go @@ -51,13 +51,13 @@ func (daemon *Daemon) containerExport(container *container.Container) (arch io.R if !system.IsOSSupported(container.OS) { return nil, fmt.Errorf("cannot export %s: %s ", container.ID, system.ErrNotSupportedOperatingSystem) } - rwlayer, err := daemon.imageService.GetRWLayerByID(container.ID, container.OS) + rwlayer, err := daemon.imageService.GetLayerByID(container.ID, container.OS) if err != nil { return nil, err } defer func() { if err != nil { - daemon.imageService.ReleaseContainerLayer(rwlayer, container.OS) + daemon.imageService.ReleaseLayer(rwlayer, container.OS) } }() @@ -78,7 +78,7 @@ func (daemon *Daemon) containerExport(container *container.Container) (arch io.R arch = ioutils.NewReadCloserWrapper(archive, func() error { err := archive.Close() rwlayer.Unmount() - daemon.imageService.ReleaseContainerLayer(rwlayer, container.OS) + daemon.imageService.ReleaseLayer(rwlayer, container.OS) return err }) daemon.LogContainerEvent(container, "export") diff --git a/components/engine/daemon/image.go b/components/engine/daemon/image.go deleted file mode 100644 index 366dff44fc..0000000000 --- a/components/engine/daemon/image.go +++ /dev/null @@ -1,266 +0,0 @@ -package daemon // import "github.com/docker/docker/daemon" - -import ( - "context" - "fmt" - "os" - - "github.com/docker/distribution/reference" - "github.com/docker/docker/api/types/events" - "github.com/docker/docker/container" - daemonevents "github.com/docker/docker/daemon/events" - "github.com/docker/docker/distribution/metadata" - "github.com/docker/docker/distribution/xfer" - "github.com/docker/docker/errdefs" - "github.com/docker/docker/image" - "github.com/docker/docker/layer" - dockerreference "github.com/docker/docker/reference" - "github.com/docker/docker/registry" - "github.com/docker/libtrust" - "github.com/opencontainers/go-digest" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" -) - -// errImageDoesNotExist is error returned when no image can be found for a reference. -type errImageDoesNotExist struct { - ref reference.Reference -} - -func (e errImageDoesNotExist) Error() string { - ref := e.ref - if named, ok := ref.(reference.Named); ok { - ref = reference.TagNameOnly(named) - } - return fmt.Sprintf("No such image: %s", reference.FamiliarString(ref)) -} - -func (e errImageDoesNotExist) NotFound() {} - -// GetImageIDAndOS returns an image ID and operating system corresponding to the image referred to by -// refOrID. -// called from list.go foldFilter() -func (i imageService) GetImageIDAndOS(refOrID string) (image.ID, string, error) { - ref, err := reference.ParseAnyReference(refOrID) - if err != nil { - return "", "", errdefs.InvalidParameter(err) - } - namedRef, ok := ref.(reference.Named) - if !ok { - digested, ok := ref.(reference.Digested) - if !ok { - return "", "", errImageDoesNotExist{ref} - } - id := image.IDFromDigest(digested.Digest()) - if img, err := i.imageStore.Get(id); err == nil { - return id, img.OperatingSystem(), nil - } - return "", "", errImageDoesNotExist{ref} - } - - if digest, err := i.referenceStore.Get(namedRef); err == nil { - // Search the image stores to get the operating system, defaulting to host OS. - id := image.IDFromDigest(digest) - if img, err := i.imageStore.Get(id); err == nil { - return id, img.OperatingSystem(), nil - } - } - - // Search based on ID - if id, err := i.imageStore.Search(refOrID); err == nil { - img, err := i.imageStore.Get(id) - if err != nil { - return "", "", errImageDoesNotExist{ref} - } - return id, img.OperatingSystem(), nil - } - - return "", "", errImageDoesNotExist{ref} -} - -// GetImage returns an image corresponding to the image referred to by refOrID. -func (i *imageService) GetImage(refOrID string) (*image.Image, error) { - imgID, _, err := i.GetImageIDAndOS(refOrID) - if err != nil { - return nil, err - } - return i.imageStore.Get(imgID) -} - -type containerStore interface { - // used by image delete - First(container.StoreFilter) *container.Container - // used by image prune, and image list - List() []*container.Container - // TODO: remove, only used for CommitBuildStep - Get(string) *container.Container -} - -type imageService struct { - eventsService *daemonevents.Events - containers containerStore - downloadManager *xfer.LayerDownloadManager - uploadManager *xfer.LayerUploadManager - - // TODO: should accept a trust service instead of a key - trustKey libtrust.PrivateKey - - registryService registry.Service - referenceStore dockerreference.Store - distributionMetadataStore metadata.Store - imageStore image.Store - layerStores map[string]layer.Store // By operating system - - pruneRunning int32 -} - -// called from info.go -func (i *imageService) CountImages() int { - return len(i.imageStore.Map()) -} - -// called from list.go to filter containers -func (i *imageService) Children(id image.ID) []image.ID { - return i.imageStore.Children(id) -} - -// TODO: accept an opt struct instead of container? -// called from create.go -func (i *imageService) GetRWLayer(container *container.Container, initFunc layer.MountInit) (layer.RWLayer, error) { - var layerID layer.ChainID - if container.ImageID != "" { - img, err := i.imageStore.Get(container.ImageID) - if err != nil { - return nil, err - } - layerID = img.RootFS.ChainID() - } - - rwLayerOpts := &layer.CreateRWLayerOpts{ - MountLabel: container.MountLabel, - InitFunc: initFunc, - StorageOpt: container.HostConfig.StorageOpt, - } - - // Indexing by OS is safe here as validation of OS has already been performed in create() (the only - // caller), and guaranteed non-nil - return i.layerStores[container.OS].CreateRWLayer(container.ID, layerID, rwLayerOpts) -} - -// called from daemon.go Daemon.restore(), and Daemon.containerExport() -func (i *imageService) GetRWLayerByID(cid string, os string) (layer.RWLayer, error) { - return i.layerStores[os].GetRWLayer(cid) -} - -// called from info.go -func (i *imageService) GraphDriverStatuses() map[string][][2]string { - result := make(map[string][][2]string) - for os, store := range i.layerStores { - result[os] = store.DriverStatus() - } - return result -} - -// called from daemon.go Daemon.Shutdown(), and Daemon.Cleanup() (cleanup is actually continerCleanup) -func (i *imageService) GetContainerMountID(cid string, os string) (string, error) { - return i.layerStores[os].GetMountID(cid) -} - -// called from daemon.go Daemon.Shutdown() -func (i *imageService) Cleanup() { - for os, ls := range i.layerStores { - if ls != nil { - if err := ls.Cleanup(); err != nil { - logrus.Errorf("Error during layer Store.Cleanup(): %v %s", err, os) - } - } - } -} - -// moved from Daemon.GraphDriverName, multiple calls -func (i *imageService) GraphDriverForOS(os string) string { - return i.layerStores[os].DriverName() -} - -// called from delete.go Daemon.cleanupContainer(), and Daemon.containerExport() -func (i *imageService) ReleaseContainerLayer(rwlayer layer.RWLayer, containerOS string) error { - metadata, err := i.layerStores[containerOS].ReleaseRWLayer(rwlayer) - layer.LogReleaseMetadata(metadata) - if err != nil && err != layer.ErrMountDoesNotExist && !os.IsNotExist(errors.Cause(err)) { - return errors.Wrapf(err, "driver %q failed to remove root filesystem", - i.layerStores[containerOS].DriverName()) - } - return nil -} - -// called from disk_usage.go -func (i *imageService) LayerDiskUsage(ctx context.Context) (int64, error) { - var allLayersSize int64 - layerRefs := i.getLayerRefs() - for _, ls := range i.layerStores { - allLayers := ls.Map() - for _, l := range allLayers { - select { - case <-ctx.Done(): - return allLayersSize, ctx.Err() - default: - size, err := l.DiffSize() - if err == nil { - if _, ok := layerRefs[l.ChainID()]; ok { - allLayersSize += size - } else { - logrus.Warnf("found leaked image layer %v", l.ChainID()) - } - } else { - logrus.Warnf("failed to get diff size for layer %v", l.ChainID()) - } - } - } - } - return allLayersSize, nil -} - -func (i *imageService) getLayerRefs() map[layer.ChainID]int { - tmpImages := i.imageStore.Map() - layerRefs := map[layer.ChainID]int{} - for id, img := range tmpImages { - dgst := digest.Digest(id) - if len(i.referenceStore.References(dgst)) == 0 && len(i.imageStore.Children(id)) != 0 { - continue - } - - rootFS := *img.RootFS - rootFS.DiffIDs = nil - for _, id := range img.RootFS.DiffIDs { - rootFS.Append(id) - chid := rootFS.ChainID() - layerRefs[chid]++ - } - } - - return layerRefs -} - -// LogImageEvent generates an event related to an image with only the default attributes. -func (i *imageService) LogImageEvent(imageID, refName, action string) { - i.LogImageEventWithAttributes(imageID, refName, action, map[string]string{}) -} - -// LogImageEventWithAttributes generates an event related to an image with specific given attributes. -func (i *imageService) LogImageEventWithAttributes(imageID, refName, action string, attributes map[string]string) { - img, err := i.GetImage(imageID) - if err == nil && img.Config != nil { - // image has not been removed yet. - // it could be missing if the event is `delete`. - copyAttributes(attributes, img.Config.Labels) - } - if refName != "" { - attributes["name"] = refName - } - actor := events.Actor{ - ID: imageID, - Attributes: attributes, - } - - i.eventsService.Log(action, events.ImageEventType, actor) -} diff --git a/components/engine/daemon/image_events.go b/components/engine/daemon/image_events.go deleted file mode 100644 index 0d364f7b58..0000000000 --- a/components/engine/daemon/image_events.go +++ /dev/null @@ -1,29 +0,0 @@ -package daemon // import "github.com/docker/docker/daemon" - -import ( - "github.com/docker/docker/api/types/events" -) - -// LogImageEvent generates an event related to an image with only the default attributes. -func (daemon *Daemon) LogImageEvent(imageID, refName, action string) { - daemon.LogImageEventWithAttributes(imageID, refName, action, map[string]string{}) -} - -// LogImageEventWithAttributes generates an event related to an image with specific given attributes. -func (daemon *Daemon) LogImageEventWithAttributes(imageID, refName, action string, attributes map[string]string) { - img, err := daemon.GetImage(imageID) - if err == nil && img.Config != nil { - // image has not been removed yet. - // it could be missing if the event is `delete`. - copyAttributes(attributes, img.Config.Labels) - } - if refName != "" { - attributes["name"] = refName - } - actor := events.Actor{ - ID: imageID, - Attributes: attributes, - } - - daemon.EventsService.Log(action, events.ImageEventType, actor) -} diff --git a/components/engine/daemon/cache.go b/components/engine/daemon/images/cache.go similarity index 79% rename from components/engine/daemon/cache.go rename to components/engine/daemon/images/cache.go index 13ddfbe41f..3b433106e8 100644 --- a/components/engine/daemon/cache.go +++ b/components/engine/daemon/images/cache.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "github.com/docker/docker/builder" @@ -7,7 +7,7 @@ import ( ) // MakeImageCache creates a stateful image cache. -func (i *imageService) MakeImageCache(sourceRefs []string) builder.ImageCache { +func (i *ImageService) MakeImageCache(sourceRefs []string) builder.ImageCache { if len(sourceRefs) == 0 { return cache.NewLocal(i.imageStore) } diff --git a/components/engine/daemon/images/image.go b/components/engine/daemon/images/image.go new file mode 100644 index 0000000000..7dda553138 --- /dev/null +++ b/components/engine/daemon/images/image.go @@ -0,0 +1,75 @@ +package images // import "github.com/docker/docker/daemon/images" + +import ( + "fmt" + + "github.com/docker/distribution/reference" + "github.com/docker/docker/errdefs" + "github.com/docker/docker/image" +) + +// ErrImageDoesNotExist is error returned when no image can be found for a reference. +type ErrImageDoesNotExist struct { + ref reference.Reference +} + +func (e ErrImageDoesNotExist) Error() string { + ref := e.ref + if named, ok := ref.(reference.Named); ok { + ref = reference.TagNameOnly(named) + } + return fmt.Sprintf("No such image: %s", reference.FamiliarString(ref)) +} + +// NotFound implements the NotFound interface +func (e ErrImageDoesNotExist) NotFound() {} + +// GetImageIDAndOS returns an image ID and operating system corresponding to the image referred to by +// refOrID. +// called from list.go foldFilter() +func (i ImageService) GetImageIDAndOS(refOrID string) (image.ID, string, error) { + ref, err := reference.ParseAnyReference(refOrID) + if err != nil { + return "", "", errdefs.InvalidParameter(err) + } + namedRef, ok := ref.(reference.Named) + if !ok { + digested, ok := ref.(reference.Digested) + if !ok { + return "", "", ErrImageDoesNotExist{ref} + } + id := image.IDFromDigest(digested.Digest()) + if img, err := i.imageStore.Get(id); err == nil { + return id, img.OperatingSystem(), nil + } + return "", "", ErrImageDoesNotExist{ref} + } + + if digest, err := i.referenceStore.Get(namedRef); err == nil { + // Search the image stores to get the operating system, defaulting to host OS. + id := image.IDFromDigest(digest) + if img, err := i.imageStore.Get(id); err == nil { + return id, img.OperatingSystem(), nil + } + } + + // Search based on ID + if id, err := i.imageStore.Search(refOrID); err == nil { + img, err := i.imageStore.Get(id) + if err != nil { + return "", "", ErrImageDoesNotExist{ref} + } + return id, img.OperatingSystem(), nil + } + + return "", "", ErrImageDoesNotExist{ref} +} + +// GetImage returns an image corresponding to the image referred to by refOrID. +func (i *ImageService) GetImage(refOrID string) (*image.Image, error) { + imgID, _, err := i.GetImageIDAndOS(refOrID) + if err != nil { + return nil, err + } + return i.imageStore.Get(imgID) +} diff --git a/components/engine/daemon/image_builder.go b/components/engine/daemon/images/image_builder.go similarity index 95% rename from components/engine/daemon/image_builder.go rename to components/engine/daemon/images/image_builder.go index 50a5212884..b88e1bc0fa 100644 --- a/components/engine/daemon/image_builder.go +++ b/components/engine/daemon/images/image_builder.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "io" @@ -8,7 +8,6 @@ import ( "github.com/docker/docker/api/types/backend" "github.com/docker/docker/builder" "github.com/docker/docker/image" - "github.com/docker/docker/image/cache" "github.com/docker/docker/layer" "github.com/docker/docker/pkg/containerfs" "github.com/docker/docker/pkg/stringid" @@ -138,7 +137,7 @@ func newROLayerForImage(img *image.Image, layerStore layer.Store) (builder.ROLay } // TODO: could this use the regular daemon PullImage ? -func (i *imageService) pullForBuilder(ctx context.Context, name string, authConfigs map[string]types.AuthConfig, output io.Writer, os string) (*image.Image, error) { +func (i *ImageService) pullForBuilder(ctx context.Context, name string, authConfigs map[string]types.AuthConfig, output io.Writer, os string) (*image.Image, error) { ref, err := reference.ParseNormalizedNamed(name) if err != nil { return nil, err @@ -166,7 +165,7 @@ func (i *imageService) pullForBuilder(ctx context.Context, name string, authConf // GetImageAndReleasableLayer returns an image and releaseable layer for a reference or ID. // Every call to GetImageAndReleasableLayer MUST call releasableLayer.Release() to prevent // leaking of layers. -func (i *imageService) GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (builder.Image, builder.ROLayer, error) { +func (i *ImageService) GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (builder.Image, builder.ROLayer, error) { if refOrID == "" { if !system.IsOSSupported(opts.OS) { return nil, nil, system.ErrNotSupportedOperatingSystem @@ -204,7 +203,7 @@ func (i *imageService) GetImageAndReleasableLayer(ctx context.Context, refOrID s // CreateImage creates a new image by adding a config and ID to the image store. // This is similar to LoadImage() except that it receives JSON encoded bytes of // an image instead of a tar archive. -func (i *imageService) CreateImage(config []byte, parent string) (builder.Image, error) { +func (i *ImageService) CreateImage(config []byte, parent string) (builder.Image, error) { id, err := i.imageStore.Create(config) if err != nil { return nil, errors.Wrapf(err, "failed to create image") diff --git a/components/engine/daemon/image_commit.go b/components/engine/daemon/images/image_commit.go similarity index 94% rename from components/engine/daemon/image_commit.go rename to components/engine/daemon/images/image_commit.go index 6188b52577..4caba9f27b 100644 --- a/components/engine/daemon/image_commit.go +++ b/components/engine/daemon/images/image_commit.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "encoding/json" @@ -13,7 +13,7 @@ import ( ) // CommitImage creates a new image from a commit config -func (i *imageService) CommitImage(c backend.CommitConfig) (image.ID, error) { +func (i *ImageService) CommitImage(c backend.CommitConfig) (image.ID, error) { layerStore, ok := i.layerStores[c.ContainerOS] if !ok { return "", system.ErrNotSupportedOperatingSystem @@ -114,7 +114,7 @@ func exportContainerRw(layerStore layer.Store, id, mountLabel string) (arch io.R // * it doesn't log a container commit event // // This is a temporary shim. Should be removed when builder stops using commit. -func (i *imageService) CommitBuildStep(c backend.CommitConfig) (image.ID, error) { +func (i *ImageService) CommitBuildStep(c backend.CommitConfig) (image.ID, error) { container := i.containers.Get(c.ContainerID) if container == nil { // TODO: use typed error diff --git a/components/engine/daemon/image_delete.go b/components/engine/daemon/images/image_delete.go similarity index 96% rename from components/engine/daemon/image_delete.go rename to components/engine/daemon/images/image_delete.go index 2615f05ca7..60f60afaa7 100644 --- a/components/engine/daemon/image_delete.go +++ b/components/engine/daemon/images/image_delete.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "fmt" @@ -60,7 +60,7 @@ const ( // meaning any delete conflicts will cause the image to not be deleted and the // conflict will not be reported. // -func (i *imageService) ImageDelete(imageRef string, force, prune bool) ([]types.ImageDeleteResponseItem, error) { +func (i *ImageService) ImageDelete(imageRef string, force, prune bool) ([]types.ImageDeleteResponseItem, error) { start := time.Now() records := []types.ImageDeleteResponseItem{} @@ -229,7 +229,7 @@ func isImageIDPrefix(imageID, possiblePrefix string) bool { // repositoryRef must not be an image ID but a repository name followed by an // optional tag or digest reference. If tag or digest is omitted, the default // tag is used. Returns the resolved image reference and an error. -func (i *imageService) removeImageRef(ref reference.Named) (reference.Named, error) { +func (i *ImageService) removeImageRef(ref reference.Named) (reference.Named, error) { ref = reference.TagNameOnly(ref) // Ignore the boolean value returned, as far as we're concerned, this @@ -245,7 +245,7 @@ func (i *imageService) removeImageRef(ref reference.Named) (reference.Named, err // on the first encountered error. Removed references are logged to this // daemon's event service. An "Untagged" types.ImageDeleteResponseItem is added to the // given list of records. -func (i *imageService) removeAllReferencesToImageID(imgID image.ID, records *[]types.ImageDeleteResponseItem) error { +func (i *ImageService) removeAllReferencesToImageID(imgID image.ID, records *[]types.ImageDeleteResponseItem) error { imageRefs := i.referenceStore.References(imgID.Digest()) for _, imageRef := range imageRefs { @@ -296,7 +296,7 @@ func (idc *imageDeleteConflict) Conflict() {} // conflict is encountered, it will be returned immediately without deleting // the image. If quiet is true, any encountered conflicts will be ignored and // the function will return nil immediately without deleting the image. -func (i *imageService) imageDeleteHelper(imgID image.ID, records *[]types.ImageDeleteResponseItem, force, prune, quiet bool) error { +func (i *ImageService) imageDeleteHelper(imgID image.ID, records *[]types.ImageDeleteResponseItem, force, prune, quiet bool) error { // First, determine if this image has any conflicts. Ignore soft conflicts // if force is true. c := conflictHard @@ -355,7 +355,7 @@ func (i *imageService) imageDeleteHelper(imgID image.ID, records *[]types.ImageD // using the image. A soft conflict is any tags/digest referencing the given // image or any stopped container using the image. If ignoreSoftConflicts is // true, this function will not check for soft conflict conditions. -func (i *imageService) checkImageDeleteConflict(imgID image.ID, mask conflictType) *imageDeleteConflict { +func (i *ImageService) checkImageDeleteConflict(imgID image.ID, mask conflictType) *imageDeleteConflict { // Check if the image has any descendant images. if mask&conflictDependentChild != 0 && len(i.imageStore.Children(imgID)) > 0 { return &imageDeleteConflict{ @@ -408,6 +408,6 @@ func (i *imageService) checkImageDeleteConflict(imgID image.ID, mask conflictTyp // imageIsDangling returns whether the given image is "dangling" which means // that there are no repository references to the given image and it has no // child images. -func (i *imageService) imageIsDangling(imgID image.ID) bool { +func (i *ImageService) imageIsDangling(imgID image.ID) bool { return !(len(i.referenceStore.References(imgID.Digest())) > 0 || len(i.imageStore.Children(imgID)) > 0) } diff --git a/components/engine/daemon/images/image_events.go b/components/engine/daemon/images/image_events.go new file mode 100644 index 0000000000..d0b3064d70 --- /dev/null +++ b/components/engine/daemon/images/image_events.go @@ -0,0 +1,39 @@ +package images // import "github.com/docker/docker/daemon/images" + +import ( + "github.com/docker/docker/api/types/events" +) + +// LogImageEvent generates an event related to an image with only the default attributes. +func (i *ImageService) LogImageEvent(imageID, refName, action string) { + i.LogImageEventWithAttributes(imageID, refName, action, map[string]string{}) +} + +// LogImageEventWithAttributes generates an event related to an image with specific given attributes. +func (i *ImageService) LogImageEventWithAttributes(imageID, refName, action string, attributes map[string]string) { + img, err := i.GetImage(imageID) + if err == nil && img.Config != nil { + // image has not been removed yet. + // it could be missing if the event is `delete`. + copyAttributes(attributes, img.Config.Labels) + } + if refName != "" { + attributes["name"] = refName + } + actor := events.Actor{ + ID: imageID, + Attributes: attributes, + } + + i.eventsService.Log(action, events.ImageEventType, actor) +} + +// copyAttributes guarantees that labels are not mutated by event triggers. +func copyAttributes(attributes, labels map[string]string) { + if labels == nil { + return + } + for k, v := range labels { + attributes[k] = v + } +} diff --git a/components/engine/daemon/image_exporter.go b/components/engine/daemon/images/image_exporter.go similarity index 82% rename from components/engine/daemon/image_exporter.go rename to components/engine/daemon/images/image_exporter.go index 575565fc25..58105dcb71 100644 --- a/components/engine/daemon/image_exporter.go +++ b/components/engine/daemon/images/image_exporter.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "io" @@ -11,7 +11,7 @@ import ( // stream. All images with the given tag and all versions containing // the same tag are exported. names is the set of tags to export, and // outStream is the writer which the images are written to. -func (i *imageService) ExportImage(names []string, outStream io.Writer) error { +func (i *ImageService) ExportImage(names []string, outStream io.Writer) error { imageExporter := tarexport.NewTarExporter(i.imageStore, i.layerStores, i.referenceStore, i) return imageExporter.Save(names, outStream) } @@ -19,7 +19,7 @@ func (i *imageService) ExportImage(names []string, outStream io.Writer) error { // LoadImage uploads a set of images into the repository. This is the // complement of ImageExport. The input stream is an uncompressed tar // ball containing images and metadata. -func (i *imageService) LoadImage(inTar io.ReadCloser, outStream io.Writer, quiet bool) error { +func (i *ImageService) LoadImage(inTar io.ReadCloser, outStream io.Writer, quiet bool) error { imageExporter := tarexport.NewTarExporter(i.imageStore, i.layerStores, i.referenceStore, i) return imageExporter.Load(inTar, outStream, quiet) } diff --git a/components/engine/daemon/image_history.go b/components/engine/daemon/images/image_history.go similarity index 93% rename from components/engine/daemon/image_history.go rename to components/engine/daemon/images/image_history.go index 621e8cec05..2b92292631 100644 --- a/components/engine/daemon/image_history.go +++ b/components/engine/daemon/images/image_history.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "fmt" @@ -11,7 +11,7 @@ import ( // ImageHistory returns a slice of ImageHistory structures for the specified image // name by walking the image lineage. -func (i *imageService) ImageHistory(name string) ([]*image.HistoryResponseItem, error) { +func (i *ImageService) ImageHistory(name string) ([]*image.HistoryResponseItem, error) { start := time.Now() img, err := i.GetImage(name) if err != nil { diff --git a/components/engine/daemon/image_import.go b/components/engine/daemon/images/image_import.go similarity index 96% rename from components/engine/daemon/image_import.go rename to components/engine/daemon/images/image_import.go index f9bf0b119b..8d54e0704f 100644 --- a/components/engine/daemon/image_import.go +++ b/components/engine/daemon/images/image_import.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "encoding/json" @@ -27,7 +27,7 @@ import ( // inConfig (if src is "-"), or from a URI specified in src. Progress output is // written to outStream. Repository and tag names can optionally be given in // the repo and tag arguments, respectively. -func (i *imageService) ImportImage(src string, repository, os string, tag string, msg string, inConfig io.ReadCloser, outStream io.Writer, changes []string) error { +func (i *ImageService) ImportImage(src string, repository, os string, tag string, msg string, inConfig io.ReadCloser, outStream io.Writer, changes []string) error { var ( rc io.ReadCloser resp *http.Response diff --git a/components/engine/daemon/image_inspect.go b/components/engine/daemon/images/image_inspect.go similarity index 92% rename from components/engine/daemon/image_inspect.go rename to components/engine/daemon/images/image_inspect.go index c00ceb343c..16c4c9b2dc 100644 --- a/components/engine/daemon/image_inspect.go +++ b/components/engine/daemon/images/image_inspect.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "time" @@ -13,7 +13,7 @@ import ( // LookupImage looks up an image by name and returns it as an ImageInspect // structure. -func (i *imageService) LookupImage(name string) (*types.ImageInspect, error) { +func (i *ImageService) LookupImage(name string) (*types.ImageInspect, error) { img, err := i.GetImage(name) if err != nil { return nil, errors.Wrapf(err, "no such image: %s", name) @@ -86,7 +86,7 @@ func (i *imageService) LookupImage(name string) (*types.ImageInspect, error) { }, } - imageInspect.GraphDriver.Name = i.GraphDriverForOS(img.OperatingSystem()) + imageInspect.GraphDriver.Name = i.layerStores[img.OperatingSystem()].DriverName() imageInspect.GraphDriver.Data = layerMetadata return imageInspect, nil diff --git a/components/engine/daemon/image_prune.go b/components/engine/daemon/images/image_prune.go similarity index 73% rename from components/engine/daemon/image_prune.go rename to components/engine/daemon/images/image_prune.go index 92399a38be..9427536759 100644 --- a/components/engine/daemon/image_prune.go +++ b/components/engine/daemon/images/image_prune.go @@ -1,11 +1,14 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( + "fmt" "sync/atomic" + "time" "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" + timetypes "github.com/docker/docker/api/types/time" "github.com/docker/docker/errdefs" "github.com/docker/docker/image" "github.com/docker/docker/layer" @@ -21,8 +24,12 @@ var imagesAcceptedFilters = map[string]bool{ "until": true, } +// errPruneRunning is returned when a prune request is received while +// one is in progress +var errPruneRunning = fmt.Errorf("a prune operation is already running") + // ImagesPrune removes unused images -func (i *imageService) ImagesPrune(ctx context.Context, pruneFilters filters.Args) (*types.ImagesPruneReport, error) { +func (i *ImageService) ImagesPrune(ctx context.Context, pruneFilters filters.Args) (*types.ImagesPruneReport, error) { if !atomic.CompareAndSwapInt32(&i.pruneRunning, 0, 1) { return nil, errPruneRunning } @@ -96,7 +103,7 @@ deleteImagesLoop: } deletedImages := []types.ImageDeleteResponseItem{} - refs := i.referenceStore.References(dgst) + refs := i.referenceStore.References(id.Digest()) if len(refs) > 0 { shouldDelete := !danglingOnly if !shouldDelete { @@ -166,3 +173,38 @@ func imageDeleteFailed(ref string, err error) bool { return true } } + +func matchLabels(pruneFilters filters.Args, labels map[string]string) bool { + if !pruneFilters.MatchKVList("label", labels) { + return false + } + // By default MatchKVList will return true if field (like 'label!') does not exist + // So we have to add additional Contains("label!") check + if pruneFilters.Contains("label!") { + if pruneFilters.MatchKVList("label!", labels) { + return false + } + } + return true +} + +func getUntilFromPruneFilters(pruneFilters filters.Args) (time.Time, error) { + until := time.Time{} + if !pruneFilters.Contains("until") { + return until, nil + } + untilFilters := pruneFilters.Get("until") + if len(untilFilters) > 1 { + return until, fmt.Errorf("more than one until filter specified") + } + ts, err := timetypes.GetTimestamp(untilFilters[0], time.Now()) + if err != nil { + return until, err + } + seconds, nanoseconds, err := timetypes.ParseTimestamps(ts, 0) + if err != nil { + return until, err + } + until = time.Unix(seconds, nanoseconds) + return until, nil +} diff --git a/components/engine/daemon/image_pull.go b/components/engine/daemon/images/image_pull.go similarity index 93% rename from components/engine/daemon/image_pull.go rename to components/engine/daemon/images/image_pull.go index f6606cb38c..41056af678 100644 --- a/components/engine/daemon/image_pull.go +++ b/components/engine/daemon/images/image_pull.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "io" @@ -19,7 +19,7 @@ import ( // PullImage initiates a pull operation. image is the repository name to pull, and // tag may be either empty, or indicate a specific tag to pull. -func (i *imageService) PullImage(ctx context.Context, image, tag, os string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error { +func (i *ImageService) PullImage(ctx context.Context, image, tag, os string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error { // Special case: "pull -a" may send an image name with a // trailing :. This is ugly, but let's not break API // compatibility. @@ -47,7 +47,7 @@ func (i *imageService) PullImage(ctx context.Context, image, tag, os string, met return i.pullImageWithReference(ctx, ref, os, metaHeaders, authConfig, outStream) } -func (i *imageService) pullImageWithReference(ctx context.Context, ref reference.Named, os string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error { +func (i *ImageService) pullImageWithReference(ctx context.Context, ref reference.Named, os string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error { // Include a buffer so that slow client connections don't affect // transfer performance. progressChan := make(chan progress.Progress, 100) @@ -89,7 +89,7 @@ func (i *imageService) pullImageWithReference(ctx context.Context, ref reference } // GetRepository returns a repository from the registry. -func (i *imageService) GetRepository(ctx context.Context, ref reference.Named, authConfig *types.AuthConfig) (dist.Repository, bool, error) { +func (i *ImageService) GetRepository(ctx context.Context, ref reference.Named, authConfig *types.AuthConfig) (dist.Repository, bool, error) { // get repository info repoInfo, err := i.registryService.ResolveRepository(ref) if err != nil { diff --git a/components/engine/daemon/image_push.go b/components/engine/daemon/images/image_push.go similarity index 93% rename from components/engine/daemon/image_push.go rename to components/engine/daemon/images/image_push.go index 23c2d1e2eb..af606a07b5 100644 --- a/components/engine/daemon/image_push.go +++ b/components/engine/daemon/images/image_push.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "io" @@ -13,7 +13,7 @@ import ( ) // PushImage initiates a push operation on the repository named localName. -func (i *imageService) PushImage(ctx context.Context, image, tag string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error { +func (i *ImageService) PushImage(ctx context.Context, image, tag string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error { ref, err := reference.ParseNormalizedNamed(image) if err != nil { return err diff --git a/components/engine/daemon/image_search.go b/components/engine/daemon/images/image_search.go similarity index 95% rename from components/engine/daemon/image_search.go rename to components/engine/daemon/images/image_search.go index 101bd2e306..a901571439 100644 --- a/components/engine/daemon/image_search.go +++ b/components/engine/daemon/images/image_search.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "strconv" @@ -22,7 +22,7 @@ var acceptedSearchFilterTags = map[string]bool{ // // TODO: this could be implemented in a registry service instead of the image // service. -func (i *imageService) SearchRegistryForImages(ctx context.Context, filtersArgs string, term string, limit int, +func (i *ImageService) SearchRegistryForImages(ctx context.Context, filtersArgs string, term string, limit int, authConfig *types.AuthConfig, headers map[string][]string) (*registrytypes.SearchResults, error) { diff --git a/components/engine/daemon/search_test.go b/components/engine/daemon/images/image_search_test.go similarity index 97% rename from components/engine/daemon/search_test.go rename to components/engine/daemon/images/image_search_test.go index 0237b103b8..74795c5865 100644 --- a/components/engine/daemon/search_test.go +++ b/components/engine/daemon/images/image_search_test.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "errors" @@ -76,8 +76,8 @@ func TestSearchRegistryForImagesErrors(t *testing.T) { }, } for index, e := range errorCases { - daemon := &Daemon{ - RegistryService: &FakeService{ + daemon := &ImageService{ + registryService: &FakeService{ shouldReturnError: e.shouldReturnError, }, } @@ -322,8 +322,8 @@ func TestSearchRegistryForImages(t *testing.T) { }, } for index, s := range successCases { - daemon := &Daemon{ - RegistryService: &FakeService{ + daemon := &ImageService{ + registryService: &FakeService{ term: term, results: s.registryResults, }, diff --git a/components/engine/daemon/image_tag.go b/components/engine/daemon/images/image_tag.go similarity index 84% rename from components/engine/daemon/image_tag.go rename to components/engine/daemon/images/image_tag.go index 6e929437f5..6456ff81f9 100644 --- a/components/engine/daemon/image_tag.go +++ b/components/engine/daemon/images/image_tag.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "github.com/docker/distribution/reference" @@ -7,7 +7,7 @@ import ( // TagImage creates the tag specified by newTag, pointing to the image named // imageName (alternatively, imageName can also be an image ID). -func (i *imageService) TagImage(imageName, repository, tag string) (string, error) { +func (i *ImageService) TagImage(imageName, repository, tag string) (string, error) { imageID, _, err := i.GetImageIDAndOS(imageName) if err != nil { return "", err @@ -28,7 +28,7 @@ func (i *imageService) TagImage(imageName, repository, tag string) (string, erro } // TagImageWithReference adds the given reference to the image ID provided. -func (i *imageService) TagImageWithReference(imageID image.ID, newTag reference.Named) error { +func (i *ImageService) TagImageWithReference(imageID image.ID, newTag reference.Named) error { if err := i.referenceStore.AddTag(newTag, imageID.Digest(), true); err != nil { return err } diff --git a/components/engine/daemon/image_getsize_unix.go b/components/engine/daemon/images/image_unix.go similarity index 83% rename from components/engine/daemon/image_getsize_unix.go rename to components/engine/daemon/images/image_unix.go index faa1edd843..3f577271a2 100644 --- a/components/engine/daemon/image_getsize_unix.go +++ b/components/engine/daemon/images/image_unix.go @@ -1,6 +1,6 @@ // +build linux freebsd -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "runtime" @@ -8,8 +8,8 @@ import ( "github.com/sirupsen/logrus" ) -// getSize returns the real size & virtual size of the container. -func (i *imageService) GetContainerLayerSize(containerID string) (int64, int64) { +// GetContainerLayerSize returns the real size & virtual size of the container. +func (i *ImageService) GetContainerLayerSize(containerID string) (int64, int64) { var ( sizeRw, sizeRootfs int64 err error diff --git a/components/engine/daemon/image_windows.go b/components/engine/daemon/images/image_windows.go similarity index 67% rename from components/engine/daemon/image_windows.go rename to components/engine/daemon/images/image_windows.go index d661400a38..6f4be49736 100644 --- a/components/engine/daemon/image_windows.go +++ b/components/engine/daemon/images/image_windows.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images import ( "github.com/docker/docker/image" @@ -7,8 +7,14 @@ import ( "github.com/pkg/errors" ) +// GetContainerLayerSize returns real size & virtual size +func (i *ImageService) GetContainerLayerSize(containerID string) (int64, int64) { + // TODO Windows + return 0, 0 +} + // GetLayerFolders returns the layer folders from an image RootFS -func (daemon *Daemon) GetLayerFolders(img *image.Image, rwLayer layer.RWLayer) ([]string, error) { +func (i *ImageService) GetLayerFolders(img *image.Image, rwLayer layer.RWLayer) ([]string, error) { folders := []string{} max := len(img.RootFS.DiffIDs) for index := 1; index <= max; index++ { @@ -17,9 +23,9 @@ func (daemon *Daemon) GetLayerFolders(img *image.Image, rwLayer layer.RWLayer) ( if !system.IsOSSupported(img.OperatingSystem()) { return nil, errors.Wrapf(system.ErrNotSupportedOperatingSystem, "cannot get layerpath for ImageID %s", img.RootFS.ChainID()) } - layerPath, err := layer.GetLayerPath(daemon.layerStores[img.OperatingSystem()], img.RootFS.ChainID()) + layerPath, err := layer.GetLayerPath(i.layerStores[img.OperatingSystem()], img.RootFS.ChainID()) if err != nil { - return nil, errors.Wrapf(err, "failed to get layer path from graphdriver %s for ImageID %s", daemon.layerStores[img.OperatingSystem()], img.RootFS.ChainID()) + return nil, errors.Wrapf(err, "failed to get layer path from graphdriver %s for ImageID %s", i.layerStores[img.OperatingSystem()], img.RootFS.ChainID()) } // Reverse order, expecting parent first folders = append([]string{layerPath}, folders...) diff --git a/components/engine/daemon/images.go b/components/engine/daemon/images/images.go similarity index 97% rename from components/engine/daemon/images.go rename to components/engine/daemon/images/images.go index 903583bbcb..46056f15b5 100644 --- a/components/engine/daemon/images.go +++ b/components/engine/daemon/images/images.go @@ -1,4 +1,4 @@ -package daemon // import "github.com/docker/docker/daemon" +package images // import "github.com/docker/docker/daemon/images" import ( "encoding/json" @@ -34,7 +34,7 @@ func (r byCreated) Swap(i, j int) { r[i], r[j] = r[j], r[i] } func (r byCreated) Less(i, j int) bool { return r[i].Created < r[j].Created } // Map returns a map of all images in the ImageStore -func (i *imageService) Map() map[image.ID]*image.Image { +func (i *ImageService) Map() map[image.ID]*image.Image { return i.imageStore.Map() } @@ -43,7 +43,7 @@ func (i *imageService) Map() map[image.ID]*image.Image { // filter is a shell glob string applied to repository names. The argument // named all controls whether all images in the graph are filtered, or just // the heads. -func (i *imageService) Images(imageFilters filters.Args, all bool, withExtraAttrs bool) ([]*types.ImageSummary, error) { +func (i *ImageService) Images(imageFilters filters.Args, all bool, withExtraAttrs bool) ([]*types.ImageSummary, error) { var ( allImages map[image.ID]*image.Image err error @@ -249,7 +249,7 @@ func (i *imageService) Images(imageFilters filters.Args, all bool, withExtraAttr // This new image contains only the layers from it's parent + 1 extra layer which contains the diff of all the layers in between. // The existing image(s) is not destroyed. // If no parent is specified, a new image with the diff of all the specified image's layers merged into a new layer that has no parents. -func (i *imageService) SquashImage(id, parent string) (string, error) { +func (i *ImageService) SquashImage(id, parent string) (string, error) { var ( img *image.Image diff --git a/components/engine/daemon/images/locals.go b/components/engine/daemon/images/locals.go new file mode 100644 index 0000000000..a57ea2da60 --- /dev/null +++ b/components/engine/daemon/images/locals.go @@ -0,0 +1,32 @@ +package images // import "github.com/docker/docker/daemon/images" + +import ( + "fmt" + + metrics "github.com/docker/go-metrics" +) + +type invalidFilter struct { + filter string + value interface{} +} + +func (e invalidFilter) Error() string { + msg := "Invalid filter '" + e.filter + if e.value != nil { + msg += fmt.Sprintf("=%s", e.value) + } + return msg + "'" +} + +func (e invalidFilter) InvalidParameter() {} + +var imageActions metrics.LabeledTimer + +func init() { + ns := metrics.NewNamespace("engine", "daemon", nil) + imageActions = ns.NewLabeledTimer("image_actions", "The number of seconds it takes to process each image action", "action") + // TODO: is it OK to register a namespace with the same name? Or does this + // need to be exported from somewhere? + metrics.Register(ns) +} diff --git a/components/engine/daemon/images/service.go b/components/engine/daemon/images/service.go new file mode 100644 index 0000000000..70a8bf4455 --- /dev/null +++ b/components/engine/daemon/images/service.go @@ -0,0 +1,229 @@ +package images // import "github.com/docker/docker/daemon/images" + +import ( + "context" + "os" + + "github.com/docker/docker/container" + daemonevents "github.com/docker/docker/daemon/events" + "github.com/docker/docker/distribution/metadata" + "github.com/docker/docker/distribution/xfer" + "github.com/docker/docker/image" + "github.com/docker/docker/layer" + dockerreference "github.com/docker/docker/reference" + "github.com/docker/docker/registry" + "github.com/docker/libtrust" + "github.com/opencontainers/go-digest" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +type containerStore interface { + // used by image delete + First(container.StoreFilter) *container.Container + // used by image prune, and image list + List() []*container.Container + // TODO: remove, only used for CommitBuildStep + Get(string) *container.Container +} + +// ImageServiceConfig is the configuration used to create a new ImageService +type ImageServiceConfig struct { + ContainerStore containerStore + DistributionMetadataStore metadata.Store + EventsService *daemonevents.Events + ImageStore image.Store + LayerStores map[string]layer.Store + MaxConcurrentDownloads int + MaxConcurrentUploads int + ReferenceStore dockerreference.Store + RegistryService registry.Service + TrustKey libtrust.PrivateKey +} + +// NewImageService returns a new ImageService from a configuration +func NewImageService(config ImageServiceConfig) *ImageService { + logrus.Debugf("Max Concurrent Downloads: %d", config.MaxConcurrentDownloads) + logrus.Debugf("Max Concurrent Uploads: %d", config.MaxConcurrentUploads) + return &ImageService{ + containers: config.ContainerStore, + distributionMetadataStore: config.DistributionMetadataStore, + downloadManager: xfer.NewLayerDownloadManager(config.LayerStores, config.MaxConcurrentDownloads), + eventsService: config.EventsService, + imageStore: config.ImageStore, + layerStores: config.LayerStores, + referenceStore: config.ReferenceStore, + registryService: config.RegistryService, + trustKey: config.TrustKey, + uploadManager: xfer.NewLayerUploadManager(config.MaxConcurrentUploads), + } +} + +// ImageService provides a backend for image management +type ImageService struct { + containers containerStore + distributionMetadataStore metadata.Store + downloadManager *xfer.LayerDownloadManager + eventsService *daemonevents.Events + imageStore image.Store + layerStores map[string]layer.Store // By operating system + pruneRunning int32 + referenceStore dockerreference.Store + registryService registry.Service + trustKey libtrust.PrivateKey + uploadManager *xfer.LayerUploadManager +} + +// CountImages returns the number of images stored by ImageService +// called from info.go +func (i *ImageService) CountImages() int { + return len(i.imageStore.Map()) +} + +// Children returns the children image.IDs for a parent image. +// called from list.go to filter containers +// TODO: refactor to expose an ancestry for image.ID? +func (i *ImageService) Children(id image.ID) []image.ID { + return i.imageStore.Children(id) +} + +// CreateLayer creates a filesystem layer for a container. +// called from create.go +// TODO: accept an opt struct instead of container? +func (i *ImageService) CreateLayer(container *container.Container, initFunc layer.MountInit) (layer.RWLayer, error) { + var layerID layer.ChainID + if container.ImageID != "" { + img, err := i.imageStore.Get(container.ImageID) + if err != nil { + return nil, err + } + layerID = img.RootFS.ChainID() + } + + rwLayerOpts := &layer.CreateRWLayerOpts{ + MountLabel: container.MountLabel, + InitFunc: initFunc, + StorageOpt: container.HostConfig.StorageOpt, + } + + // Indexing by OS is safe here as validation of OS has already been performed in create() (the only + // caller), and guaranteed non-nil + return i.layerStores[container.OS].CreateRWLayer(container.ID, layerID, rwLayerOpts) +} + +// GetLayerByID returns a layer by ID and operating system +// called from daemon.go Daemon.restore(), and Daemon.containerExport() +func (i *ImageService) GetLayerByID(cid string, os string) (layer.RWLayer, error) { + return i.layerStores[os].GetRWLayer(cid) +} + +// LayerStoreStatus returns the status for each layer store +// called from info.go +func (i *ImageService) LayerStoreStatus() map[string][][2]string { + result := make(map[string][][2]string) + for os, store := range i.layerStores { + result[os] = store.DriverStatus() + } + return result +} + +// GetLayerMountID returns the mount ID for a layer +// called from daemon.go Daemon.Shutdown(), and Daemon.Cleanup() (cleanup is actually continerCleanup) +// TODO: needs to be refactored to Unmount (see callers), or removed and replaced +// with GetLayerByID +func (i *ImageService) GetLayerMountID(cid string, os string) (string, error) { + return i.layerStores[os].GetMountID(cid) +} + +// Cleanup resources before the process is shutdown. +// called from daemon.go Daemon.Shutdown() +func (i *ImageService) Cleanup() { + for os, ls := range i.layerStores { + if ls != nil { + if err := ls.Cleanup(); err != nil { + logrus.Errorf("Error during layer Store.Cleanup(): %v %s", err, os) + } + } + } +} + +// GraphDriverForOS returns the name of the graph drvier +// moved from Daemon.GraphDriverName, used by: +// - newContainer +// - to report an error in Daemon.Mount(container) +func (i *ImageService) GraphDriverForOS(os string) string { + return i.layerStores[os].DriverName() +} + +// ReleaseLayer releases a layer allowing it to be removed +// called from delete.go Daemon.cleanupContainer(), and Daemon.containerExport() +func (i *ImageService) ReleaseLayer(rwlayer layer.RWLayer, containerOS string) error { + metadata, err := i.layerStores[containerOS].ReleaseRWLayer(rwlayer) + layer.LogReleaseMetadata(metadata) + if err != nil && err != layer.ErrMountDoesNotExist && !os.IsNotExist(errors.Cause(err)) { + return errors.Wrapf(err, "driver %q failed to remove root filesystem", + i.layerStores[containerOS].DriverName()) + } + return nil +} + +// LayerDiskUsage returns the number of bytes used by layer stores +// called from disk_usage.go +func (i *ImageService) LayerDiskUsage(ctx context.Context) (int64, error) { + var allLayersSize int64 + layerRefs := i.getLayerRefs() + for _, ls := range i.layerStores { + allLayers := ls.Map() + for _, l := range allLayers { + select { + case <-ctx.Done(): + return allLayersSize, ctx.Err() + default: + size, err := l.DiffSize() + if err == nil { + if _, ok := layerRefs[l.ChainID()]; ok { + allLayersSize += size + } else { + logrus.Warnf("found leaked image layer %v", l.ChainID()) + } + } else { + logrus.Warnf("failed to get diff size for layer %v", l.ChainID()) + } + } + } + } + return allLayersSize, nil +} + +func (i *ImageService) getLayerRefs() map[layer.ChainID]int { + tmpImages := i.imageStore.Map() + layerRefs := map[layer.ChainID]int{} + for id, img := range tmpImages { + dgst := digest.Digest(id) + if len(i.referenceStore.References(dgst)) == 0 && len(i.imageStore.Children(id)) != 0 { + continue + } + + rootFS := *img.RootFS + rootFS.DiffIDs = nil + for _, id := range img.RootFS.DiffIDs { + rootFS.Append(id) + chid := rootFS.ChainID() + layerRefs[chid]++ + } + } + + return layerRefs +} + +// UpdateConfig values +// +// called from reload.go +func (i *ImageService) UpdateConfig(maxDownloads, maxUploads *int) { + if i.downloadManager != nil && maxDownloads != nil { + i.downloadManager.SetConcurrency(*maxDownloads) + } + if i.uploadManager != nil && maxUploads != nil { + i.uploadManager.SetConcurrency(*maxUploads) + } +} diff --git a/components/engine/daemon/info.go b/components/engine/daemon/info.go index 24940be16a..6f5ae69896 100644 --- a/components/engine/daemon/info.go +++ b/components/engine/daemon/info.go @@ -80,7 +80,7 @@ func (daemon *Daemon) SystemInfo() (*types.Info, error) { var ds [][2]string drivers := "" - statuses := daemon.imageService.GraphDriverStatuses() + statuses := daemon.imageService.LayerStoreStatus() for os, gd := range daemon.graphDrivers { ds = append(ds, statuses[os]...) drivers += gd diff --git a/components/engine/daemon/list.go b/components/engine/daemon/list.go index 254cb7253a..ddc6fe70be 100644 --- a/components/engine/daemon/list.go +++ b/components/engine/daemon/list.go @@ -9,6 +9,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/container" + "github.com/docker/docker/daemon/images" "github.com/docker/docker/errdefs" "github.com/docker/docker/image" "github.com/docker/docker/volume" @@ -592,7 +593,7 @@ func (daemon *Daemon) refreshImage(s *container.Snapshot, ctx *listContext) (*ty image := s.Image // keep the original ref if still valid (hasn't changed) if image != s.ImageID { id, _, err := daemon.imageService.GetImageIDAndOS(image) - if _, isDNE := err.(errImageDoesNotExist); err != nil && !isDNE { + if _, isDNE := err.(images.ErrImageDoesNotExist); err != nil && !isDNE { return nil, err } if err != nil || id.String() != s.ImageID { diff --git a/components/engine/daemon/metrics.go b/components/engine/daemon/metrics.go index b283526d9a..02a36603c9 100644 --- a/components/engine/daemon/metrics.go +++ b/components/engine/daemon/metrics.go @@ -14,7 +14,6 @@ const metricsPluginType = "MetricsCollector" var ( containerActions metrics.LabeledTimer - imageActions metrics.LabeledTimer networkActions metrics.LabeledTimer engineInfo metrics.LabeledGauge engineCpus metrics.Gauge @@ -52,7 +51,6 @@ func init() { engineMemory = ns.NewGauge("engine_memory", "The number of bytes of memory that the host system of the engine has", metrics.Bytes) healthChecksCounter = ns.NewCounter("health_checks", "The total number of health checks") healthChecksFailedCounter = ns.NewCounter("health_checks_failed", "The total number of failed health checks") - imageActions = ns.NewLabeledTimer("image_actions", "The number of seconds it takes to process each image action", "action") stateCtr = newStateCounter(ns.NewDesc("container_states", "The count of containers in various states", metrics.Unit("containers"), "state")) ns.Add(stateCtr) diff --git a/components/engine/daemon/oci_windows.go b/components/engine/daemon/oci_windows.go index 93d4ae2297..98243bcf8a 100644 --- a/components/engine/daemon/oci_windows.go +++ b/components/engine/daemon/oci_windows.go @@ -138,7 +138,7 @@ func (daemon *Daemon) createSpec(c *container.Container) (*specs.Spec, error) { } } s.Process.User.Username = c.Config.User - s.Windows.LayerFolders, err = daemon.GetLayerFolders(img, c.RWLayer) + s.Windows.LayerFolders, err = daemon.imageService.GetLayerFolders(img, c.RWLayer) if err != nil { return nil, errors.Wrapf(err, "container %s", c.ID) } diff --git a/components/engine/daemon/reload.go b/components/engine/daemon/reload.go index 5d5cba985d..210864ff87 100644 --- a/components/engine/daemon/reload.go +++ b/components/engine/daemon/reload.go @@ -90,12 +90,6 @@ func (daemon *Daemon) reloadMaxConcurrentDownloadsAndUploads(conf *config.Config daemon.configStore.MaxConcurrentDownloads = &maxConcurrentDownloads } logrus.Debugf("Reset Max Concurrent Downloads: %d", *daemon.configStore.MaxConcurrentDownloads) - if daemon.imageService.downloadManager != nil { - daemon.imageService.downloadManager.SetConcurrency(*daemon.configStore.MaxConcurrentDownloads) - } - - // prepare reload event attributes with updatable configurations - attributes["max-concurrent-downloads"] = fmt.Sprintf("%d", *daemon.configStore.MaxConcurrentDownloads) // If no value is set for max-concurrent-upload we assume it is the default value // We always "reset" as the cost is lightweight and easy to maintain. @@ -106,10 +100,10 @@ func (daemon *Daemon) reloadMaxConcurrentDownloadsAndUploads(conf *config.Config daemon.configStore.MaxConcurrentUploads = &maxConcurrentUploads } logrus.Debugf("Reset Max Concurrent Uploads: %d", *daemon.configStore.MaxConcurrentUploads) - if daemon.imageService.uploadManager != nil { - daemon.imageService.uploadManager.SetConcurrency(*daemon.configStore.MaxConcurrentUploads) - } + daemon.imageService.UpdateConfig(conf.MaxConcurrentDownloads, conf.MaxConcurrentUploads) + // prepare reload event attributes with updatable configurations + attributes["max-concurrent-downloads"] = fmt.Sprintf("%d", *daemon.configStore.MaxConcurrentDownloads) // prepare reload event attributes with updatable configurations attributes["max-concurrent-uploads"] = fmt.Sprintf("%d", *daemon.configStore.MaxConcurrentUploads) } diff --git a/components/engine/daemon/reload_test.go b/components/engine/daemon/reload_test.go index 3e43b6d818..a2500b2bb2 100644 --- a/components/engine/daemon/reload_test.go +++ b/components/engine/daemon/reload_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/docker/docker/daemon/config" + "github.com/docker/docker/daemon/images" "github.com/docker/docker/pkg/discovery" _ "github.com/docker/docker/pkg/discovery/memory" "github.com/docker/docker/registry" @@ -21,7 +22,7 @@ func TestDaemonReloadLabels(t *testing.T) { Labels: []string{"foo:bar"}, }, }, - imageService: &imageService{}, + imageService: images.NewImageService(images.ImageServiceConfig{}), } valuesSets := make(map[string]interface{}) @@ -46,7 +47,7 @@ func TestDaemonReloadLabels(t *testing.T) { func TestDaemonReloadAllowNondistributableArtifacts(t *testing.T) { daemon := &Daemon{ configStore: &config.Config{}, - imageService: &imageService{}, + imageService: images.NewImageService(images.ImageServiceConfig{}), } var err error @@ -101,7 +102,7 @@ func TestDaemonReloadAllowNondistributableArtifacts(t *testing.T) { func TestDaemonReloadMirrors(t *testing.T) { daemon := &Daemon{ - imageService: &imageService{}, + imageService: images.NewImageService(images.ImageServiceConfig{}), } var err error daemon.RegistryService, err = registry.NewService(registry.ServiceOptions{ @@ -200,7 +201,7 @@ func TestDaemonReloadMirrors(t *testing.T) { func TestDaemonReloadInsecureRegistries(t *testing.T) { daemon := &Daemon{ - imageService: &imageService{}, + imageService: images.NewImageService(images.ImageServiceConfig{}), } var err error // initialize daemon with existing insecure registries: "127.0.0.0/8", "10.10.1.11:5000", "10.10.1.22:5000" @@ -292,7 +293,7 @@ func TestDaemonReloadInsecureRegistries(t *testing.T) { func TestDaemonReloadNotAffectOthers(t *testing.T) { daemon := &Daemon{ - imageService: &imageService{}, + imageService: images.NewImageService(images.ImageServiceConfig{}), } daemon.configStore = &config.Config{ CommonConfig: config.CommonConfig{ @@ -326,7 +327,7 @@ func TestDaemonReloadNotAffectOthers(t *testing.T) { func TestDaemonDiscoveryReload(t *testing.T) { daemon := &Daemon{ - imageService: &imageService{}, + imageService: images.NewImageService(images.ImageServiceConfig{}), } daemon.configStore = &config.Config{ CommonConfig: config.CommonConfig{ @@ -405,7 +406,7 @@ func TestDaemonDiscoveryReload(t *testing.T) { func TestDaemonDiscoveryReloadFromEmptyDiscovery(t *testing.T) { daemon := &Daemon{ - imageService: &imageService{}, + imageService: images.NewImageService(images.ImageServiceConfig{}), } daemon.configStore = &config.Config{} @@ -452,7 +453,7 @@ func TestDaemonDiscoveryReloadFromEmptyDiscovery(t *testing.T) { func TestDaemonDiscoveryReloadOnlyClusterAdvertise(t *testing.T) { daemon := &Daemon{ - imageService: &imageService{}, + imageService: images.NewImageService(images.ImageServiceConfig{}), } daemon.configStore = &config.Config{ CommonConfig: config.CommonConfig{ @@ -498,7 +499,7 @@ func TestDaemonDiscoveryReloadOnlyClusterAdvertise(t *testing.T) { func TestDaemonReloadNetworkDiagnosticPort(t *testing.T) { daemon := &Daemon{ - imageService: &imageService{}, + imageService: images.NewImageService(images.ImageServiceConfig{}), } daemon.configStore = &config.Config{} diff --git a/components/engine/daemon/start.go b/components/engine/daemon/start.go index ad61747a2f..c00bd9ceb2 100644 --- a/components/engine/daemon/start.go +++ b/components/engine/daemon/start.go @@ -223,7 +223,7 @@ func (daemon *Daemon) Cleanup(container *container.Container) { if err := daemon.conditionalUnmountOnCleanup(container); err != nil { // FIXME: remove once reference counting for graphdrivers has been refactored // Ensure that all the mounts are gone - if mountid, err := daemon.imageService.GetContainerMountID(container.ID, container.OS); err == nil { + if mountid, err := daemon.imageService.GetLayerMountID(container.ID, container.OS); err == nil { daemon.cleanupMountsByID(mountid) } } From 103568711da010eb17e860b4416bee07ea3e8a43 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 14 Feb 2018 15:19:37 -0500 Subject: [PATCH 09/20] Remove unnecessary GetImageIDAndOS use GetImage Signed-off-by: Daniel Nephin Upstream-commit: c10e6a4d15b907da22ab508770d67e1447a8d0bd Component: engine --- components/engine/daemon/images/image.go | 31 ++++++------------- .../engine/daemon/images/image_delete.go | 5 +-- components/engine/daemon/images/image_tag.go | 4 +-- components/engine/daemon/list.go | 10 +++--- 4 files changed, 20 insertions(+), 30 deletions(-) diff --git a/components/engine/daemon/images/image.go b/components/engine/daemon/images/image.go index 7dda553138..79cc07c4fd 100644 --- a/components/engine/daemon/images/image.go +++ b/components/engine/daemon/images/image.go @@ -24,32 +24,30 @@ func (e ErrImageDoesNotExist) Error() string { // NotFound implements the NotFound interface func (e ErrImageDoesNotExist) NotFound() {} -// GetImageIDAndOS returns an image ID and operating system corresponding to the image referred to by -// refOrID. -// called from list.go foldFilter() -func (i ImageService) GetImageIDAndOS(refOrID string) (image.ID, string, error) { +// GetImage returns an image corresponding to the image referred to by refOrID. +func (i *ImageService) GetImage(refOrID string) (*image.Image, error) { ref, err := reference.ParseAnyReference(refOrID) if err != nil { - return "", "", errdefs.InvalidParameter(err) + return nil, errdefs.InvalidParameter(err) } namedRef, ok := ref.(reference.Named) if !ok { digested, ok := ref.(reference.Digested) if !ok { - return "", "", ErrImageDoesNotExist{ref} + return nil, ErrImageDoesNotExist{ref} } id := image.IDFromDigest(digested.Digest()) if img, err := i.imageStore.Get(id); err == nil { - return id, img.OperatingSystem(), nil + return img, nil } - return "", "", ErrImageDoesNotExist{ref} + return nil, ErrImageDoesNotExist{ref} } if digest, err := i.referenceStore.Get(namedRef); err == nil { // Search the image stores to get the operating system, defaulting to host OS. id := image.IDFromDigest(digest) if img, err := i.imageStore.Get(id); err == nil { - return id, img.OperatingSystem(), nil + return img, nil } } @@ -57,19 +55,10 @@ func (i ImageService) GetImageIDAndOS(refOrID string) (image.ID, string, error) if id, err := i.imageStore.Search(refOrID); err == nil { img, err := i.imageStore.Get(id) if err != nil { - return "", "", ErrImageDoesNotExist{ref} + return nil, ErrImageDoesNotExist{ref} } - return id, img.OperatingSystem(), nil + return img, nil } - return "", "", ErrImageDoesNotExist{ref} -} - -// GetImage returns an image corresponding to the image referred to by refOrID. -func (i *ImageService) GetImage(refOrID string) (*image.Image, error) { - imgID, _, err := i.GetImageIDAndOS(refOrID) - if err != nil { - return nil, err - } - return i.imageStore.Get(imgID) + return nil, ErrImageDoesNotExist{ref} } diff --git a/components/engine/daemon/images/image_delete.go b/components/engine/daemon/images/image_delete.go index 60f60afaa7..60896e487a 100644 --- a/components/engine/daemon/images/image_delete.go +++ b/components/engine/daemon/images/image_delete.go @@ -64,14 +64,15 @@ func (i *ImageService) ImageDelete(imageRef string, force, prune bool) ([]types. start := time.Now() records := []types.ImageDeleteResponseItem{} - imgID, operatingSystem, err := i.GetImageIDAndOS(imageRef) + img, err := i.GetImage(imageRef) if err != nil { return nil, err } - if !system.IsOSSupported(operatingSystem) { + if !system.IsOSSupported(img.OperatingSystem()) { return nil, errors.Errorf("unable to delete image: %q", system.ErrNotSupportedOperatingSystem) } + imgID := img.ID() repoRefs := i.referenceStore.References(imgID.Digest()) using := func(c *container.Container) bool { diff --git a/components/engine/daemon/images/image_tag.go b/components/engine/daemon/images/image_tag.go index 6456ff81f9..4693611c3a 100644 --- a/components/engine/daemon/images/image_tag.go +++ b/components/engine/daemon/images/image_tag.go @@ -8,7 +8,7 @@ import ( // TagImage creates the tag specified by newTag, pointing to the image named // imageName (alternatively, imageName can also be an image ID). func (i *ImageService) TagImage(imageName, repository, tag string) (string, error) { - imageID, _, err := i.GetImageIDAndOS(imageName) + img, err := i.GetImage(imageName) if err != nil { return "", err } @@ -23,7 +23,7 @@ func (i *ImageService) TagImage(imageName, repository, tag string) (string, erro } } - err = i.TagImageWithReference(imageID, newTag) + err = i.TagImageWithReference(img.ID(), newTag) return reference.FamiliarString(newTag), err } diff --git a/components/engine/daemon/list.go b/components/engine/daemon/list.go index ddc6fe70be..84cacc344a 100644 --- a/components/engine/daemon/list.go +++ b/components/engine/daemon/list.go @@ -324,17 +324,17 @@ func (daemon *Daemon) foldFilter(view container.View, config *types.ContainerLis if psFilters.Contains("ancestor") { ancestorFilter = true psFilters.WalkValues("ancestor", func(ancestor string) error { - id, _, err := daemon.imageService.GetImageIDAndOS(ancestor) + img, err := daemon.imageService.GetImage(ancestor) if err != nil { logrus.Warnf("Error while looking up for image %v", ancestor) return nil } - if imagesFilter[id] { + if imagesFilter[img.ID()] { // Already seen this ancestor, skip it return nil } // Then walk down the graph and put the imageIds in imagesFilter - populateImageFilterByParents(imagesFilter, id, daemon.imageService.Children) + populateImageFilterByParents(imagesFilter, img.ID(), daemon.imageService.Children) return nil }) } @@ -592,11 +592,11 @@ func (daemon *Daemon) refreshImage(s *container.Snapshot, ctx *listContext) (*ty c := s.Container image := s.Image // keep the original ref if still valid (hasn't changed) if image != s.ImageID { - id, _, err := daemon.imageService.GetImageIDAndOS(image) + img, err := daemon.imageService.GetImage(image) if _, isDNE := err.(images.ErrImageDoesNotExist); err != nil && !isDNE { return nil, err } - if err != nil || id.String() != s.ImageID { + if err != nil || img.ImageID() != s.ImageID { // ref changed, we need to use original ID image = s.ImageID } From 6aa61cd6833040234fca0b62feae2fce5bf83576 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 12 Feb 2018 21:54:12 +0000 Subject: [PATCH 10/20] Migrate volumes tests in integration-cli to api tests This fix migrates volumes tests in integration-cli to api tests in integration/ Signed-off-by: Yong Tang Upstream-commit: d896f87c0595134fa2f0787dad30b237815f233f Component: engine --- .../docker_api_volumes_test.go | 103 ---------------- .../integration/internal/container/ops.go | 19 +++ .../engine/integration/volume/main_test.go | 33 +++++ .../engine/integration/volume/volume_test.go | 115 ++++++++++++++++++ 4 files changed, 167 insertions(+), 103 deletions(-) delete mode 100644 components/engine/integration-cli/docker_api_volumes_test.go create mode 100644 components/engine/integration/volume/main_test.go create mode 100644 components/engine/integration/volume/volume_test.go diff --git a/components/engine/integration-cli/docker_api_volumes_test.go b/components/engine/integration-cli/docker_api_volumes_test.go deleted file mode 100644 index 65a9652092..0000000000 --- a/components/engine/integration-cli/docker_api_volumes_test.go +++ /dev/null @@ -1,103 +0,0 @@ -package main - -import ( - "fmt" - "path/filepath" - "strings" - "time" - - "github.com/docker/docker/api/types/filters" - volumetypes "github.com/docker/docker/api/types/volume" - "github.com/docker/docker/client" - "github.com/docker/docker/integration-cli/checker" - "github.com/go-check/check" - "golang.org/x/net/context" -) - -func (s *DockerSuite) TestVolumesAPIList(c *check.C) { - prefix, _ := getPrefixAndSlashFromDaemonPlatform() - cid, _ := dockerCmd(c, "run", "-d", "-v", prefix+"/foo", "busybox") - - cli, err := client.NewEnvClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - container, err := cli.ContainerInspect(context.Background(), strings.TrimSpace(cid)) - c.Assert(err, checker.IsNil) - vname := container.Mounts[0].Name - - volumes, err := cli.VolumeList(context.Background(), filters.Args{}) - c.Assert(err, checker.IsNil) - - found := false - for _, vol := range volumes.Volumes { - if vol.Name == vname { - found = true - break - } - } - c.Assert(found, checker.Equals, true) -} - -func (s *DockerSuite) TestVolumesAPICreate(c *check.C) { - config := volumetypes.VolumesCreateBody{ - Name: "test", - } - - cli, err := client.NewEnvClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - vol, err := cli.VolumeCreate(context.Background(), config) - c.Assert(err, check.IsNil) - - c.Assert(filepath.Base(filepath.Dir(vol.Mountpoint)), checker.Equals, config.Name) -} - -func (s *DockerSuite) TestVolumesAPIRemove(c *check.C) { - prefix, _ := getPrefixAndSlashFromDaemonPlatform() - cid, _ := dockerCmd(c, "run", "-d", "-v", prefix+"/foo", "--name=test", "busybox") - - cli, err := client.NewEnvClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - container, err := cli.ContainerInspect(context.Background(), strings.TrimSpace(cid)) - c.Assert(err, checker.IsNil) - vname := container.Mounts[0].Name - - err = cli.VolumeRemove(context.Background(), vname, false) - c.Assert(err.Error(), checker.Contains, "volume is in use") - - dockerCmd(c, "rm", "-f", "test") - err = cli.VolumeRemove(context.Background(), vname, false) - c.Assert(err, checker.IsNil) -} - -func (s *DockerSuite) TestVolumesAPIInspect(c *check.C) { - config := volumetypes.VolumesCreateBody{ - Name: "test", - } - - // sampling current time minus a minute so to now have false positive in case of delays - now := time.Now().Truncate(time.Minute) - - cli, err := client.NewEnvClient() - c.Assert(err, checker.IsNil) - defer cli.Close() - - _, err = cli.VolumeCreate(context.Background(), config) - c.Assert(err, check.IsNil) - - vol, err := cli.VolumeInspect(context.Background(), config.Name) - c.Assert(err, checker.IsNil) - c.Assert(vol.Name, checker.Equals, config.Name) - - // comparing CreatedAt field time for the new volume to now. Removing a minute from both to avoid false positive - testCreatedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(vol.CreatedAt)) - c.Assert(err, check.IsNil) - testCreatedAt = testCreatedAt.Truncate(time.Minute) - if !testCreatedAt.Equal(now) { - c.Assert(fmt.Errorf("Time Volume is CreatedAt not equal to current time"), check.NotNil) - } -} diff --git a/components/engine/integration/internal/container/ops.go b/components/engine/integration/internal/container/ops.go index e3d538bef1..9360527d37 100644 --- a/components/engine/integration/internal/container/ops.go +++ b/components/engine/integration/internal/container/ops.go @@ -1,6 +1,8 @@ package container import ( + "fmt" + containertypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/strslice" "github.com/docker/go-connections/nat" @@ -57,3 +59,20 @@ func WithWorkingDir(dir string) func(*TestContainerConfig) { c.Config.WorkingDir = dir } } + +// WithVolume sets the volume of the container +func WithVolume(name string) func(*TestContainerConfig) { + return func(c *TestContainerConfig) { + if c.Config.Volumes == nil { + c.Config.Volumes = map[string]struct{}{} + } + c.Config.Volumes[name] = struct{}{} + } +} + +// WithBind sets the bind mount of the container +func WithBind(src, target string) func(*TestContainerConfig) { + return func(c *TestContainerConfig) { + c.HostConfig.Binds = append(c.HostConfig.Binds, fmt.Sprintf("%s:%s", src, target)) + } +} diff --git a/components/engine/integration/volume/main_test.go b/components/engine/integration/volume/main_test.go new file mode 100644 index 0000000000..206f7377ae --- /dev/null +++ b/components/engine/integration/volume/main_test.go @@ -0,0 +1,33 @@ +package volume // import "github.com/docker/docker/integration/volume" + +import ( + "fmt" + "os" + "testing" + + "github.com/docker/docker/internal/test/environment" +) + +var testEnv *environment.Execution + +func TestMain(m *testing.M) { + var err error + testEnv, err = environment.New() + if err != nil { + fmt.Println(err) + os.Exit(1) + } + err = environment.EnsureFrozenImagesLinux(testEnv) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + testEnv.Print() + os.Exit(m.Run()) +} + +func setupTest(t *testing.T) func() { + environment.ProtectAll(t, testEnv) + return func() { testEnv.Clean(t) } +} diff --git a/components/engine/integration/volume/volume_test.go b/components/engine/integration/volume/volume_test.go new file mode 100644 index 0000000000..38ce5782e1 --- /dev/null +++ b/components/engine/integration/volume/volume_test.go @@ -0,0 +1,115 @@ +package volume + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" + volumetypes "github.com/docker/docker/api/types/volume" + "github.com/docker/docker/integration/internal/container" + "github.com/docker/docker/integration/internal/request" + "github.com/docker/docker/internal/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestVolumesCreateAndList(t *testing.T) { + defer setupTest(t)() + client := request.NewAPIClient(t) + ctx := context.Background() + + name := t.Name() + vol, err := client.VolumeCreate(ctx, volumetypes.VolumesCreateBody{ + Name: name, + }) + require.NoError(t, err) + + expected := types.Volume{ + // Ignore timestamp of CreatedAt + CreatedAt: vol.CreatedAt, + Driver: "local", + Scope: "local", + Name: name, + Options: map[string]string{}, + Mountpoint: fmt.Sprintf("%s/volumes/%s/_data", testEnv.DaemonInfo.DockerRootDir, name), + } + assert.Equal(t, vol, expected) + + volumes, err := client.VolumeList(ctx, filters.Args{}) + require.NoError(t, err) + + assert.Equal(t, len(volumes.Volumes), 1) + assert.NotNil(t, volumes.Volumes[0]) + assert.Equal(t, *volumes.Volumes[0], expected) +} + +func TestVolumesRemove(t *testing.T) { + defer setupTest(t)() + client := request.NewAPIClient(t) + ctx := context.Background() + + prefix, _ := getPrefixAndSlashFromDaemonPlatform() + + id := container.Create(t, ctx, client, container.WithVolume(prefix+"foo")) + + c, err := client.ContainerInspect(ctx, id) + require.NoError(t, err) + vname := c.Mounts[0].Name + + err = client.VolumeRemove(ctx, vname, false) + testutil.ErrorContains(t, err, "volume is in use") + + err = client.ContainerRemove(ctx, id, types.ContainerRemoveOptions{ + Force: true, + }) + require.NoError(t, err) + + err = client.VolumeRemove(ctx, vname, false) + require.NoError(t, err) +} + +func TestVolumesInspect(t *testing.T) { + defer setupTest(t)() + client := request.NewAPIClient(t) + ctx := context.Background() + + // sampling current time minus a minute so to now have false positive in case of delays + now := time.Now().Truncate(time.Minute) + + name := t.Name() + _, err := client.VolumeCreate(ctx, volumetypes.VolumesCreateBody{ + Name: name, + }) + require.NoError(t, err) + + vol, err := client.VolumeInspect(ctx, name) + require.NoError(t, err) + + expected := types.Volume{ + // Ignore timestamp of CreatedAt + CreatedAt: vol.CreatedAt, + Driver: "local", + Scope: "local", + Name: name, + Options: map[string]string{}, + Mountpoint: fmt.Sprintf("%s/volumes/%s/_data", testEnv.DaemonInfo.DockerRootDir, name), + } + assert.Equal(t, vol, expected) + + // comparing CreatedAt field time for the new volume to now. Removing a minute from both to avoid false positive + testCreatedAt, err := time.Parse(time.RFC3339, strings.TrimSpace(vol.CreatedAt)) + require.NoError(t, err) + testCreatedAt = testCreatedAt.Truncate(time.Minute) + assert.Equal(t, testCreatedAt.Equal(now), true, "Time Volume is CreatedAt not equal to current time") +} + +func getPrefixAndSlashFromDaemonPlatform() (prefix, slash string) { + if testEnv.OSType == "windows" { + return "c:", `\` + } + return "", "/" +} From 55e0468775ac12057a8e8d43b219c7ceb61db658 Mon Sep 17 00:00:00 2001 From: Dennis Chen Date: Tue, 27 Feb 2018 06:28:29 +0000 Subject: [PATCH 11/20] Add `busybox:latest` into the frozen images Adding `busybox:latest` and `busybox:glibc` as the frozen images Signed-off-by: Dennis Chen Upstream-commit: 3ae45c5f173d88ba621116f9e1b5611fe687e050 Component: engine --- components/engine/Dockerfile | 3 ++- components/engine/Dockerfile.aarch64 | 3 ++- components/engine/Dockerfile.armhf | 3 ++- components/engine/Dockerfile.e2e | 3 ++- components/engine/Dockerfile.ppc64le | 3 ++- components/engine/Dockerfile.s390x | 3 ++- 6 files changed, 12 insertions(+), 6 deletions(-) diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile index 49a8c59967..ada8b241e7 100644 --- a/components/engine/Dockerfile +++ b/components/engine/Dockerfile @@ -130,7 +130,8 @@ RUN apt-get update && apt-get install -y jq ca-certificates --no-install-recomme COPY contrib/download-frozen-image-v2.sh / RUN /download-frozen-image-v2.sh /docker-frozen-images \ buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ - busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \ + busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \ + busybox:glibc@sha256:0b55a30394294ab23b9afd58fab94e61a923f5834fba7ddbae7f8e0c11ba85e6 \ debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) diff --git a/components/engine/Dockerfile.aarch64 b/components/engine/Dockerfile.aarch64 index 068d80516f..394f593c2e 100644 --- a/components/engine/Dockerfile.aarch64 +++ b/components/engine/Dockerfile.aarch64 @@ -147,7 +147,8 @@ RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ - busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \ + busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \ + busybox:glibc@sha256:0b55a30394294ab23b9afd58fab94e61a923f5834fba7ddbae7f8e0c11ba85e6 \ debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) diff --git a/components/engine/Dockerfile.armhf b/components/engine/Dockerfile.armhf index 0e03fb0a1a..58d52aadce 100644 --- a/components/engine/Dockerfile.armhf +++ b/components/engine/Dockerfile.armhf @@ -133,7 +133,8 @@ RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ - busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \ + busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \ + busybox:glibc@sha256:0b55a30394294ab23b9afd58fab94e61a923f5834fba7ddbae7f8e0c11ba85e6 \ debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) diff --git a/components/engine/Dockerfile.e2e b/components/engine/Dockerfile.e2e index ce13170e32..bd700c98e2 100644 --- a/components/engine/Dockerfile.e2e +++ b/components/engine/Dockerfile.e2e @@ -17,7 +17,8 @@ WORKDIR /go/src/github.com/docker/docker/ COPY contrib/download-frozen-image-v2.sh contrib/download-frozen-image-v2.sh RUN contrib/download-frozen-image-v2.sh /output/docker-frozen-images \ buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ - busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \ + busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \ + busybox:glibc@sha256:0b55a30394294ab23b9afd58fab94e61a923f5834fba7ddbae7f8e0c11ba85e6 \ debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c diff --git a/components/engine/Dockerfile.ppc64le b/components/engine/Dockerfile.ppc64le index 07f4b01492..5af94acf5b 100644 --- a/components/engine/Dockerfile.ppc64le +++ b/components/engine/Dockerfile.ppc64le @@ -131,7 +131,8 @@ RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ - busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \ + busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \ + busybox:glibc@sha256:0b55a30394294ab23b9afd58fab94e61a923f5834fba7ddbae7f8e0c11ba85e6 \ debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) diff --git a/components/engine/Dockerfile.s390x b/components/engine/Dockerfile.s390x index dc55d57231..9d772dd073 100644 --- a/components/engine/Dockerfile.s390x +++ b/components/engine/Dockerfile.s390x @@ -125,7 +125,8 @@ RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \ buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \ - busybox:1.27-glibc@sha256:8c8f261a462eead45ab8e610d3e8f7a1e4fd1cd9bed5bc0a0c386784ab105d8e \ + busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \ + busybox:glibc@sha256:0b55a30394294ab23b9afd58fab94e61a923f5834fba7ddbae7f8e0c11ba85e6 \ debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \ hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) From 37a473147f3ba26df5f8f607f8a3dfb34342f1a1 Mon Sep 17 00:00:00 2001 From: Dennis Chen Date: Tue, 27 Feb 2018 06:49:30 +0000 Subject: [PATCH 12/20] Network testing with `busybox:glibc` Using the `busybox:glibc` instead of `busybox:latest` to the network related test cases (`ping` issue). Signed-off-by: Dennis Chen Upstream-commit: 0d31dee5ec724731607e277a415b1ca4ecb7b2c4 Component: engine --- .../integration-cli/docker_api_swarm_test.go | 2 + .../docker_cli_network_unix_test.go | 14 ++--- .../integration-cli/docker_cli_run_test.go | 4 +- .../integration-cli/docker_cli_swarm_test.go | 6 +-- .../docker_experimental_network_test.go | 52 +++++++++---------- .../integration-cli/fixtures/load/frozen.go | 3 -- .../internal/test/environment/protect.go | 2 +- 7 files changed, 41 insertions(+), 42 deletions(-) diff --git a/components/engine/integration-cli/docker_api_swarm_test.go b/components/engine/integration-cli/docker_api_swarm_test.go index fd0b4e6278..b5edfd2c04 100644 --- a/components/engine/integration-cli/docker_api_swarm_test.go +++ b/components/engine/integration-cli/docker_api_swarm_test.go @@ -910,6 +910,8 @@ func (s *DockerSwarmSuite) TestAPIDuplicateNetworks(c *check.C) { // Test case for 30178 func (s *DockerSwarmSuite) TestAPISwarmHealthcheckNone(c *check.C) { + // Issue #36386 can be a independent one, which is worth further investigation. + c.Skip("Root cause of Issue #36386 is needed") d := s.AddDaemon(c, true, true) out, err := d.Cmd("network", "create", "-d", "overlay", "lb") diff --git a/components/engine/integration-cli/docker_cli_network_unix_test.go b/components/engine/integration-cli/docker_cli_network_unix_test.go index 90e4f6c1f2..4e9edba5a5 100644 --- a/components/engine/integration-cli/docker_cli_network_unix_test.go +++ b/components/engine/integration-cli/docker_cli_network_unix_test.go @@ -1541,10 +1541,10 @@ func (s *DockerSuite) TestUserDefinedNetworkConnectDisconnectAlias(c *check.C) { dockerCmd(c, "network", "create", "-d", "bridge", "net1") dockerCmd(c, "network", "create", "-d", "bridge", "net2") - cid, _ := dockerCmd(c, "run", "-d", "--net=net1", "--name=first", "--net-alias=foo", "busybox", "top") + cid, _ := dockerCmd(c, "run", "-d", "--net=net1", "--name=first", "--net-alias=foo", "busybox:glibc", "top") c.Assert(waitRun("first"), check.IsNil) - dockerCmd(c, "run", "-d", "--net=net1", "--name=second", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=net1", "--name=second", "busybox:glibc", "top") c.Assert(waitRun("second"), check.IsNil) // ping first container and its alias @@ -1581,7 +1581,7 @@ func (s *DockerSuite) TestUserDefinedNetworkConnectDisconnectAlias(c *check.C) { c.Assert(err, check.IsNil) // verify the alias option is rejected when running on predefined network - out, _, err := dockerCmdWithError("run", "--rm", "--name=any", "--net-alias=any", "busybox", "top") + out, _, err := dockerCmdWithError("run", "--rm", "--name=any", "--net-alias=any", "busybox:glibc", "top") c.Assert(err, checker.NotNil, check.Commentf("out: %s", out)) c.Assert(out, checker.Contains, runconfig.ErrUnsupportedNetworkAndAlias.Error()) @@ -1595,10 +1595,10 @@ func (s *DockerSuite) TestUserDefinedNetworkConnectivity(c *check.C) { testRequires(c, DaemonIsLinux, NotUserNamespace) dockerCmd(c, "network", "create", "-d", "bridge", "br.net1") - dockerCmd(c, "run", "-d", "--net=br.net1", "--name=c1.net1", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=br.net1", "--name=c1.net1", "busybox:glibc", "top") c.Assert(waitRun("c1.net1"), check.IsNil) - dockerCmd(c, "run", "-d", "--net=br.net1", "--name=c2.net1", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=br.net1", "--name=c2.net1", "busybox:glibc", "top") c.Assert(waitRun("c2.net1"), check.IsNil) // ping first container by its unqualified name @@ -1643,9 +1643,9 @@ func (s *DockerSuite) TestDockerNetworkInternalMode(c *check.C) { nr := getNetworkResource(c, "internal") c.Assert(nr.Internal, checker.True) - dockerCmd(c, "run", "-d", "--net=internal", "--name=first", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=internal", "--name=first", "busybox:glibc", "top") c.Assert(waitRun("first"), check.IsNil) - dockerCmd(c, "run", "-d", "--net=internal", "--name=second", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=internal", "--name=second", "busybox:glibc", "top") c.Assert(waitRun("second"), check.IsNil) out, _, err := dockerCmdWithError("exec", "first", "ping", "-W", "4", "-c", "1", "www.google.com") c.Assert(err, check.NotNil) diff --git a/components/engine/integration-cli/docker_cli_run_test.go b/components/engine/integration-cli/docker_cli_run_test.go index 776f0e5ba5..aa781a47e1 100644 --- a/components/engine/integration-cli/docker_cli_run_test.go +++ b/components/engine/integration-cli/docker_cli_run_test.go @@ -294,7 +294,7 @@ func (s *DockerSuite) TestUserDefinedNetworkAlias(c *check.C) { testRequires(c, DaemonIsLinux, NotUserNamespace, NotArm) dockerCmd(c, "network", "create", "-d", "bridge", "net1") - cid1, _ := dockerCmd(c, "run", "-d", "--net=net1", "--name=first", "--net-alias=foo1", "--net-alias=foo2", "busybox", "top") + cid1, _ := dockerCmd(c, "run", "-d", "--net=net1", "--name=first", "--net-alias=foo1", "--net-alias=foo2", "busybox:glibc", "top") c.Assert(waitRun("first"), check.IsNil) // Check if default short-id alias is added automatically @@ -302,7 +302,7 @@ func (s *DockerSuite) TestUserDefinedNetworkAlias(c *check.C) { aliases := inspectField(c, id, "NetworkSettings.Networks.net1.Aliases") c.Assert(aliases, checker.Contains, stringid.TruncateID(id)) - cid2, _ := dockerCmd(c, "run", "-d", "--net=net1", "--name=second", "busybox", "top") + cid2, _ := dockerCmd(c, "run", "-d", "--net=net1", "--name=second", "busybox:glibc", "top") c.Assert(waitRun("second"), check.IsNil) // Check if default short-id alias is added automatically diff --git a/components/engine/integration-cli/docker_cli_swarm_test.go b/components/engine/integration-cli/docker_cli_swarm_test.go index 4a5ec9a566..8add18e1ec 100644 --- a/components/engine/integration-cli/docker_cli_swarm_test.go +++ b/components/engine/integration-cli/docker_cli_swarm_test.go @@ -345,13 +345,13 @@ func (s *DockerSwarmSuite) TestSwarmContainerEndpointOptions(c *check.C) { c.Assert(err, checker.IsNil, check.Commentf(out)) c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") - _, err = d.Cmd("run", "-d", "--net=foo", "--name=first", "--net-alias=first-alias", "busybox", "top") + _, err = d.Cmd("run", "-d", "--net=foo", "--name=first", "--net-alias=first-alias", "busybox:glibc", "top") c.Assert(err, checker.IsNil, check.Commentf(out)) - _, err = d.Cmd("run", "-d", "--net=foo", "--name=second", "busybox", "top") + _, err = d.Cmd("run", "-d", "--net=foo", "--name=second", "busybox:glibc", "top") c.Assert(err, checker.IsNil, check.Commentf(out)) - _, err = d.Cmd("run", "-d", "--net=foo", "--net-alias=third-alias", "busybox", "top") + _, err = d.Cmd("run", "-d", "--net=foo", "--net-alias=third-alias", "busybox:glibc", "top") c.Assert(err, checker.IsNil, check.Commentf(out)) // ping first container and its alias, also ping third and anonymous container by its alias diff --git a/components/engine/integration-cli/docker_experimental_network_test.go b/components/engine/integration-cli/docker_experimental_network_test.go index fb20331631..9d6be69607 100644 --- a/components/engine/integration-cli/docker_experimental_network_test.go +++ b/components/engine/integration-cli/docker_experimental_network_test.go @@ -143,8 +143,8 @@ func (s *DockerNetworkSuite) TestDockerNetworkMacvlanMultiSubnet(c *check.C) { // Ensure the network was created assertNwIsAvailable(c, "dualstackbridge") // start dual stack containers and verify the user specified --ip and --ip6 addresses on subnets 172.28.100.0/24 and 2001:db8:abc2::/64 - dockerCmd(c, "run", "-d", "--net=dualstackbridge", "--name=first", "--ip", "172.28.100.20", "--ip6", "2001:db8:abc2::20", "busybox", "top") - dockerCmd(c, "run", "-d", "--net=dualstackbridge", "--name=second", "--ip", "172.28.100.21", "--ip6", "2001:db8:abc2::21", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=dualstackbridge", "--name=first", "--ip", "172.28.100.20", "--ip6", "2001:db8:abc2::20", "busybox:glibc", "top") + dockerCmd(c, "run", "-d", "--net=dualstackbridge", "--name=second", "--ip", "172.28.100.21", "--ip6", "2001:db8:abc2::21", "busybox:glibc", "top") // Inspect and store the v4 address from specified container on the network dualstackbridge ip := inspectField(c, "first", "NetworkSettings.Networks.dualstackbridge.IPAddress") @@ -160,8 +160,8 @@ func (s *DockerNetworkSuite) TestDockerNetworkMacvlanMultiSubnet(c *check.C) { c.Assert(err, check.IsNil) // start dual stack containers and verify the user specified --ip and --ip6 addresses on subnets 172.28.102.0/24 and 2001:db8:abc4::/64 - dockerCmd(c, "run", "-d", "--net=dualstackbridge", "--name=third", "--ip", "172.28.102.20", "--ip6", "2001:db8:abc4::20", "busybox", "top") - dockerCmd(c, "run", "-d", "--net=dualstackbridge", "--name=fourth", "--ip", "172.28.102.21", "--ip6", "2001:db8:abc4::21", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=dualstackbridge", "--name=third", "--ip", "172.28.102.20", "--ip6", "2001:db8:abc4::20", "busybox:glibc", "top") + dockerCmd(c, "run", "-d", "--net=dualstackbridge", "--name=fourth", "--ip", "172.28.102.21", "--ip6", "2001:db8:abc4::21", "busybox:glibc", "top") // Inspect and store the v4 address from specified container on the network dualstackbridge ip = inspectField(c, "third", "NetworkSettings.Networks.dualstackbridge.IPAddress") @@ -198,8 +198,8 @@ func (s *DockerNetworkSuite) TestDockerNetworkIpvlanL2MultiSubnet(c *check.C) { // Ensure the network was created assertNwIsAvailable(c, "dualstackl2") // start dual stack containers and verify the user specified --ip and --ip6 addresses on subnets 172.28.200.0/24 and 2001:db8:abc8::/64 - dockerCmd(c, "run", "-d", "--net=dualstackl2", "--name=first", "--ip", "172.28.200.20", "--ip6", "2001:db8:abc8::20", "busybox", "top") - dockerCmd(c, "run", "-d", "--net=dualstackl2", "--name=second", "--ip", "172.28.200.21", "--ip6", "2001:db8:abc8::21", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=dualstackl2", "--name=first", "--ip", "172.28.200.20", "--ip6", "2001:db8:abc8::20", "busybox:glibc", "top") + dockerCmd(c, "run", "-d", "--net=dualstackl2", "--name=second", "--ip", "172.28.200.21", "--ip6", "2001:db8:abc8::21", "busybox:glibc", "top") // Inspect and store the v4 address from specified container on the network dualstackl2 ip := inspectField(c, "first", "NetworkSettings.Networks.dualstackl2.IPAddress") @@ -214,8 +214,8 @@ func (s *DockerNetworkSuite) TestDockerNetworkIpvlanL2MultiSubnet(c *check.C) { c.Assert(err, check.IsNil) // start dual stack containers and verify the user specified --ip and --ip6 addresses on subnets 172.28.202.0/24 and 2001:db8:abc6::/64 - dockerCmd(c, "run", "-d", "--net=dualstackl2", "--name=third", "--ip", "172.28.202.20", "--ip6", "2001:db8:abc6::20", "busybox", "top") - dockerCmd(c, "run", "-d", "--net=dualstackl2", "--name=fourth", "--ip", "172.28.202.21", "--ip6", "2001:db8:abc6::21", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=dualstackl2", "--name=third", "--ip", "172.28.202.20", "--ip6", "2001:db8:abc6::20", "busybox:glibc", "top") + dockerCmd(c, "run", "-d", "--net=dualstackl2", "--name=fourth", "--ip", "172.28.202.21", "--ip6", "2001:db8:abc6::21", "busybox:glibc", "top") // Inspect and store the v4 address from specified container on the network dualstackl2 ip = inspectField(c, "third", "NetworkSettings.Networks.dualstackl2.IPAddress") @@ -253,8 +253,8 @@ func (s *DockerNetworkSuite) TestDockerNetworkIpvlanL3MultiSubnet(c *check.C) { assertNwIsAvailable(c, "dualstackl3") // start dual stack containers and verify the user specified --ip and --ip6 addresses on subnets 172.28.10.0/24 and 2001:db8:abc9::/64 - dockerCmd(c, "run", "-d", "--net=dualstackl3", "--name=first", "--ip", "172.28.10.20", "--ip6", "2001:db8:abc9::20", "busybox", "top") - dockerCmd(c, "run", "-d", "--net=dualstackl3", "--name=second", "--ip", "172.28.10.21", "--ip6", "2001:db8:abc9::21", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=dualstackl3", "--name=first", "--ip", "172.28.10.20", "--ip6", "2001:db8:abc9::20", "busybox:glibc", "top") + dockerCmd(c, "run", "-d", "--net=dualstackl3", "--name=second", "--ip", "172.28.10.21", "--ip6", "2001:db8:abc9::21", "busybox:glibc", "top") // Inspect and store the v4 address from specified container on the network dualstackl3 ip := inspectField(c, "first", "NetworkSettings.Networks.dualstackl3.IPAddress") @@ -269,8 +269,8 @@ func (s *DockerNetworkSuite) TestDockerNetworkIpvlanL3MultiSubnet(c *check.C) { c.Assert(err, check.IsNil) // start dual stack containers and verify the user specified --ip and --ip6 addresses on subnets 172.28.12.0/24 and 2001:db8:abc7::/64 - dockerCmd(c, "run", "-d", "--net=dualstackl3", "--name=third", "--ip", "172.28.12.20", "--ip6", "2001:db8:abc7::20", "busybox", "top") - dockerCmd(c, "run", "-d", "--net=dualstackl3", "--name=fourth", "--ip", "172.28.12.21", "--ip6", "2001:db8:abc7::21", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=dualstackl3", "--name=third", "--ip", "172.28.12.20", "--ip6", "2001:db8:abc7::20", "busybox:glibc", "top") + dockerCmd(c, "run", "-d", "--net=dualstackl3", "--name=fourth", "--ip", "172.28.12.21", "--ip6", "2001:db8:abc7::21", "busybox:glibc", "top") // Inspect and store the v4 address from specified container on the network dualstackl3 ip = inspectField(c, "third", "NetworkSettings.Networks.dualstackl3.IPAddress") @@ -356,9 +356,9 @@ func (s *DockerSuite) TestDockerNetworkMacVlanBridgeNilParent(c *check.C) { assertNwIsAvailable(c, "dm-nil-parent") // start two containers on the same subnet - dockerCmd(c, "run", "-d", "--net=dm-nil-parent", "--name=first", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=dm-nil-parent", "--name=first", "busybox:glibc", "top") c.Assert(waitRun("first"), check.IsNil) - dockerCmd(c, "run", "-d", "--net=dm-nil-parent", "--name=second", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=dm-nil-parent", "--name=second", "busybox:glibc", "top") c.Assert(waitRun("second"), check.IsNil) // intra-network communications should succeed @@ -375,9 +375,9 @@ func (s *DockerSuite) TestDockerNetworkMacVlanBridgeInternalMode(c *check.C) { c.Assert(nr.Internal, checker.True) // start two containers on the same subnet - cli.DockerCmd(c, "run", "-d", "--net=dm-internal", "--name=first", "busybox", "top") + cli.DockerCmd(c, "run", "-d", "--net=dm-internal", "--name=first", "busybox:glibc", "top") c.Assert(waitRun("first"), check.IsNil) - cli.DockerCmd(c, "run", "-d", "--net=dm-internal", "--name=second", "busybox", "top") + cli.DockerCmd(c, "run", "-d", "--net=dm-internal", "--name=second", "busybox:glibc", "top") c.Assert(waitRun("second"), check.IsNil) // access outside of the network should fail @@ -395,9 +395,9 @@ func (s *DockerSuite) TestDockerNetworkIpvlanL2NilParent(c *check.C) { assertNwIsAvailable(c, "di-nil-parent") // start two containers on the same subnet - dockerCmd(c, "run", "-d", "--net=di-nil-parent", "--name=first", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=di-nil-parent", "--name=first", "busybox:glibc", "top") c.Assert(waitRun("first"), check.IsNil) - dockerCmd(c, "run", "-d", "--net=di-nil-parent", "--name=second", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=di-nil-parent", "--name=second", "busybox:glibc", "top") c.Assert(waitRun("second"), check.IsNil) // intra-network communications should succeed @@ -414,9 +414,9 @@ func (s *DockerSuite) TestDockerNetworkIpvlanL2InternalMode(c *check.C) { c.Assert(nr.Internal, checker.True) // start two containers on the same subnet - cli.DockerCmd(c, "run", "-d", "--net=di-internal", "--name=first", "busybox", "top") + cli.DockerCmd(c, "run", "-d", "--net=di-internal", "--name=first", "busybox:glibc", "top") c.Assert(waitRun("first"), check.IsNil) - cli.DockerCmd(c, "run", "-d", "--net=di-internal", "--name=second", "busybox", "top") + cli.DockerCmd(c, "run", "-d", "--net=di-internal", "--name=second", "busybox:glibc", "top") c.Assert(waitRun("second"), check.IsNil) // access outside of the network should fail @@ -434,9 +434,9 @@ func (s *DockerSuite) TestDockerNetworkIpvlanL3NilParent(c *check.C) { assertNwIsAvailable(c, "di-nil-parent-l3") // start two containers on separate subnets - dockerCmd(c, "run", "-d", "--ip=172.28.220.10", "--net=di-nil-parent-l3", "--name=first", "busybox", "top") + dockerCmd(c, "run", "-d", "--ip=172.28.220.10", "--net=di-nil-parent-l3", "--name=first", "busybox:glibc", "top") c.Assert(waitRun("first"), check.IsNil) - dockerCmd(c, "run", "-d", "--ip=172.28.230.10", "--net=di-nil-parent-l3", "--name=second", "busybox", "top") + dockerCmd(c, "run", "-d", "--ip=172.28.230.10", "--net=di-nil-parent-l3", "--name=second", "busybox:glibc", "top") c.Assert(waitRun("second"), check.IsNil) // intra-network communications should succeed @@ -454,9 +454,9 @@ func (s *DockerSuite) TestDockerNetworkIpvlanL3InternalMode(c *check.C) { c.Assert(nr.Internal, checker.True) // start two containers on separate subnets - cli.DockerCmd(c, "run", "-d", "--ip=172.28.220.10", "--net=di-internal-l3", "--name=first", "busybox", "top") + cli.DockerCmd(c, "run", "-d", "--ip=172.28.220.10", "--net=di-internal-l3", "--name=first", "busybox:glibc", "top") c.Assert(waitRun("first"), check.IsNil) - cli.DockerCmd(c, "run", "-d", "--ip=172.28.230.10", "--net=di-internal-l3", "--name=second", "busybox", "top") + cli.DockerCmd(c, "run", "-d", "--ip=172.28.230.10", "--net=di-internal-l3", "--name=second", "busybox:glibc", "top") c.Assert(waitRun("second"), check.IsNil) // access outside of the network should fail @@ -496,9 +496,9 @@ func (s *DockerSuite) TestDockerNetworkMacVlanSubinterface(c *check.C) { assertNwIsAvailable(c, netName) // start containers on 802.1q tagged '-o parent' sub-interface - dockerCmd(c, "run", "-d", "--net=dm-subinterface", "--name=first", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=dm-subinterface", "--name=first", "busybox:glibc", "top") c.Assert(waitRun("first"), check.IsNil) - dockerCmd(c, "run", "-d", "--net=dm-subinterface", "--name=second", "busybox", "top") + dockerCmd(c, "run", "-d", "--net=dm-subinterface", "--name=second", "busybox:glibc", "top") c.Assert(waitRun("second"), check.IsNil) // verify containers can communicate _, _, err := dockerCmdWithError("exec", "second", "ping", "-c", "1", "first") diff --git a/components/engine/integration-cli/fixtures/load/frozen.go b/components/engine/integration-cli/fixtures/load/frozen.go index 7e104fe76c..5701a216a2 100644 --- a/components/engine/integration-cli/fixtures/load/frozen.go +++ b/components/engine/integration-cli/fixtures/load/frozen.go @@ -37,9 +37,6 @@ func FrozenImagesLinux(client client.APIClient, images ...string) error { if img == "hello-world:frozen" { srcName = "hello-world:latest" } - if img == "busybox:1.27-glibc" { - img = "busybox:latest" - } loadImages = append(loadImages, struct{ srcName, destName string }{ srcName: srcName, destName: img, diff --git a/components/engine/internal/test/environment/protect.go b/components/engine/internal/test/environment/protect.go index 482dde60b6..ffbf985b73 100644 --- a/components/engine/internal/test/environment/protect.go +++ b/components/engine/internal/test/environment/protect.go @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/require" ) -var frozenImages = []string{"busybox:1.27-glibc", "hello-world:frozen", "debian:jessie"} +var frozenImages = []string{"busybox:latest", "busybox:glibc", "hello-world:frozen", "debian:jessie"} type protectedElements struct { containers map[string]struct{} From 1269ee770f5b7b39a654dd622766296e4e7e8761 Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Tue, 27 Feb 2018 09:16:13 +0100 Subject: [PATCH 13/20] Do not copy golang in itself Signed-off-by: Vincent Demeester Upstream-commit: 30498c7c5f2e63a48fa285d117b4388d2a4961e6 Component: engine --- components/engine/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile index 49a8c59967..99403083c5 100644 --- a/components/engine/Dockerfile +++ b/components/engine/Dockerfile @@ -242,7 +242,6 @@ COPY --from=runc /opt/runc/ /usr/local/bin/ COPY --from=containerd /opt/containerd/ /usr/local/bin/ COPY --from=proxy /opt/proxy/ /usr/local/bin/ COPY --from=dockercli /opt/dockercli /usr/local/cli -COPY --from=golang /usr/local/go /usr/local/go COPY --from=registry /usr/local/bin/registry* /usr/local/bin/ COPY --from=notary /usr/local/bin/notary* /usr/local/bin/ COPY --from=criu /opt/criu/ /usr/local/ From 1988af7bf7f039562129d482a81bf6672cc4da3a Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Tue, 27 Feb 2018 13:42:29 +0100 Subject: [PATCH 14/20] Clean some docker_cli_build_tests that are cli-only Remove TestBuildRenamedDockerfile and TestBuildDockerfileOutsideContext that are cli-only tests (and already tested in the docker/cli repository). Also adds some comments on few tests that could be migrate to docker/cli. Signed-off-by: Vincent Demeester Upstream-commit: 894c213b3bd6f4d8f344837b5b5084360a013680 Component: engine --- .../integration-cli/docker_cli_build_test.go | 134 +++--------------- 1 file changed, 18 insertions(+), 116 deletions(-) diff --git a/components/engine/integration-cli/docker_cli_build_test.go b/components/engine/integration-cli/docker_cli_build_test.go index 5dd1d8b355..fc2dafe411 100644 --- a/components/engine/integration-cli/docker_cli_build_test.go +++ b/components/engine/integration-cli/docker_cli_build_test.go @@ -1439,6 +1439,7 @@ func (s *DockerSuite) TestBuildRelativeCopy(c *check.C) { )) } +// FIXME(vdemeester) should be unit test func (s *DockerSuite) TestBuildBlankName(c *check.C) { name := "testbuildblankname" testCases := []struct { @@ -2066,6 +2067,7 @@ func (s *DockerSuite) TestBuildNoContext(c *check.C) { } } +// FIXME(vdemeester) migrate to docker/cli e2e func (s *DockerSuite) TestBuildDockerfileStdin(c *check.C) { name := "stdindockerfile" tmpDir, err := ioutil.TempDir("", "fake-context") @@ -2085,6 +2087,7 @@ CMD ["cat", "/foo"]`), c.Assert(strings.TrimSpace(string(res)), checker.Equals, `[cat /foo]`) } +// FIXME(vdemeester) migrate to docker/cli tests (unit or e2e) func (s *DockerSuite) TestBuildDockerfileStdinConflict(c *check.C) { name := "stdindockerfiletarcontext" icmd.RunCmd(icmd.Cmd{ @@ -2401,6 +2404,7 @@ func (s *DockerSuite) TestBuildDockerignoringDockerfile(c *check.C) { build.WithFile("Dockerfile", dockerfile), build.WithFile(".dockerignore", "Dockerfile\n"), )) + // FIXME(vdemeester) why twice ? buildImageSuccessfully(c, name, build.WithBuildContext(c, build.WithFile("Dockerfile", dockerfile), build.WithFile(".dockerignore", "./Dockerfile\n"), @@ -2420,6 +2424,7 @@ func (s *DockerSuite) TestBuildDockerignoringRenamedDockerfile(c *check.C) { build.WithFile("MyDockerfile", dockerfile), build.WithFile(".dockerignore", "MyDockerfile\n"), )) + // FIXME(vdemeester) why twice ? buildImageSuccessfully(c, name, cli.WithFlags("-f", "MyDockerfile"), build.WithBuildContext(c, build.WithFile("Dockerfile", "Should not use me"), build.WithFile("MyDockerfile", dockerfile), @@ -3045,6 +3050,7 @@ func (s *DockerSuite) TestBuildAddTarXzGz(c *check.C) { buildImageSuccessfully(c, name, build.WithExternalBuildContext(ctx)) } +// FIXME(vdemeester) most of the from git tests could be moved to `docker/cli` e2e tests func (s *DockerSuite) TestBuildFromGit(c *check.C) { name := "testbuildfromgit" git := fakegit.New(c, "repo", map[string]string{ @@ -3422,6 +3428,7 @@ func (s *DockerSuite) TestBuildLabelsCache(c *check.C) { } +// FIXME(vdemeester) port to docker/cli e2e tests (api tests should test suppressOutput option though) func (s *DockerSuite) TestBuildNotVerboseSuccess(c *check.C) { // This test makes sure that -q works correctly when build is successful: // stdout has only the image ID (long image ID) and stderr is empty. @@ -3472,6 +3479,7 @@ func (s *DockerSuite) TestBuildNotVerboseSuccess(c *check.C) { } +// FIXME(vdemeester) migrate to docker/cli tests func (s *DockerSuite) TestBuildNotVerboseFailureWithNonExistImage(c *check.C) { // This test makes sure that -q works correctly when build fails by // comparing between the stderr output in quiet mode and in stdout @@ -3492,6 +3500,7 @@ func (s *DockerSuite) TestBuildNotVerboseFailureWithNonExistImage(c *check.C) { } } +// FIXME(vdemeester) migrate to docker/cli tests func (s *DockerSuite) TestBuildNotVerboseFailure(c *check.C) { // This test makes sure that -q works correctly when build fails by // comparing between the stderr output in quiet mode and in stdout @@ -3519,6 +3528,7 @@ func (s *DockerSuite) TestBuildNotVerboseFailure(c *check.C) { } } +// FIXME(vdemeester) migrate to docker/cli tests func (s *DockerSuite) TestBuildNotVerboseFailureRemote(c *check.C) { // This test ensures that when given a wrong URL, stderr in quiet mode and // stderr in verbose mode are identical. @@ -3548,6 +3558,7 @@ func (s *DockerSuite) TestBuildNotVerboseFailureRemote(c *check.C) { } } +// FIXME(vdemeester) migrate to docker/cli tests func (s *DockerSuite) TestBuildStderr(c *check.C) { // This test just makes sure that no non-error output goes // to stderr @@ -3688,67 +3699,6 @@ CMD cat /foo/file`), } -// FIXME(vdemeester) part of this should be unit test, other part should be clearer -func (s *DockerSuite) TestBuildRenamedDockerfile(c *check.C) { - ctx := fakecontext.New(c, "", fakecontext.WithFiles(map[string]string{ - "Dockerfile": "FROM busybox\nRUN echo from Dockerfile", - "files/Dockerfile": "FROM busybox\nRUN echo from files/Dockerfile", - "files/dFile": "FROM busybox\nRUN echo from files/dFile", - "dFile": "FROM busybox\nRUN echo from dFile", - "files/dFile2": "FROM busybox\nRUN echo from files/dFile2", - })) - defer ctx.Close() - - cli.Docker(cli.Args("build", "-t", "test1", "."), cli.InDir(ctx.Dir)).Assert(c, icmd.Expected{ - Out: "from Dockerfile", - }) - - cli.Docker(cli.Args("build", "-f", filepath.Join("files", "Dockerfile"), "-t", "test2", "."), cli.InDir(ctx.Dir)).Assert(c, icmd.Expected{ - Out: "from files/Dockerfile", - }) - - cli.Docker(cli.Args("build", fmt.Sprintf("--file=%s", filepath.Join("files", "dFile")), "-t", "test3", "."), cli.InDir(ctx.Dir)).Assert(c, icmd.Expected{ - Out: "from files/dFile", - }) - - cli.Docker(cli.Args("build", "--file=dFile", "-t", "test4", "."), cli.InDir(ctx.Dir)).Assert(c, icmd.Expected{ - Out: "from dFile", - }) - - dirWithNoDockerfile, err := ioutil.TempDir(os.TempDir(), "test5") - c.Assert(err, check.IsNil) - nonDockerfileFile := filepath.Join(dirWithNoDockerfile, "notDockerfile") - if _, err = os.Create(nonDockerfileFile); err != nil { - c.Fatal(err) - } - cli.Docker(cli.Args("build", fmt.Sprintf("--file=%s", nonDockerfileFile), "-t", "test5", "."), cli.InDir(ctx.Dir)).Assert(c, icmd.Expected{ - ExitCode: 1, - Err: fmt.Sprintf("unable to prepare context: the Dockerfile (%s) must be within the build context", nonDockerfileFile), - }) - - cli.Docker(cli.Args("build", "-f", filepath.Join("..", "Dockerfile"), "-t", "test6", ".."), cli.InDir(filepath.Join(ctx.Dir, "files"))).Assert(c, icmd.Expected{ - Out: "from Dockerfile", - }) - - cli.Docker(cli.Args("build", "-f", filepath.Join(ctx.Dir, "files", "Dockerfile"), "-t", "test7", ".."), cli.InDir(filepath.Join(ctx.Dir, "files"))).Assert(c, icmd.Expected{ - Out: "from files/Dockerfile", - }) - - cli.Docker(cli.Args("build", "-f", filepath.Join("..", "Dockerfile"), "-t", "test8", "."), cli.InDir(filepath.Join(ctx.Dir, "files"))).Assert(c, icmd.Expected{ - ExitCode: 1, - Err: "must be within the build context", - }) - - tmpDir := os.TempDir() - cli.Docker(cli.Args("build", "-t", "test9", ctx.Dir), cli.InDir(tmpDir)).Assert(c, icmd.Expected{ - Out: "from Dockerfile", - }) - - cli.Docker(cli.Args("build", "-f", "dFile2", "-t", "test10", "."), cli.InDir(filepath.Join(ctx.Dir, "files"))).Assert(c, icmd.Expected{ - Out: "from files/dFile2", - }) -} - func (s *DockerSuite) TestBuildFromMixedcaseDockerfile(c *check.C) { testRequires(c, UnixCli) // Dockerfile overwrites dockerfile on windows testRequires(c, DaemonIsLinux) @@ -3772,6 +3722,7 @@ func (s *DockerSuite) TestBuildFromMixedcaseDockerfile(c *check.C) { }) } +// FIXME(vdemeester) should migrate to docker/cli tests func (s *DockerSuite) TestBuildFromURLWithF(c *check.C) { server := fakestorage.New(c, "", fakecontext.WithFiles(map[string]string{"baz": `FROM busybox RUN echo from baz @@ -3798,6 +3749,7 @@ RUN find /tmp/`})) } +// FIXME(vdemeester) should migrate to docker/cli tests func (s *DockerSuite) TestBuildFromStdinWithF(c *check.C) { testRequires(c, DaemonIsLinux) // TODO Windows: This test is flaky; no idea why ctx := fakecontext.New(c, "", fakecontext.WithDockerfile(`FROM busybox @@ -3840,61 +3792,6 @@ func (s *DockerSuite) TestBuildFromOfficialNames(c *check.C) { } } -func (s *DockerSuite) TestBuildDockerfileOutsideContext(c *check.C) { - testRequires(c, UnixCli, DaemonIsLinux) // uses os.Symlink: not implemented in windows at the time of writing (go-1.4.2) - - name := "testbuilddockerfileoutsidecontext" - tmpdir, err := ioutil.TempDir("", name) - c.Assert(err, check.IsNil) - defer os.RemoveAll(tmpdir) - ctx := filepath.Join(tmpdir, "context") - if err := os.MkdirAll(ctx, 0755); err != nil { - c.Fatal(err) - } - if err := ioutil.WriteFile(filepath.Join(ctx, "Dockerfile"), []byte("FROM scratch\nENV X Y"), 0644); err != nil { - c.Fatal(err) - } - wd, err := os.Getwd() - if err != nil { - c.Fatal(err) - } - defer os.Chdir(wd) - if err := os.Chdir(ctx); err != nil { - c.Fatal(err) - } - if err := ioutil.WriteFile(filepath.Join(tmpdir, "outsideDockerfile"), []byte("FROM scratch\nENV x y"), 0644); err != nil { - c.Fatal(err) - } - if err := os.Symlink(filepath.Join("..", "outsideDockerfile"), filepath.Join(ctx, "dockerfile1")); err != nil { - c.Fatal(err) - } - if err := os.Symlink(filepath.Join(tmpdir, "outsideDockerfile"), filepath.Join(ctx, "dockerfile2")); err != nil { - c.Fatal(err) - } - - for _, dockerfilePath := range []string{ - filepath.Join("..", "outsideDockerfile"), - filepath.Join(ctx, "dockerfile1"), - filepath.Join(ctx, "dockerfile2"), - } { - result := dockerCmdWithResult("build", "-t", name, "--no-cache", "-f", dockerfilePath, ".") - result.Assert(c, icmd.Expected{ - Err: "must be within the build context", - ExitCode: 1, - }) - deleteImages(name) - } - - os.Chdir(tmpdir) - - // Path to Dockerfile should be resolved relative to working directory, not relative to context. - // There is a Dockerfile in the context, but since there is no Dockerfile in the current directory, the following should fail - out, _, err := dockerCmdWithError("build", "-t", name, "--no-cache", "-f", "Dockerfile", ctx) - if err == nil { - c.Fatalf("Expected error. Out: %s", out) - } -} - // FIXME(vdemeester) should be a unit test func (s *DockerSuite) TestBuildSpaces(c *check.C) { // Test to make sure that leading/trailing spaces on a command @@ -4186,6 +4083,7 @@ func (s *DockerTrustSuite) TestTrustedBuildUntrustedTag(c *check.C) { }) } +// FIXME(vdemeester) should migrate to docker/cli e2e tests func (s *DockerTrustSuite) TestBuildContextDirIsSymlink(c *check.C) { testRequires(c, DaemonIsLinux) tempDir, err := ioutil.TempDir("", "test-build-dir-is-symlink-") @@ -5130,6 +5028,7 @@ func (s *DockerSuite) TestBuildCacheRootSource(c *check.C) { } // #19375 +// FIXME(vdemeester) should migrate to docker/cli tests func (s *DockerSuite) TestBuildFailsGitNotCallable(c *check.C) { buildImage("gitnotcallable", cli.WithEnvironmentVariables("PATH="), build.WithContextPath("github.com/docker/v1.10-migrator.git")).Assert(c, icmd.Expected{ @@ -6447,6 +6346,7 @@ CMD echo foo c.Assert(strings.TrimSpace(out), checker.Equals, `["/bin/sh","-c","echo foo"]`) } +// FIXME(vdemeester) should migrate to docker/cli tests func (s *DockerSuite) TestBuildIidFile(c *check.C) { tmpDir, err := ioutil.TempDir("", "TestBuildIidFile") if err != nil { @@ -6471,6 +6371,7 @@ ENV BAR BAZ`), c.Assert(d.String(), checker.Equals, getIDByName(c, name)) } +// FIXME(vdemeester) should migrate to docker/cli tests func (s *DockerSuite) TestBuildIidFileCleanupOnFail(c *check.C) { tmpDir, err := ioutil.TempDir("", "TestBuildIidFileCleanupOnFail") if err != nil { @@ -6493,6 +6394,7 @@ func (s *DockerSuite) TestBuildIidFileCleanupOnFail(c *check.C) { c.Assert(os.IsNotExist(err), check.Equals, true) } +// FIXME(vdemeester) should migrate to docker/cli tests func (s *DockerSuite) TestBuildIidFileSquash(c *check.C) { testRequires(c, ExperimentalDaemon) tmpDir, err := ioutil.TempDir("", "TestBuildIidFileSquash") From a0d675b3881ec6b4c3afe576bf3f347392a2f251 Mon Sep 17 00:00:00 2001 From: John Howard Date: Fri, 9 Feb 2018 10:08:47 -0800 Subject: [PATCH 15/20] Windows: Remove servicing mode Signed-off-by: John Howard Upstream-commit: d4f37c08858b90e8603741ba92de8e0d39b88eb3 Component: engine --- components/engine/daemon/monitor.go | 5 +- components/engine/daemon/monitor_linux.go | 11 --- components/engine/daemon/monitor_windows.go | 53 ---------- components/engine/daemon/oci_windows.go | 3 - .../integration-cli/docker_cli_run_test.go | 29 ------ .../libcontainerd/client_local_windows.go | 97 ++++++------------- components/engine/libcontainerd/types.go | 4 +- 7 files changed, 32 insertions(+), 170 deletions(-) delete mode 100644 components/engine/daemon/monitor_linux.go delete mode 100644 components/engine/daemon/monitor_windows.go diff --git a/components/engine/daemon/monitor.go b/components/engine/daemon/monitor.go index ce897d9397..5e740dd4fe 100644 --- a/components/engine/daemon/monitor.go +++ b/components/engine/daemon/monitor.go @@ -111,10 +111,7 @@ func (daemon *Daemon) ProcessEvent(id string, e libcontainerd.EventType, ei libc } daemon.setStateCounter(c) - if err := c.CheckpointTo(daemon.containersReplica); err != nil { - return err - } - return daemon.postRunProcessing(c, ei) + return c.CheckpointTo(daemon.containersReplica) } if execConfig := c.ExecCommands.Get(ei.ProcessID); execConfig != nil { diff --git a/components/engine/daemon/monitor_linux.go b/components/engine/daemon/monitor_linux.go deleted file mode 100644 index cb2aa72dbb..0000000000 --- a/components/engine/daemon/monitor_linux.go +++ /dev/null @@ -1,11 +0,0 @@ -package daemon // import "github.com/docker/docker/daemon" - -import ( - "github.com/docker/docker/container" - "github.com/docker/docker/libcontainerd" -) - -// postRunProcessing perfoms any processing needed on the container after it has stopped. -func (daemon *Daemon) postRunProcessing(_ *container.Container, _ libcontainerd.EventInfo) error { - return nil -} diff --git a/components/engine/daemon/monitor_windows.go b/components/engine/daemon/monitor_windows.go deleted file mode 100644 index 5db81347c6..0000000000 --- a/components/engine/daemon/monitor_windows.go +++ /dev/null @@ -1,53 +0,0 @@ -package daemon // import "github.com/docker/docker/daemon" - -import ( - "context" - - "github.com/docker/docker/container" - "github.com/docker/docker/libcontainerd" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" -) - -// postRunProcessing starts a servicing container if required -func (daemon *Daemon) postRunProcessing(c *container.Container, ei libcontainerd.EventInfo) error { - if ei.ExitCode == 0 && ei.UpdatePending { - spec, err := daemon.createSpec(c) - if err != nil { - return err - } - // Turn on servicing - spec.Windows.Servicing = true - - copts, err := daemon.getLibcontainerdCreateOptions(c) - if err != nil { - return err - } - - // Create a new servicing container, which will start, complete the - // update, and merge back the results if it succeeded, all as part of - // the below function call. - ctx := context.Background() - svcID := c.ID + "_servicing" - logger := logrus.WithField("container", svcID) - if err := daemon.containerd.Create(ctx, svcID, spec, copts); err != nil { - c.SetExitCode(-1) - return errors.Wrap(err, "post-run update servicing failed") - } - _, err = daemon.containerd.Start(ctx, svcID, "", false, nil) - if err != nil { - logger.WithError(err).Warn("failed to run servicing container") - if err := daemon.containerd.Delete(ctx, svcID); err != nil { - logger.WithError(err).Warn("failed to delete servicing container") - } - } else { - if _, _, err := daemon.containerd.DeleteTask(ctx, svcID); err != nil { - logger.WithError(err).Warn("failed to delete servicing container task") - } - if err := daemon.containerd.Delete(ctx, svcID); err != nil { - logger.WithError(err).Warn("failed to delete servicing container") - } - } - } - return nil -} diff --git a/components/engine/daemon/oci_windows.go b/components/engine/daemon/oci_windows.go index e4b536f4e4..c6bda280da 100644 --- a/components/engine/daemon/oci_windows.go +++ b/components/engine/daemon/oci_windows.go @@ -326,9 +326,6 @@ func (daemon *Daemon) createSpecWindowsFields(c *container.Container, s *specs.S s.Windows.CredentialSpec = cs } - // Assume we are not starting a container for a servicing operation - s.Windows.Servicing = false - return nil } diff --git a/components/engine/integration-cli/docker_cli_run_test.go b/components/engine/integration-cli/docker_cli_run_test.go index 776f0e5ba5..134bcb80a9 100644 --- a/components/engine/integration-cli/docker_cli_run_test.go +++ b/components/engine/integration-cli/docker_cli_run_test.go @@ -4237,35 +4237,6 @@ func (s *DockerSuite) TestRunCredentialSpecWellFormed(c *check.C) { dockerCmd(c, "run", `--security-opt=credentialspec=file://valid.json`, "busybox", "true") } -// Windows specific test to ensure that a servicing app container is started -// if necessary once a container exits. It does this by forcing a no-op -// servicing event and verifying the event from Hyper-V-Compute -func (s *DockerSuite) TestRunServicingContainer(c *check.C) { - testRequires(c, DaemonIsWindows, SameHostDaemon) - - // This functionality does not exist in post-RS3 builds. - // Note we get the version number from the full build string, as Windows - // reports Windows 8 version 6.2 build 9200 from non-manifested binaries. - // Ref: https://msdn.microsoft.com/en-us/library/windows/desktop/ms724451(v=vs.85).aspx - v, err := kernel.GetKernelVersion() - c.Assert(err, checker.IsNil) - build, _ := strconv.Atoi(strings.Split(strings.SplitN(v.String(), " ", 3)[2][1:], ".")[0]) - if build > 16299 { - c.Skip("Disabled on post-RS3 builds") - } - - out := cli.DockerCmd(c, "run", "-d", testEnv.PlatformDefaults.BaseImage, "cmd", "/c", "mkdir c:\\programdata\\Microsoft\\Windows\\ContainerUpdates\\000_000_d99f45d0-ffc8-4af7-bd9c-ea6a62e035c9_200 && sc control cexecsvc 255").Combined() - containerID := strings.TrimSpace(out) - cli.WaitExited(c, containerID, 60*time.Second) - - result := icmd.RunCommand("powershell", "echo", `(Get-WinEvent -ProviderName "Microsoft-Windows-Hyper-V-Compute" -FilterXPath 'Event[System[EventID=2010]]' -MaxEvents 1).Message`) - result.Assert(c, icmd.Success) - out2 := result.Combined() - c.Assert(out2, checker.Contains, `"Servicing":true`, check.Commentf("Servicing container does not appear to have been started: %s", out2)) - c.Assert(out2, checker.Contains, `Windows Container (Servicing)`, check.Commentf("Didn't find 'Windows Container (Servicing): %s", out2)) - c.Assert(out2, checker.Contains, containerID+"_servicing", check.Commentf("Didn't find '%s_servicing': %s", containerID+"_servicing", out2)) -} - func (s *DockerSuite) TestRunDuplicateMount(c *check.C) { testRequires(c, SameHostDaemon, DaemonIsLinux, NotUserNamespace) diff --git a/components/engine/libcontainerd/client_local_windows.go b/components/engine/libcontainerd/client_local_windows.go index bbe127a6d7..6e3454e514 100644 --- a/components/engine/libcontainerd/client_local_windows.go +++ b/components/engine/libcontainerd/client_local_windows.go @@ -82,7 +82,7 @@ func (c *client) Version(ctx context.Context) (containerd.Version, error) { // | | Isolation=Process | Isolation=Hyper-V | // +-----------------+--------------------------------------------+---------------------------------------------------+ // | VolumePath | \\?\\Volume{GUIDa} | | -// | LayerFolderPath | %root%\windowsfilter\containerID | %root%\windowsfilter\containerID (servicing only) | +// | LayerFolderPath | %root%\windowsfilter\containerID | | // | Layers[] | ID=GUIDb;Path=%root%\windowsfilter\layerID | ID=GUIDb;Path=%root%\windowsfilter\layerID | // | HvRuntime | | ImagePath=%root%\BaseLayerID\UtilityVM | // +-----------------+--------------------------------------------+---------------------------------------------------+ @@ -104,7 +104,6 @@ func (c *client) Version(ctx context.Context) (containerd.Version, error) { // "MappedDirectories": [], // "HvPartition": false, // "EndpointList": ["eef2649d-bb17-4d53-9937-295a8efe6f2c"], -// "Servicing": false //} // // Isolation=Hyper-V example: @@ -126,7 +125,6 @@ func (c *client) Version(ctx context.Context) (containerd.Version, error) { // "HvRuntime": { // "ImagePath": "C:\\\\control\\\\windowsfilter\\\\65bf96e5760a09edf1790cb229e2dfb2dbd0fcdc0bf7451bae099106bfbfea0c\\\\UtilityVM" // }, -// "Servicing": false //} func (c *client) Create(_ context.Context, id string, spec *specs.Spec, runtimeOptions interface{}) error { if ctr := c.getContainer(id); ctr != nil { @@ -155,7 +153,6 @@ func (c *client) createWindows(id string, spec *specs.Spec, runtimeOptions inter IgnoreFlushesDuringBoot: spec.Windows.IgnoreFlushesDuringBoot, HostName: spec.Hostname, HvPartition: false, - Servicing: spec.Windows.Servicing, } if spec.Windows.Resources != nil { @@ -324,9 +321,6 @@ func (c *client) createWindows(id string, spec *specs.Spec, runtimeOptions inter waitCh: make(chan struct{}), } - // Start the container. If this is a servicing container, this call - // will block until the container is done with the servicing - // execution. logger.Debug("starting container") if err = hcsContainer.Start(); err != nil { c.logger.WithError(err).Error("failed to start container") @@ -525,9 +519,7 @@ func (c *client) createLinux(id string, spec *specs.Spec, runtimeOptions interfa waitCh: make(chan struct{}), } - // Start the container. If this is a servicing container, this call - // will block until the container is done with the servicing - // execution. + // Start the container. logger.Debug("starting container") if err = hcsContainer.Start(); err != nil { c.logger.WithError(err).Error("failed to start container") @@ -588,14 +580,14 @@ func (c *client) Start(_ context.Context, id, _ string, withStdin bool, attachSt ) if ctr.ociSpec.Process != nil { emulateConsole = ctr.ociSpec.Process.Terminal - createStdErrPipe = !ctr.ociSpec.Process.Terminal && !ctr.ociSpec.Windows.Servicing + createStdErrPipe = !ctr.ociSpec.Process.Terminal } createProcessParms := &hcsshim.ProcessConfig{ EmulateConsole: emulateConsole, WorkingDirectory: ctr.ociSpec.Process.Cwd, - CreateStdInPipe: !ctr.ociSpec.Windows.Servicing, - CreateStdOutPipe: !ctr.ociSpec.Windows.Servicing, + CreateStdInPipe: true, + CreateStdOutPipe: true, CreateStdErrPipe: createStdErrPipe, } @@ -655,21 +647,6 @@ func (c *client) Start(_ context.Context, id, _ string, withStdin bool, attachSt } logger.WithField("pid", p.pid).Debug("init process started") - // If this is a servicing container, wait on the process synchronously here and - // if it succeeds, wait for it cleanly shutdown and merge into the parent container. - if ctr.ociSpec.Windows.Servicing { - // reapProcess takes the lock - ctr.Unlock() - defer ctr.Lock() - exitCode := c.reapProcess(ctr, p) - - if exitCode != 0 { - return -1, errors.Errorf("libcontainerd: servicing container %s returned non-zero exit code %d", ctr.id, exitCode) - } - - return p.pid, nil - } - dio, err := newIOFromProcess(newProcess, ctr.ociSpec.Process.Terminal) if err != nil { logger.WithError(err).Error("failed to get stdio pipes") @@ -1275,7 +1252,6 @@ func (c *client) reapProcess(ctr *container, p *process) int { eventErr = fmt.Errorf("hcsProcess.Close() failed %s", err) } - var pendingUpdates bool if p.id == InitProcessName { // Update container status ctr.Lock() @@ -1285,16 +1261,6 @@ func (c *client) reapProcess(ctr *container, p *process) int { close(ctr.waitCh) ctr.Unlock() - // Handle any servicing - if exitCode == 0 && ctr.isWindows && !ctr.ociSpec.Windows.Servicing { - pendingUpdates, err = ctr.hcsContainer.HasPendingUpdates() - logger.Infof("Pending updates: %v", pendingUpdates) - if err != nil { - logger.WithError(err). - Warnf("failed to check for pending updates (container may have been killed)") - } - } - if err := c.shutdownContainer(ctr); err != nil { exitCode = -1 logger.WithError(err).Warn("failed to shutdown container") @@ -1320,37 +1286,34 @@ func (c *client) reapProcess(ctr *container, p *process) int { } } - if !(ctr.isWindows && ctr.ociSpec.Windows.Servicing) { - c.eventQ.append(ctr.id, func() { - ei := EventInfo{ - ContainerID: ctr.id, - ProcessID: p.id, - Pid: uint32(p.pid), - ExitCode: uint32(exitCode), - ExitedAt: exitedAt, - UpdatePending: pendingUpdates, - Error: eventErr, - } - c.logger.WithFields(logrus.Fields{ + c.eventQ.append(ctr.id, func() { + ei := EventInfo{ + ContainerID: ctr.id, + ProcessID: p.id, + Pid: uint32(p.pid), + ExitCode: uint32(exitCode), + ExitedAt: exitedAt, + Error: eventErr, + } + c.logger.WithFields(logrus.Fields{ + "container": ctr.id, + "event": EventExit, + "event-info": ei, + }).Info("sending event") + err := c.backend.ProcessEvent(ctr.id, EventExit, ei) + if err != nil { + c.logger.WithError(err).WithFields(logrus.Fields{ "container": ctr.id, "event": EventExit, "event-info": ei, - }).Info("sending event") - err := c.backend.ProcessEvent(ctr.id, EventExit, ei) - if err != nil { - c.logger.WithError(err).WithFields(logrus.Fields{ - "container": ctr.id, - "event": EventExit, - "event-info": ei, - }).Error("failed to process event") - } - if p.id != InitProcessName { - ctr.Lock() - delete(ctr.execs, p.id) - ctr.Unlock() - } - }) - } + }).Error("failed to process event") + } + if p.id != InitProcessName { + ctr.Lock() + delete(ctr.execs, p.id) + ctr.Unlock() + } + }) return exitCode } diff --git a/components/engine/libcontainerd/types.go b/components/engine/libcontainerd/types.go index 97e4fee735..96ffbe2676 100644 --- a/components/engine/libcontainerd/types.go +++ b/components/engine/libcontainerd/types.go @@ -71,9 +71,7 @@ type EventInfo struct { ExitCode uint32 ExitedAt time.Time OOMKilled bool - // Windows Only field - UpdatePending bool - Error error + Error error } // Backend defines callbacks that the client of the library needs to implement. From bedfb491b42666149a4933ad2f250422261f09e3 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 12 Feb 2018 01:18:35 +0000 Subject: [PATCH 16/20] Migrate several docker rm tests to api tests This fix migrates several docker rm tests to api tests Signed-off-by: Yong Tang Upstream-commit: 6bd4f4801b244555213f0040b9885033e99d4ae8 Component: engine --- .../integration-cli/docker_cli_rm_test.go | 55 --------- .../integration/container/remove_test.go | 113 ++++++++++++++++++ 2 files changed, 113 insertions(+), 55 deletions(-) create mode 100644 components/engine/integration/container/remove_test.go diff --git a/components/engine/integration-cli/docker_cli_rm_test.go b/components/engine/integration-cli/docker_cli_rm_test.go index d281704a7b..5942b8286d 100644 --- a/components/engine/integration-cli/docker_cli_rm_test.go +++ b/components/engine/integration-cli/docker_cli_rm_test.go @@ -1,56 +1,11 @@ package main import ( - "io/ioutil" - "os" - "github.com/docker/docker/integration-cli/checker" "github.com/docker/docker/integration-cli/cli/build" "github.com/go-check/check" ) -func (s *DockerSuite) TestRmContainerWithRemovedVolume(c *check.C) { - testRequires(c, SameHostDaemon) - - prefix, slash := getPrefixAndSlashFromDaemonPlatform() - - tempDir, err := ioutil.TempDir("", "test-rm-container-with-removed-volume-") - if err != nil { - c.Fatalf("failed to create temporary directory: %s", tempDir) - } - defer os.RemoveAll(tempDir) - - dockerCmd(c, "run", "--name", "losemyvolumes", "-v", tempDir+":"+prefix+slash+"test", "busybox", "true") - - err = os.RemoveAll(tempDir) - c.Assert(err, check.IsNil) - - dockerCmd(c, "rm", "-v", "losemyvolumes") -} - -func (s *DockerSuite) TestRmContainerWithVolume(c *check.C) { - prefix, slash := getPrefixAndSlashFromDaemonPlatform() - - dockerCmd(c, "run", "--name", "foo", "-v", prefix+slash+"srv", "busybox", "true") - - dockerCmd(c, "rm", "-v", "foo") -} - -func (s *DockerSuite) TestRmContainerRunning(c *check.C) { - createRunningContainer(c, "foo") - - res, _, err := dockerCmdWithError("rm", "foo") - c.Assert(err, checker.NotNil, check.Commentf("Expected error, can't rm a running container")) - c.Assert(res, checker.Contains, "cannot remove a running container") -} - -func (s *DockerSuite) TestRmContainerForceRemoveRunning(c *check.C) { - createRunningContainer(c, "foo") - - // Stop then remove with -f - dockerCmd(c, "rm", "-f", "foo") -} - func (s *DockerSuite) TestRmContainerOrphaning(c *check.C) { dockerfile1 := `FROM busybox:latest ENTRYPOINT ["true"]` @@ -75,13 +30,3 @@ func (s *DockerSuite) TestRmContainerOrphaning(c *check.C) { c.Assert(out, checker.Contains, img1, check.Commentf("Orphaned container (could not find %q in docker images): %s", img1, out)) } - -func (s *DockerSuite) TestRmInvalidContainer(c *check.C) { - out, _, err := dockerCmdWithError("rm", "unknown") - c.Assert(err, checker.NotNil, check.Commentf("Expected error on rm unknown container, got none")) - c.Assert(out, checker.Contains, "No such container") -} - -func createRunningContainer(c *check.C, name string) { - runSleepingContainer(c, "-dt", "--name", name) -} diff --git a/components/engine/integration/container/remove_test.go b/components/engine/integration/container/remove_test.go new file mode 100644 index 0000000000..bf55dd22c7 --- /dev/null +++ b/components/engine/integration/container/remove_test.go @@ -0,0 +1,113 @@ +package container // import "github.com/docker/docker/integration/container" + +import ( + "context" + "os" + "testing" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/integration/internal/container" + "github.com/docker/docker/integration/internal/request" + "github.com/docker/docker/internal/testutil" + "github.com/gotestyourself/gotestyourself/fs" + "github.com/gotestyourself/gotestyourself/poll" + "github.com/gotestyourself/gotestyourself/skip" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func getPrefixAndSlashFromDaemonPlatform() (prefix, slash string) { + if testEnv.OSType == "windows" { + return "c:", `\` + } + return "", "/" +} + +// Test case for #5244: `docker rm` fails if bind dir doesn't exist anymore +func TestRemoveContainerWithRemovedVolume(t *testing.T) { + skip.If(t, !testEnv.IsLocalDaemon()) + + defer setupTest(t)() + ctx := context.Background() + client := request.NewAPIClient(t) + + prefix, slash := getPrefixAndSlashFromDaemonPlatform() + + tempDir := fs.NewDir(t, "test-rm-container-with-removed-volume", fs.WithMode(0755)) + defer tempDir.Remove() + + cID := container.Run(t, ctx, client, container.WithCmd("true"), container.WithBind(tempDir.Path(), prefix+slash+"test")) + poll.WaitOn(t, container.IsInState(ctx, client, cID, "exited"), poll.WithDelay(100*time.Millisecond)) + + err := os.RemoveAll(tempDir.Path()) + require.NoError(t, err) + + err = client.ContainerRemove(ctx, cID, types.ContainerRemoveOptions{ + RemoveVolumes: true, + }) + require.NoError(t, err) + + _, _, err = client.ContainerInspectWithRaw(ctx, cID, true) + testutil.ErrorContains(t, err, "No such container") +} + +// Test case for #2099/#2125 +func TestRemoveContainerWithVolume(t *testing.T) { + defer setupTest(t)() + ctx := context.Background() + client := request.NewAPIClient(t) + + prefix, slash := getPrefixAndSlashFromDaemonPlatform() + + cID := container.Run(t, ctx, client, container.WithCmd("true"), container.WithVolume(prefix+slash+"srv")) + poll.WaitOn(t, container.IsInState(ctx, client, cID, "exited"), poll.WithDelay(100*time.Millisecond)) + + insp, _, err := client.ContainerInspectWithRaw(ctx, cID, true) + require.NoError(t, err) + assert.Equal(t, len(insp.Mounts), 1) + volName := insp.Mounts[0].Name + + err = client.ContainerRemove(ctx, cID, types.ContainerRemoveOptions{ + RemoveVolumes: true, + }) + require.NoError(t, err) + + volumes, err := client.VolumeList(ctx, filters.NewArgs(filters.Arg("name", volName))) + require.NoError(t, err) + assert.Equal(t, len(volumes.Volumes), 0) +} + +func TestRemoveContainerRunning(t *testing.T) { + defer setupTest(t)() + ctx := context.Background() + client := request.NewAPIClient(t) + + cID := container.Run(t, ctx, client) + + err := client.ContainerRemove(ctx, cID, types.ContainerRemoveOptions{}) + testutil.ErrorContains(t, err, "cannot remove a running container") +} + +func TestRemoveContainerForceRemoveRunning(t *testing.T) { + defer setupTest(t)() + ctx := context.Background() + client := request.NewAPIClient(t) + + cID := container.Run(t, ctx, client) + + err := client.ContainerRemove(ctx, cID, types.ContainerRemoveOptions{ + Force: true, + }) + require.NoError(t, err) +} + +func TestRemoveInvalidContainer(t *testing.T) { + defer setupTest(t)() + ctx := context.Background() + client := request.NewAPIClient(t) + + err := client.ContainerRemove(ctx, "unknown", types.ContainerRemoveOptions{}) + testutil.ErrorContains(t, err, "No such container") +} From 3b71e8410393a79d95b9e2cc8613655dcd2654b6 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 27 Feb 2018 16:46:14 -0500 Subject: [PATCH 17/20] Migrate some copy tests to integration Signed-off-by: Daniel Nephin Upstream-commit: 00d409f03ed825f623b6ef8ec5a3a91cd26194c2 Component: engine --- .../docker_cli_cp_from_container_test.go | 62 +-------- .../docker_cli_cp_to_container_test.go | 122 ------------------ .../docker_cli_cp_utils_test.go | 12 -- .../engine/integration/container/copy_test.go | 65 ++++++++++ 4 files changed, 66 insertions(+), 195 deletions(-) create mode 100644 components/engine/integration/container/copy_test.go diff --git a/components/engine/integration-cli/docker_cli_cp_from_container_test.go b/components/engine/integration-cli/docker_cli_cp_from_container_test.go index 0a282f5c09..499be54522 100644 --- a/components/engine/integration-cli/docker_cli_cp_from_container_test.go +++ b/components/engine/integration-cli/docker_cli_cp_from_container_test.go @@ -8,8 +8,6 @@ import ( "github.com/go-check/check" ) -// docker cp CONTAINER:PATH LOCALPATH - // Try all of the test cases from the archive package which implements the // internals of `docker cp` and ensure that the behavior matches when actually // copying to and from containers. @@ -20,67 +18,9 @@ import ( // 3. DST parent directory must exist. // 4. If DST exists as a file, it must not end with a trailing separator. -// First get these easy error cases out of the way. - -// Test for error when SRC does not exist. -func (s *DockerSuite) TestCpFromErrSrcNotExists(c *check.C) { - containerID := makeTestContainer(c, testContainerOptions{}) - - tmpDir := getTestDir(c, "test-cp-from-err-src-not-exists") - defer os.RemoveAll(tmpDir) - - err := runDockerCp(c, containerCpPath(containerID, "file1"), tmpDir, nil) - c.Assert(err, checker.NotNil) - - c.Assert(isCpNotExist(err), checker.True, check.Commentf("expected IsNotExist error, but got %T: %s", err, err)) -} - -// Test for error when SRC ends in a trailing -// path separator but it exists as a file. -func (s *DockerSuite) TestCpFromErrSrcNotDir(c *check.C) { - testRequires(c, DaemonIsLinux) - containerID := makeTestContainer(c, testContainerOptions{addContent: true}) - - tmpDir := getTestDir(c, "test-cp-from-err-src-not-dir") - defer os.RemoveAll(tmpDir) - - err := runDockerCp(c, containerCpPathTrailingSep(containerID, "file1"), tmpDir, nil) - c.Assert(err, checker.NotNil) - - c.Assert(isCpNotDir(err), checker.True, check.Commentf("expected IsNotDir error, but got %T: %s", err, err)) -} - -// Test for error when DST ends in a trailing -// path separator but exists as a file. -func (s *DockerSuite) TestCpFromErrDstNotDir(c *check.C) { - testRequires(c, DaemonIsLinux) - containerID := makeTestContainer(c, testContainerOptions{addContent: true}) - - tmpDir := getTestDir(c, "test-cp-from-err-dst-not-dir") - defer os.RemoveAll(tmpDir) - - makeTestContentInDir(c, tmpDir) - - // Try with a file source. - srcPath := containerCpPath(containerID, "/file1") - dstPath := cpPathTrailingSep(tmpDir, "file1") - - err := runDockerCp(c, srcPath, dstPath, nil) - c.Assert(err, checker.NotNil) - - c.Assert(isCpNotDir(err), checker.True, check.Commentf("expected IsNotDir error, but got %T: %s", err, err)) - - // Try with a directory source. - srcPath = containerCpPath(containerID, "/dir1") - - err = runDockerCp(c, srcPath, dstPath, nil) - c.Assert(err, checker.NotNil) - - c.Assert(isCpNotDir(err), checker.True, check.Commentf("expected IsNotDir error, but got %T: %s", err, err)) -} - // Check that copying from a container to a local symlink copies to the symlink // target and does not overwrite the local symlink itself. +// TODO: move to docker/cli and/or integration/container/copy_test.go func (s *DockerSuite) TestCpFromSymlinkDestination(c *check.C) { testRequires(c, DaemonIsLinux) containerID := makeTestContainer(c, testContainerOptions{addContent: true}) diff --git a/components/engine/integration-cli/docker_cli_cp_to_container_test.go b/components/engine/integration-cli/docker_cli_cp_to_container_test.go index 24c1fe2288..77567a3b95 100644 --- a/components/engine/integration-cli/docker_cli_cp_to_container_test.go +++ b/components/engine/integration-cli/docker_cli_cp_to_container_test.go @@ -2,15 +2,11 @@ package main import ( "os" - "runtime" - "strings" "github.com/docker/docker/integration-cli/checker" "github.com/go-check/check" ) -// docker cp LOCALPATH CONTAINER:PATH - // Try all of the test cases from the archive package which implements the // internals of `docker cp` and ensure that the behavior matches when actually // copying to and from containers. @@ -21,124 +17,6 @@ import ( // 3. DST parent directory must exist. // 4. If DST exists as a file, it must not end with a trailing separator. -// First get these easy error cases out of the way. - -// Test for error when SRC does not exist. -func (s *DockerSuite) TestCpToErrSrcNotExists(c *check.C) { - containerID := makeTestContainer(c, testContainerOptions{}) - - tmpDir := getTestDir(c, "test-cp-to-err-src-not-exists") - defer os.RemoveAll(tmpDir) - - srcPath := cpPath(tmpDir, "file1") - dstPath := containerCpPath(containerID, "file1") - _, srcStatErr := os.Stat(srcPath) - c.Assert(os.IsNotExist(srcStatErr), checker.True) - - err := runDockerCp(c, srcPath, dstPath, nil) - if runtime.GOOS == "windows" { - // Go 1.9+ on Windows returns a different error for `os.Stat()`, see - // https://github.com/golang/go/commit/6144c7270e5812d9de8fb97456ee4e5ae657fcbb#diff-f63e1a4b4377b2fe0b05011db3df9599 - // - // Go 1.8: CreateFile C:\not-exist: The system cannot find the file specified. - // Go 1.9: GetFileAttributesEx C:\not-exist: The system cannot find the file specified. - // - // Due to the CLI using a different version than the daemon, comparing the - // error message won't work, so just hard-code the common part here. - // - // TODO this should probably be a test in the CLI repository instead - c.Assert(strings.ToLower(err.Error()), checker.Contains, "cannot find the file specified") - c.Assert(strings.ToLower(err.Error()), checker.Contains, strings.ToLower(tmpDir)) - } else { - c.Assert(strings.ToLower(err.Error()), checker.Contains, strings.ToLower(srcStatErr.Error())) - } -} - -// Test for error when SRC ends in a trailing -// path separator but it exists as a file. -func (s *DockerSuite) TestCpToErrSrcNotDir(c *check.C) { - containerID := makeTestContainer(c, testContainerOptions{}) - - tmpDir := getTestDir(c, "test-cp-to-err-src-not-dir") - defer os.RemoveAll(tmpDir) - - makeTestContentInDir(c, tmpDir) - - srcPath := cpPathTrailingSep(tmpDir, "file1") - dstPath := containerCpPath(containerID, "testDir") - - err := runDockerCp(c, srcPath, dstPath, nil) - c.Assert(err, checker.NotNil) - - c.Assert(isCpNotDir(err), checker.True, check.Commentf("expected IsNotDir error, but got %T: %s", err, err)) -} - -// Test for error when SRC is a valid file or directory, -// but the DST parent directory does not exist. -func (s *DockerSuite) TestCpToErrDstParentNotExists(c *check.C) { - testRequires(c, DaemonIsLinux) - containerID := makeTestContainer(c, testContainerOptions{addContent: true}) - - tmpDir := getTestDir(c, "test-cp-to-err-dst-parent-not-exists") - defer os.RemoveAll(tmpDir) - - makeTestContentInDir(c, tmpDir) - - // Try with a file source. - srcPath := cpPath(tmpDir, "file1") - dstPath := containerCpPath(containerID, "/notExists", "file1") - - err := runDockerCp(c, srcPath, dstPath, nil) - c.Assert(err, checker.NotNil) - - c.Assert(isCpNotExist(err), checker.True, check.Commentf("expected IsNotExist error, but got %T: %s", err, err)) - - // Try with a directory source. - srcPath = cpPath(tmpDir, "dir1") - - err = runDockerCp(c, srcPath, dstPath, nil) - c.Assert(err, checker.NotNil) - - c.Assert(isCpNotExist(err), checker.True, check.Commentf("expected IsNotExist error, but got %T: %s", err, err)) -} - -// Test for error when DST ends in a trailing path separator but exists as a -// file. Also test that we cannot overwrite an existing directory with a -// non-directory and cannot overwrite an existing -func (s *DockerSuite) TestCpToErrDstNotDir(c *check.C) { - testRequires(c, DaemonIsLinux) - containerID := makeTestContainer(c, testContainerOptions{addContent: true}) - - tmpDir := getTestDir(c, "test-cp-to-err-dst-not-dir") - defer os.RemoveAll(tmpDir) - - makeTestContentInDir(c, tmpDir) - - // Try with a file source. - srcPath := cpPath(tmpDir, "dir1/file1-1") - dstPath := containerCpPathTrailingSep(containerID, "file1") - - // The client should encounter an error trying to stat the destination - // and then be unable to copy since the destination is asserted to be a - // directory but does not exist. - err := runDockerCp(c, srcPath, dstPath, nil) - c.Assert(err, checker.NotNil) - - c.Assert(isCpDirNotExist(err), checker.True, check.Commentf("expected DirNotExist error, but got %T: %s", err, err)) - - // Try with a directory source. - srcPath = cpPath(tmpDir, "dir1") - - // The client should encounter an error trying to stat the destination and - // then decide to extract to the parent directory instead with a rebased - // name in the source archive, but this directory would overwrite the - // existing file with the same name. - err = runDockerCp(c, srcPath, dstPath, nil) - c.Assert(err, checker.NotNil) - - c.Assert(isCannotOverwriteNonDirWithDir(err), checker.True, check.Commentf("expected CannotOverwriteNonDirWithDir error, but got %T: %s", err, err)) -} - // Check that copying from a local path to a symlink in a container copies to // the symlink target and does not overwrite the container symlink itself. func (s *DockerSuite) TestCpToSymlinkDestination(c *check.C) { diff --git a/components/engine/integration-cli/docker_cli_cp_utils_test.go b/components/engine/integration-cli/docker_cli_cp_utils_test.go index 402a87ea90..79a016f0c6 100644 --- a/components/engine/integration-cli/docker_cli_cp_utils_test.go +++ b/components/engine/integration-cli/docker_cli_cp_utils_test.go @@ -228,18 +228,10 @@ func getTestDir(c *check.C, label string) (tmpDir string) { return } -func isCpNotExist(err error) bool { - return strings.Contains(strings.ToLower(err.Error()), "could not find the file") -} - func isCpDirNotExist(err error) bool { return strings.Contains(err.Error(), archive.ErrDirNotExists.Error()) } -func isCpNotDir(err error) bool { - return strings.Contains(err.Error(), archive.ErrNotDirectory.Error()) || strings.Contains(err.Error(), "filename, directory name, or volume label syntax is incorrect") -} - func isCpCannotCopyDir(err error) bool { return strings.Contains(err.Error(), archive.ErrCannotCopyDir.Error()) } @@ -248,10 +240,6 @@ func isCpCannotCopyReadOnly(err error) bool { return strings.Contains(err.Error(), "marked read-only") } -func isCannotOverwriteNonDirWithDir(err error) bool { - return strings.Contains(err.Error(), "cannot overwrite non-directory") -} - func fileContentEquals(c *check.C, filename, contents string) (err error) { c.Logf("checking that file %q contains %q\n", filename, contents) diff --git a/components/engine/integration/container/copy_test.go b/components/engine/integration/container/copy_test.go new file mode 100644 index 0000000000..43dc31f2f2 --- /dev/null +++ b/components/engine/integration/container/copy_test.go @@ -0,0 +1,65 @@ +package container // import "github.com/docker/docker/integration/container" + +import ( + "context" + "fmt" + "testing" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" + "github.com/docker/docker/integration/internal/container" + "github.com/docker/docker/internal/testutil" + "github.com/gotestyourself/gotestyourself/skip" + "github.com/stretchr/testify/require" +) + +func TestCopyFromContainerPathDoesNotExist(t *testing.T) { + defer setupTest(t)() + + ctx := context.Background() + apiclient := testEnv.APIClient() + cid := container.Create(t, ctx, apiclient) + + _, _, err := apiclient.CopyFromContainer(ctx, cid, "/dne") + require.True(t, client.IsErrNotFound(err)) + expected := fmt.Sprintf("No such container:path: %s:%s", cid, "/dne") + testutil.ErrorContains(t, err, expected) +} + +func TestCopyFromContainerPathIsNotDir(t *testing.T) { + defer setupTest(t)() + skip.If(t, testEnv.OSType == "windows") + + ctx := context.Background() + apiclient := testEnv.APIClient() + cid := container.Create(t, ctx, apiclient) + + _, _, err := apiclient.CopyFromContainer(ctx, cid, "/etc/passwd/") + require.Contains(t, err.Error(), "not a directory") +} + +func TestCopyToContainerPathDoesNotExist(t *testing.T) { + defer setupTest(t)() + skip.If(t, testEnv.OSType == "windows") + + ctx := context.Background() + apiclient := testEnv.APIClient() + cid := container.Create(t, ctx, apiclient) + + err := apiclient.CopyToContainer(ctx, cid, "/dne", nil, types.CopyToContainerOptions{}) + require.True(t, client.IsErrNotFound(err)) + expected := fmt.Sprintf("No such container:path: %s:%s", cid, "/dne") + testutil.ErrorContains(t, err, expected) +} + +func TestCopyToContainerPathIsNotDir(t *testing.T) { + defer setupTest(t)() + skip.If(t, testEnv.OSType == "windows") + + ctx := context.Background() + apiclient := testEnv.APIClient() + cid := container.Create(t, ctx, apiclient) + + err := apiclient.CopyToContainer(ctx, cid, "/etc/passwd/", nil, types.CopyToContainerOptions{}) + require.Contains(t, err.Error(), "not a directory") +} From 6f429fb8a310e1baa0c4830e051e9cde48fca270 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 26 Feb 2018 15:23:18 -0800 Subject: [PATCH 18/20] Migrate config inspect test to api test This fix migrates config inspect test in integration-cli to api test. Signed-off-by: Yong Tang Upstream-commit: 4b99d782079dc390c2d8fb78f6973bbeee7d8a47 Component: engine --- .../docker_cli_config_inspect_test.go | 68 ------------------- .../engine/integration/config/config_test.go | 25 +++++++ 2 files changed, 25 insertions(+), 68 deletions(-) delete mode 100644 components/engine/integration-cli/docker_cli_config_inspect_test.go diff --git a/components/engine/integration-cli/docker_cli_config_inspect_test.go b/components/engine/integration-cli/docker_cli_config_inspect_test.go deleted file mode 100644 index ba4e80f070..0000000000 --- a/components/engine/integration-cli/docker_cli_config_inspect_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// +build !windows - -package main - -import ( - "encoding/json" - - "github.com/docker/docker/api/types/swarm" - "github.com/docker/docker/integration-cli/checker" - "github.com/go-check/check" -) - -func (s *DockerSwarmSuite) TestConfigInspect(c *check.C) { - d := s.AddDaemon(c, true, true) - - testName := "test_config" - id := d.CreateConfig(c, swarm.ConfigSpec{ - Annotations: swarm.Annotations{ - Name: testName, - }, - Data: []byte("TESTINGDATA"), - }) - c.Assert(id, checker.Not(checker.Equals), "", check.Commentf("configs: %s", id)) - - config := d.GetConfig(c, id) - c.Assert(config.Spec.Name, checker.Equals, testName) - - out, err := d.Cmd("config", "inspect", testName) - c.Assert(err, checker.IsNil, check.Commentf(out)) - - var configs []swarm.Config - c.Assert(json.Unmarshal([]byte(out), &configs), checker.IsNil) - c.Assert(configs, checker.HasLen, 1) -} - -func (s *DockerSwarmSuite) TestConfigInspectMultiple(c *check.C) { - d := s.AddDaemon(c, true, true) - - testNames := []string{ - "test0", - "test1", - } - for _, n := range testNames { - id := d.CreateConfig(c, swarm.ConfigSpec{ - Annotations: swarm.Annotations{ - Name: n, - }, - Data: []byte("TESTINGDATA"), - }) - c.Assert(id, checker.Not(checker.Equals), "", check.Commentf("configs: %s", id)) - - config := d.GetConfig(c, id) - c.Assert(config.Spec.Name, checker.Equals, n) - - } - - args := []string{ - "config", - "inspect", - } - args = append(args, testNames...) - out, err := d.Cmd(args...) - c.Assert(err, checker.IsNil, check.Commentf(out)) - - var configs []swarm.Config - c.Assert(json.Unmarshal([]byte(out), &configs), checker.IsNil) - c.Assert(configs, checker.HasLen, 2) -} diff --git a/components/engine/integration/config/config_test.go b/components/engine/integration/config/config_test.go index c152be59bf..4e31b205ee 100644 --- a/components/engine/integration/config/config_test.go +++ b/components/engine/integration/config/config_test.go @@ -2,6 +2,7 @@ package config import ( "bytes" + "encoding/json" "sort" "testing" "time" @@ -327,3 +328,27 @@ func waitAndAssert(t *testing.T, timeout time.Duration, f func(*testing.T) bool) time.Sleep(100 * time.Millisecond) } } + +func TestConfigInspect(t *testing.T) { + skip.If(t, testEnv.DaemonInfo.OSType != "linux") + + defer setupTest(t)() + d := swarm.NewSwarm(t, testEnv) + defer d.Stop(t) + client, err := client.NewClientWithOpts(client.WithHost((d.Sock()))) + require.NoError(t, err) + + ctx := context.Background() + + testName := t.Name() + configID := createConfig(ctx, t, client, testName, []byte("TESTINGDATA"), nil) + + insp, body, err := client.ConfigInspectWithRaw(ctx, configID) + require.NoError(t, err) + assert.Equal(t, insp.Spec.Name, testName) + + var config swarmtypes.Config + err = json.Unmarshal(body, &config) + require.NoError(t, err) + assert.Equal(t, config, insp) +} From 3711359ab54e745cde20a640abd490408fe04704 Mon Sep 17 00:00:00 2001 From: mnussbaum Date: Tue, 27 Feb 2018 19:07:06 +0000 Subject: [PATCH 19/20] Clean-up after container unit test Remove temp directories and close file loggers in container unit tests. Signed-off-by: mnussbaum Upstream-commit: 07d5446fe27cb92d881df48be6e8a6510d9608b0 Component: engine --- components/engine/container/container_unit_test.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/components/engine/container/container_unit_test.go b/components/engine/container/container_unit_test.go index 858124abdb..863a47a1f2 100644 --- a/components/engine/container/container_unit_test.go +++ b/components/engine/container/container_unit_test.go @@ -3,6 +3,7 @@ package container // import "github.com/docker/docker/container" import ( "fmt" "io/ioutil" + "os" "path/filepath" "testing" @@ -74,6 +75,7 @@ func TestContainerSecretReferenceDestTarget(t *testing.T) { func TestContainerLogPathSetForJSONFileLogger(t *testing.T) { containerRoot, err := ioutil.TempDir("", "TestContainerLogPathSetForJSONFileLogger") require.NoError(t, err) + defer os.RemoveAll(containerRoot) c := &Container{ Config: &container.Config{}, @@ -86,8 +88,9 @@ func TestContainerLogPathSetForJSONFileLogger(t *testing.T) { Root: containerRoot, } - _, err = c.StartLogger() + logger, err := c.StartLogger() require.NoError(t, err) + defer logger.Close() expectedLogPath, err := filepath.Abs(filepath.Join(containerRoot, fmt.Sprintf("%s-json.log", c.ID))) require.NoError(t, err) @@ -97,6 +100,7 @@ func TestContainerLogPathSetForJSONFileLogger(t *testing.T) { func TestContainerLogPathSetForRingLogger(t *testing.T) { containerRoot, err := ioutil.TempDir("", "TestContainerLogPathSetForRingLogger") require.NoError(t, err) + defer os.RemoveAll(containerRoot) c := &Container{ Config: &container.Config{}, @@ -112,8 +116,9 @@ func TestContainerLogPathSetForRingLogger(t *testing.T) { Root: containerRoot, } - _, err = c.StartLogger() + logger, err := c.StartLogger() require.NoError(t, err) + defer logger.Close() expectedLogPath, err := filepath.Abs(filepath.Join(containerRoot, fmt.Sprintf("%s-json.log", c.ID))) require.NoError(t, err) From 6150221be34e401da3514cab1e8ccb6eabc6f812 Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Tue, 27 Feb 2018 09:20:55 +0100 Subject: [PATCH 20/20] Use golang official image instead debian stretch Signed-off-by: Vincent Demeester Upstream-commit: 572cb664621ca08e912a006c5c9cf2fb568ce757 Component: engine --- components/engine/Dockerfile | 41 +++++++++++++++--------------------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile index 99403083c5..b0b611479b 100644 --- a/components/engine/Dockerfile +++ b/components/engine/Dockerfile @@ -32,23 +32,14 @@ # the case. Therefore, you don't have to disable it anymore. # -FROM buildpack-deps:stretch AS base +FROM golang:1.9.4 AS base +# FIXME(vdemeester) this is kept for other script depending on it to not fail right away +# Remove this once the other scripts uses something else to detect the version +ENV GO_VERSION 1.9.4 # allow replacing httpredir or deb mirror ARG APT_MIRROR=deb.debian.org RUN sed -ri "s/(httpredir|deb).debian.org/$APT_MIRROR/g" /etc/apt/sources.list - -FROM base AS golang -# IMPORTANT: If the version of Go is updated, the Windows to Linux CI machines -# will need updating, to avoid errors. Ping #docker-maintainers on IRC -# with a heads-up. -# IMPORTANT: When updating this please note that stdlib archive/tar pkg is vendored -ENV GO_VERSION 1.9.4 -RUN curl -fsSL "https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz" \ - | tar -xzC /usr/local -ENV PATH=/usr/local/go/bin:/go/bin:$PATH GOPATH=/go - - FROM base AS criu # Install CRIU for checkpoint/restore support ENV CRIU_VERSION 3.6 @@ -69,7 +60,7 @@ RUN apt-get update && apt-get install -y \ && make PREFIX=/opt/criu install-criu -FROM golang AS registry +FROM base AS registry # Install two versions of the registry. The first is an older version that # only supports schema1 manifests. The second is a newer version that supports # both. This allows integration-cli tests to cover push/pull with both schema1 @@ -89,7 +80,7 @@ RUN set -x \ -FROM golang AS notary +FROM base AS notary # Install notary and notary-server ENV NOTARY_VERSION v0.5.0 RUN set -x \ @@ -113,7 +104,7 @@ RUN git clone https://github.com/docker/docker-py.git /docker-py \ -FROM golang AS swagger +FROM base AS swagger # Install go-swagger for validating swagger.yaml ENV GO_SWAGGER_COMMIT c28258affb0b6251755d92489ef685af8d4ff3eb RUN set -x \ @@ -124,7 +115,7 @@ RUN set -x \ && rm -rf "$GOPATH" -FROM golang AS frozen-images +FROM base AS frozen-images RUN apt-get update && apt-get install -y jq ca-certificates --no-install-recommends # Get useful and necessary Hub images so we can "docker load" locally instead of pulling COPY contrib/download-frozen-image-v2.sh / @@ -136,44 +127,44 @@ RUN /download-frozen-image-v2.sh /docker-frozen-images \ # See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list) # Just a little hack so we don't have to install these deps twice, once for runc and once for dockerd -FROM golang AS runtime-dev +FROM base AS runtime-dev RUN apt-get update && apt-get install -y \ libapparmor-dev \ libseccomp-dev -FROM golang AS tomlv +FROM base AS tomlv ENV INSTALL_BINARY_NAME=tomlv COPY hack/dockerfile/install/install.sh ./install.sh COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./ RUN PREFIX=/opt/$INSTALL_BINARY_NAME ./install.sh $INSTALL_BINARY_NAME -FROM golang AS vndr +FROM base AS vndr ENV INSTALL_BINARY_NAME=vndr COPY hack/dockerfile/install/install.sh ./install.sh COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./ RUN PREFIX=/opt/$INSTALL_BINARY_NAME ./install.sh $INSTALL_BINARY_NAME -FROM golang AS containerd +FROM base AS containerd RUN apt-get update && apt-get install -y btrfs-tools ENV INSTALL_BINARY_NAME=containerd COPY hack/dockerfile/install/install.sh ./install.sh COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./ RUN PREFIX=/opt/$INSTALL_BINARY_NAME ./install.sh $INSTALL_BINARY_NAME -FROM golang AS proxy +FROM base AS proxy ENV INSTALL_BINARY_NAME=proxy COPY hack/dockerfile/install/install.sh ./install.sh COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./ RUN PREFIX=/opt/$INSTALL_BINARY_NAME ./install.sh $INSTALL_BINARY_NAME -FROM golang AS gometalinter +FROM base AS gometalinter ENV INSTALL_BINARY_NAME=gometalinter COPY hack/dockerfile/install/install.sh ./install.sh COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./ RUN PREFIX=/opt/$INSTALL_BINARY_NAME ./install.sh $INSTALL_BINARY_NAME -FROM golang AS dockercli +FROM base AS dockercli ENV INSTALL_BINARY_NAME=dockercli COPY hack/dockerfile/install/install.sh ./install.sh COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./ @@ -231,6 +222,8 @@ RUN apt-get update && apt-get install -y \ vim-common \ xfsprogs \ zip \ + bzip2 \ + xz-utils \ --no-install-recommends COPY --from=swagger /usr/local/bin/swagger* /usr/local/bin/ COPY --from=frozen-images /docker-frozen-images /docker-frozen-images