From 758ffaf7459066dce4bbdd1a712bb3a495ca7c3b Mon Sep 17 00:00:00 2001 From: John Howard Date: Fri, 23 Feb 2018 15:29:26 -0800 Subject: [PATCH] LCOW: Auto-select OS Signed-off-by: John Howard Addresses https://github.com/moby/moby/pull/35089#issuecomment-367802698. This change enables the daemon to automatically select an image under LCOW that can be used if the API doesn't specify an explicit platform. For example: FROM supertest2014/nyan ADD Dockerfile / And docker build . will download the linux image (not a multi-manifest image) And similarly docker pull ubuntu will match linux/amd64 Upstream-commit: 35193c0e7dc301e1d2f6ea96e0ce34ffd2d4b88d Component: engine --- .../engine/builder/dockerfile/dispatchers.go | 6 ++-- .../engine/daemon/images/image_builder.go | 2 +- components/engine/daemon/images/image_pull.go | 6 ---- components/engine/distribution/pull.go | 6 ---- components/engine/distribution/pull_v2.go | 31 ++++++++++++++----- .../engine/distribution/pull_v2_unix.go | 6 ++-- .../engine/distribution/pull_v2_windows.go | 25 ++++++++------- components/engine/image/store.go | 3 +- components/engine/pkg/system/lcow.go | 4 +-- 9 files changed, 49 insertions(+), 40 deletions(-) diff --git a/components/engine/builder/dockerfile/dispatchers.go b/components/engine/builder/dockerfile/dispatchers.go index 4d47c208b7..4a71b6765e 100644 --- a/components/engine/builder/dockerfile/dispatchers.go +++ b/components/engine/builder/dockerfile/dispatchers.go @@ -228,7 +228,7 @@ func (d *dispatchRequest) getOsFromFlagsAndStage(stageOS string) string { // multi-arch aware yet, it is guaranteed to only hold the OS part here. return d.builder.options.Platform default: - return runtime.GOOS + return "" // Auto-select } } @@ -247,9 +247,9 @@ func (d *dispatchRequest) getImageOrStage(name string, stageOS string) (builder. imageImage.OS = runtime.GOOS if runtime.GOOS == "windows" { switch os { - case "windows", "": + case "windows": return nil, errors.New("Windows does not support FROM scratch") - case "linux": + case "linux", "": if !system.LCOWSupported() { return nil, errors.New("Linux containers are not supported on this system") } diff --git a/components/engine/daemon/images/image_builder.go b/components/engine/daemon/images/image_builder.go index ca7d0fda4a..b792721ee4 100644 --- a/components/engine/daemon/images/image_builder.go +++ b/components/engine/daemon/images/image_builder.go @@ -166,7 +166,7 @@ func (i *ImageService) pullForBuilder(ctx context.Context, name string, authConf // Every call to GetImageAndReleasableLayer MUST call releasableLayer.Release() to prevent // leaking of layers. func (i *ImageService) GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (builder.Image, builder.ROLayer, error) { - if refOrID == "" { + if refOrID == "" { // ie FROM scratch if !system.IsOSSupported(opts.OS) { return nil, nil, system.ErrNotSupportedOperatingSystem } diff --git a/components/engine/daemon/images/image_pull.go b/components/engine/daemon/images/image_pull.go index 238c38b6b3..ed8ecb9abe 100644 --- a/components/engine/daemon/images/image_pull.go +++ b/components/engine/daemon/images/image_pull.go @@ -3,7 +3,6 @@ package images // import "github.com/docker/docker/daemon/images" import ( "context" "io" - "runtime" "strings" "time" @@ -65,11 +64,6 @@ func (i *ImageService) pullImageWithReference(ctx context.Context, ref reference close(writesDone) }() - // Default to the host OS platform in case it hasn't been populated with an explicit value. - if os == "" { - os = runtime.GOOS - } - imagePullConfig := &distribution.ImagePullConfig{ Config: distribution.Config{ MetaHeaders: metaHeaders, diff --git a/components/engine/distribution/pull.go b/components/engine/distribution/pull.go index 0240eb05f7..0c5237f6c0 100644 --- a/components/engine/distribution/pull.go +++ b/components/engine/distribution/pull.go @@ -3,7 +3,6 @@ package distribution // import "github.com/docker/docker/distribution" import ( "context" "fmt" - "runtime" "github.com/docker/distribution/reference" "github.com/docker/docker/api" @@ -115,11 +114,6 @@ func Pull(ctx context.Context, ref reference.Named, imagePullConfig *ImagePullCo continue } - // Make sure we default the OS if it hasn't been supplied - if imagePullConfig.OS == "" { - imagePullConfig.OS = runtime.GOOS - } - if err := puller.Pull(ctx, ref, imagePullConfig.OS); err != nil { // Was this pull cancelled? If so, don't try to fall // back. diff --git a/components/engine/distribution/pull_v2.go b/components/engine/distribution/pull_v2.go index 60a894b1c3..9e07c88d19 100644 --- a/components/engine/distribution/pull_v2.go +++ b/components/engine/distribution/pull_v2.go @@ -509,6 +509,14 @@ func (p *v2Puller) pullSchema1(ctx context.Context, ref reference.Reference, unv } } + // In the situation that the API call didn't specify an OS explicitly, but + // we support the operating system, switch to that operating system. + // eg FROM supertest2014/nyan with no platform specifier, and docker build + // with no --platform= flag under LCOW. + if requestedOS == "" && system.IsOSSupported(configOS) { + requestedOS = configOS + } + // Early bath if the requested OS doesn't match that of the configuration. // This avoids doing the download, only to potentially fail later. if !strings.EqualFold(configOS, requestedOS) { @@ -618,9 +626,10 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s // Early bath if the requested OS doesn't match that of the configuration. // This avoids doing the download, only to potentially fail later. - if !strings.EqualFold(configPlatform.OS, requestedOS) { + if !system.IsOSSupported(configPlatform.OS) { return "", "", fmt.Errorf("cannot download image with operating system %q when requesting %q", configPlatform.OS, requestedOS) } + requestedOS = configPlatform.OS // Populate diff ids in descriptors to avoid downloading foreign layers // which have been side loaded @@ -629,6 +638,10 @@ func (p *v2Puller) pullSchema2(ctx context.Context, ref reference.Named, mfst *s } } + if requestedOS == "" { + requestedOS = runtime.GOOS + } + if p.config.DownloadManager != nil { go func() { var ( @@ -722,18 +735,22 @@ func receiveConfig(s ImageConfigStore, configChan <-chan []byte, errChan <-chan // pullManifestList handles "manifest lists" which point to various // platform-specific manifests. -func (p *v2Puller) pullManifestList(ctx context.Context, ref reference.Named, mfstList *manifestlist.DeserializedManifestList, os string) (id digest.Digest, manifestListDigest digest.Digest, err error) { +func (p *v2Puller) pullManifestList(ctx context.Context, ref reference.Named, mfstList *manifestlist.DeserializedManifestList, requestedOS string) (id digest.Digest, manifestListDigest digest.Digest, err error) { manifestListDigest, err = schema2ManifestDigest(ref, mfstList) if err != nil { return "", "", err } - logrus.Debugf("%s resolved to a manifestList object with %d entries; looking for a %s/%s match", ref, len(mfstList.Manifests), os, runtime.GOARCH) + logOS := requestedOS // May be "" indicating any OS + if logOS == "" { + logOS = "*" + } + logrus.Debugf("%s resolved to a manifestList object with %d entries; looking for a %s/%s match", ref, len(mfstList.Manifests), logOS, runtime.GOARCH) - manifestMatches := filterManifests(mfstList.Manifests, os) + manifestMatches := filterManifests(mfstList.Manifests, requestedOS) if len(manifestMatches) == 0 { - errMsg := fmt.Sprintf("no matching manifest for %s/%s in the manifest list entries", os, runtime.GOARCH) + errMsg := fmt.Sprintf("no matching manifest for %s/%s in the manifest list entries", logOS, runtime.GOARCH) logrus.Debugf(errMsg) return "", "", errors.New(errMsg) } @@ -764,12 +781,12 @@ func (p *v2Puller) pullManifestList(ctx context.Context, ref reference.Named, mf switch v := manifest.(type) { case *schema1.SignedManifest: - id, _, err = p.pullSchema1(ctx, manifestRef, v, os) + id, _, err = p.pullSchema1(ctx, manifestRef, v, manifestMatches[0].Platform.OS) if err != nil { return "", "", err } case *schema2.DeserializedManifest: - id, _, err = p.pullSchema2(ctx, manifestRef, v, os) + id, _, err = p.pullSchema2(ctx, manifestRef, v, manifestMatches[0].Platform.OS) if err != nil { return "", "", err } diff --git a/components/engine/distribution/pull_v2_unix.go b/components/engine/distribution/pull_v2_unix.go index 0be8a03242..65c9594386 100644 --- a/components/engine/distribution/pull_v2_unix.go +++ b/components/engine/distribution/pull_v2_unix.go @@ -16,13 +16,13 @@ func (ld *v2LayerDescriptor) open(ctx context.Context) (distribution.ReadSeekClo return blobs.Open(ctx, ld.digest) } -func filterManifests(manifests []manifestlist.ManifestDescriptor, os string) []manifestlist.ManifestDescriptor { +func filterManifests(manifests []manifestlist.ManifestDescriptor, _ string) []manifestlist.ManifestDescriptor { var matches []manifestlist.ManifestDescriptor for _, manifestDescriptor := range manifests { - if manifestDescriptor.Platform.Architecture == runtime.GOARCH && manifestDescriptor.Platform.OS == os { + if manifestDescriptor.Platform.Architecture == runtime.GOARCH && manifestDescriptor.Platform.OS == runtime.GOOS { matches = append(matches, manifestDescriptor) - logrus.Debugf("found match for %s/%s with media type %s, digest %s", os, runtime.GOARCH, manifestDescriptor.MediaType, manifestDescriptor.Digest.String()) + logrus.Debugf("found match for %s/%s with media type %s, digest %s", runtime.GOOS, runtime.GOARCH, manifestDescriptor.MediaType, manifestDescriptor.Digest.String()) } } return matches diff --git a/components/engine/distribution/pull_v2_windows.go b/components/engine/distribution/pull_v2_windows.go index 432a36119d..fc736e336d 100644 --- a/components/engine/distribution/pull_v2_windows.go +++ b/components/engine/distribution/pull_v2_windows.go @@ -62,24 +62,27 @@ func (ld *v2LayerDescriptor) open(ctx context.Context) (distribution.ReadSeekClo return rsc, err } -func filterManifests(manifests []manifestlist.ManifestDescriptor, os string) []manifestlist.ManifestDescriptor { - osVersion := "" - if os == "windows" { - version := system.GetOSVersion() - osVersion = fmt.Sprintf("%d.%d.%d", version.MajorVersion, version.MinorVersion, version.Build) - logrus.Debugf("will prefer entries with version %s", osVersion) - } +func filterManifests(manifests []manifestlist.ManifestDescriptor, requestedOS string) []manifestlist.ManifestDescriptor { + version := system.GetOSVersion() + osVersion := fmt.Sprintf("%d.%d.%d", version.MajorVersion, version.MinorVersion, version.Build) + logrus.Debugf("will prefer Windows entries with version %s", osVersion) var matches []manifestlist.ManifestDescriptor + foundWindowsMatch := false for _, manifestDescriptor := range manifests { - if manifestDescriptor.Platform.Architecture == runtime.GOARCH && manifestDescriptor.Platform.OS == os { + if (manifestDescriptor.Platform.Architecture == runtime.GOARCH) && + ((requestedOS != "" && manifestDescriptor.Platform.OS == requestedOS) || // Explicit user request for an OS we know we support + (requestedOS == "" && system.IsOSSupported(manifestDescriptor.Platform.OS))) { // No user requested OS, but one we can support matches = append(matches, manifestDescriptor) - logrus.Debugf("found match for %s/%s %s with media type %s, digest %s", os, runtime.GOARCH, manifestDescriptor.Platform.OSVersion, manifestDescriptor.MediaType, manifestDescriptor.Digest.String()) + logrus.Debugf("found match %s/%s %s with media type %s, digest %s", manifestDescriptor.Platform.OS, runtime.GOARCH, manifestDescriptor.Platform.OSVersion, manifestDescriptor.MediaType, manifestDescriptor.Digest.String()) + if strings.EqualFold("windows", manifestDescriptor.Platform.OS) { + foundWindowsMatch = true + } } else { - logrus.Debugf("ignoring %s/%s %s with media type %s, digest %s", os, runtime.GOARCH, manifestDescriptor.Platform.OSVersion, manifestDescriptor.MediaType, manifestDescriptor.Digest.String()) + logrus.Debugf("ignoring %s/%s %s with media type %s, digest %s", manifestDescriptor.Platform.OS, manifestDescriptor.Platform.Architecture, manifestDescriptor.Platform.OSVersion, manifestDescriptor.MediaType, manifestDescriptor.Digest.String()) } } - if os == "windows" { + if foundWindowsMatch { sort.Stable(manifestsByVersion{osVersion, matches}) } return matches diff --git a/components/engine/image/store.go b/components/engine/image/store.go index 9fd7d7dcf3..1a8a8a2451 100644 --- a/components/engine/image/store.go +++ b/components/engine/image/store.go @@ -76,7 +76,8 @@ func (is *store) restore() error { var l layer.Layer if chainID := img.RootFS.ChainID(); chainID != "" { if !system.IsOSSupported(img.OperatingSystem()) { - return system.ErrNotSupportedOperatingSystem + logrus.Errorf("not restoring image with unsupported operating system %v, %v, %s", dgst, chainID, img.OperatingSystem()) + return nil } l, err = is.lss[img.OperatingSystem()].Get(chainID) if err != nil { diff --git a/components/engine/pkg/system/lcow.go b/components/engine/pkg/system/lcow.go index 5c3fbfe6f4..dc8ef9f96d 100644 --- a/components/engine/pkg/system/lcow.go +++ b/components/engine/pkg/system/lcow.go @@ -59,10 +59,10 @@ func ParsePlatform(in string) *specs.Platform { // IsOSSupported determines if an operating system is supported by the host func IsOSSupported(os string) bool { - if runtime.GOOS == os { + if strings.EqualFold(runtime.GOOS, os) { return true } - if LCOWSupported() && os == "linux" { + if LCOWSupported() && strings.EqualFold(os, "linux") { return true } return false