diff --git a/components/engine/api/server/backend/build/backend.go b/components/engine/api/server/backend/build/backend.go index 546ad5f86d..5e04e837a1 100644 --- a/components/engine/api/server/backend/build/backend.go +++ b/components/engine/api/server/backend/build/backend.go @@ -73,7 +73,7 @@ func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string return "", err } if config.ProgressWriter.AuxFormatter != nil { - if err = config.ProgressWriter.AuxFormatter.Emit(types.BuildResult{ID: imageID}); err != nil { + if err = config.ProgressWriter.AuxFormatter.Emit("moby.image.id", types.BuildResult{ID: imageID}); err != nil { return "", err } } diff --git a/components/engine/api/server/router/build/build_routes.go b/components/engine/api/server/router/build/build_routes.go index 071402ae70..c4699f3d86 100644 --- a/components/engine/api/server/router/build/build_routes.go +++ b/components/engine/api/server/router/build/build_routes.go @@ -14,7 +14,6 @@ import ( "strings" "sync" - "github.com/containerd/containerd/platforms" "github.com/docker/docker/api/server/httputils" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/backend" @@ -24,8 +23,7 @@ import ( "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/progress" "github.com/docker/docker/pkg/streamformatter" - "github.com/docker/docker/pkg/system" - "github.com/docker/go-units" + units "github.com/docker/go-units" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -72,17 +70,7 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui options.Target = r.FormValue("target") options.RemoteContext = r.FormValue("remote") if versions.GreaterThanOrEqualTo(version, "1.32") { - apiPlatform := r.FormValue("platform") - if apiPlatform != "" { - sp, err := platforms.Parse(apiPlatform) - if err != nil { - return nil, err - } - if err := system.ValidatePlatform(sp); err != nil { - return nil, err - } - options.Platform = &sp - } + options.Platform = r.FormValue("platform") } if r.Form.Get("shmsize") != "" { @@ -243,6 +231,10 @@ func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r * return errdefs.InvalidParameter(errors.New("squash is only supported with experimental mode")) } + if buildOptions.Version == types.BuilderBuildKit && !br.daemon.HasExperimental() { + return errdefs.InvalidParameter(errors.New("buildkit is only supported with experimental mode")) + } + out := io.Writer(output) if buildOptions.SuppressOutput { out = notVerboseBuffer @@ -255,10 +247,6 @@ func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r * return progress.NewProgressReader(in, progressOutput, r.ContentLength, "Downloading context", buildOptions.RemoteContext) } - if buildOptions.Version == types.BuilderBuildKit && !br.daemon.HasExperimental() { - return errdefs.InvalidParameter(errors.New("buildkit is only supported with experimental mode")) - } - wantAux := versions.GreaterThanOrEqualTo(version, "1.30") imgID, err := br.backend.Build(ctx, backend.BuildConfig{ diff --git a/components/engine/api/types/client.go b/components/engine/api/types/client.go index 33bc98e0bb..3b698c2c24 100644 --- a/components/engine/api/types/client.go +++ b/components/engine/api/types/client.go @@ -7,8 +7,7 @@ import ( "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/filters" - "github.com/docker/go-units" - specs "github.com/opencontainers/image-spec/specs-go/v1" + units "github.com/docker/go-units" ) // CheckpointCreateOptions holds parameters to create a checkpoint from a container @@ -181,7 +180,7 @@ type ImageBuildOptions struct { ExtraHosts []string // List of extra hosts Target string SessionID string - Platform *specs.Platform + Platform string // Version specifies the version of the unerlying builder to use Version BuilderVersion // BuildID is an optional identifier that can be passed together with the diff --git a/components/engine/builder/builder-next/adapters/containerimage/pull.go b/components/engine/builder/builder-next/adapters/containerimage/pull.go index f76ac5a1aa..2b6c214b8b 100644 --- a/components/engine/builder/builder-next/adapters/containerimage/pull.go +++ b/components/engine/builder/builder-next/adapters/containerimage/pull.go @@ -644,7 +644,7 @@ func showProgress(ctx context.Context, ongoing *jobs, cs content.Store, pw progr // featured. type jobs struct { name string - added map[digest.Digest]job + added map[digest.Digest]*job mu sync.Mutex resolved bool } @@ -658,7 +658,7 @@ type job struct { func newJobs(name string) *jobs { return &jobs{ name: name, - added: make(map[digest.Digest]job), + added: make(map[digest.Digest]*job), } } @@ -669,17 +669,17 @@ func (j *jobs) add(desc ocispec.Descriptor) { if _, ok := j.added[desc.Digest]; ok { return } - j.added[desc.Digest] = job{ + j.added[desc.Digest] = &job{ Descriptor: desc, started: time.Now(), } } -func (j *jobs) jobs() []job { +func (j *jobs) jobs() []*job { j.mu.Lock() defer j.mu.Unlock() - descs := make([]job, 0, len(j.added)) + descs := make([]*job, 0, len(j.added)) for _, j := range j.added { descs = append(descs, j) } diff --git a/components/engine/builder/builder-next/builder.go b/components/engine/builder/builder-next/builder.go index 5a82cddf44..b1d31a5225 100644 --- a/components/engine/builder/builder-next/builder.go +++ b/components/engine/builder/builder-next/builder.go @@ -2,7 +2,6 @@ package buildkit import ( "context" - "encoding/json" "io" "strings" "sync" @@ -14,7 +13,8 @@ import ( "github.com/docker/docker/api/types/backend" "github.com/docker/docker/builder" "github.com/docker/docker/daemon/images" - "github.com/docker/docker/pkg/jsonmessage" + "github.com/docker/docker/pkg/streamformatter" + "github.com/docker/docker/pkg/system" controlapi "github.com/moby/buildkit/api/services/control" "github.com/moby/buildkit/control" "github.com/moby/buildkit/identity" @@ -209,8 +209,17 @@ func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder. frontendAttrs["no-cache"] = "" } - if opt.Options.Platform != nil { - frontendAttrs["platform"] = platforms.Format(*opt.Options.Platform) + if opt.Options.Platform != "" { + // same as in newBuilder in builder/dockerfile.builder.go + // TODO: remove once opt.Options.Platform is of type specs.Platform + sp, err := platforms.Parse(opt.Options.Platform) + if err != nil { + return nil, err + } + if err := system.ValidatePlatform(sp); err != nil { + return nil, err + } + frontendAttrs["platform"] = opt.Options.Platform } exporterAttrs := map[string]string{} @@ -228,6 +237,8 @@ func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder. Session: opt.Options.SessionID, } + aux := streamformatter.AuxFormatter{opt.ProgressWriter.Output} + eg, ctx := errgroup.WithContext(ctx) eg.Go(func() error { @@ -240,7 +251,7 @@ func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder. return errors.Errorf("missing image id") } out.ImageID = id - return nil + return aux.Emit("moby.image.id", types.BuildResult{ID: id}) }) ch := make(chan *controlapi.StatusResponse) @@ -258,25 +269,9 @@ func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder. if err != nil { return err } - - auxJSONBytes, err := json.Marshal(dt) - if err != nil { + if err := aux.Emit("moby.buildkit.trace", dt); err != nil { return err } - auxJSON := new(json.RawMessage) - *auxJSON = auxJSONBytes - msgJSON, err := json.Marshal(&jsonmessage.JSONMessage{ID: "moby.buildkit.trace", Aux: auxJSON}) - if err != nil { - return err - } - msgJSON = append(msgJSON, []byte("\r\n")...) - n, err := opt.ProgressWriter.Output.Write(msgJSON) - if err != nil { - return err - } - if n != len(msgJSON) { - return io.ErrShortWrite - } } return nil }) diff --git a/components/engine/builder/dockerfile/builder.go b/components/engine/builder/dockerfile/builder.go index b585347079..ee95f23689 100644 --- a/components/engine/builder/dockerfile/builder.go +++ b/components/engine/builder/dockerfile/builder.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/containerd/containerd/platforms" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/backend" "github.com/docker/docker/api/types/container" @@ -25,6 +26,7 @@ import ( "github.com/moby/buildkit/frontend/dockerfile/parser" "github.com/moby/buildkit/frontend/dockerfile/shell" "github.com/moby/buildkit/session" + specs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/sync/syncmap" @@ -111,7 +113,11 @@ func (bm *BuildManager) Build(ctx context.Context, config backend.BuildConfig) ( PathCache: bm.pathCache, IDMappings: bm.idMappings, } - return newBuilder(ctx, builderOptions).build(source, dockerfile) + b, err := newBuilder(ctx, builderOptions) + if err != nil { + return nil, err + } + return b.build(source, dockerfile) } func (bm *BuildManager) initializeClientSession(ctx context.Context, cancel func(), options *types.ImageBuildOptions) (builder.Source, error) { @@ -175,10 +181,11 @@ type Builder struct { pathCache pathCache containerManager *containerManager imageProber ImageProber + platform *specs.Platform } // newBuilder creates a new Dockerfile builder from an optional dockerfile and a Options. -func newBuilder(clientCtx context.Context, options builderOptions) *Builder { +func newBuilder(clientCtx context.Context, options builderOptions) (*Builder, error) { config := options.Options if config == nil { config = new(types.ImageBuildOptions) @@ -199,7 +206,20 @@ func newBuilder(clientCtx context.Context, options builderOptions) *Builder { containerManager: newContainerManager(options.Backend), } - return b + // same as in Builder.Build in builder/builder-next/builder.go + // TODO: remove once config.Platform is of type specs.Platform + if config.Platform != "" { + sp, err := platforms.Parse(config.Platform) + if err != nil { + return nil, err + } + if err := system.ValidatePlatform(sp); err != nil { + return nil, err + } + b.platform = &sp + } + + return b, nil } // Build 'LABEL' command(s) from '--label' options and add to the last stage @@ -257,7 +277,7 @@ func emitImageID(aux *streamformatter.AuxFormatter, state *dispatchState) error if aux == nil || state.imageID == "" { return nil } - return aux.Emit(types.BuildResult{ID: state.imageID}) + return aux.Emit("", types.BuildResult{ID: state.imageID}) } func processMetaArg(meta instructions.ArgCommand, shlex *shell.Lex, args *BuildArgs) error { @@ -365,9 +385,12 @@ func BuildFromConfig(config *container.Config, changes []string, os string) (*co return nil, errdefs.InvalidParameter(err) } - b := newBuilder(context.Background(), builderOptions{ + b, err := newBuilder(context.Background(), builderOptions{ Options: &types.ImageBuildOptions{NoCache: true}, }) + if err != nil { + return nil, err + } // ensure that the commands are valid for _, n := range dockerfile.AST.Children { diff --git a/components/engine/builder/dockerfile/copy.go b/components/engine/builder/dockerfile/copy.go index 7e9dc6036a..74e245bdc4 100644 --- a/components/engine/builder/dockerfile/copy.go +++ b/components/engine/builder/dockerfile/copy.go @@ -87,7 +87,7 @@ func copierFromDispatchRequest(req dispatchRequest, download sourceDownloader, i pathCache: req.builder.pathCache, download: download, imageSource: imageSource, - platform: req.builder.options.Platform, + platform: req.builder.platform, } } diff --git a/components/engine/builder/dockerfile/dispatchers.go b/components/engine/builder/dockerfile/dispatchers.go index f59b7c844c..6ee2b17cda 100644 --- a/components/engine/builder/dockerfile/dispatchers.go +++ b/components/engine/builder/dockerfile/dispatchers.go @@ -146,7 +146,7 @@ func (d *dispatchRequest) getImageMount(imageRefOrID string) (*imageMount, error imageRefOrID = stage.Image localOnly = true } - return d.builder.imageSources.Get(imageRefOrID, localOnly, d.builder.options.Platform) + return d.builder.imageSources.Get(imageRefOrID, localOnly, d.builder.platform) } // FROM [--platform=platform] imagename[:tag | @digest] [AS build-stage-name] @@ -238,7 +238,7 @@ func (d *dispatchRequest) getImageOrStage(name string, platform *specs.Platform) } if platform == nil { - platform = d.builder.options.Platform + platform = d.builder.platform } // Windows cannot support a container with no base image unless it is LCOW. diff --git a/components/engine/builder/dockerfile/dispatchers_test.go b/components/engine/builder/dockerfile/dispatchers_test.go index 047a8742e8..c61a45b03a 100644 --- a/components/engine/builder/dockerfile/dispatchers_test.go +++ b/components/engine/builder/dockerfile/dispatchers_test.go @@ -6,7 +6,6 @@ import ( "runtime" "testing" - "github.com/containerd/containerd/platforms" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/backend" "github.com/docker/docker/api/types/container" @@ -23,8 +22,7 @@ import ( func newBuilderWithMockBackend() *Builder { mockBackend := &MockBackend{} - defaultPlatform := platforms.DefaultSpec() - opts := &types.ImageBuildOptions{Platform: &defaultPlatform} + opts := &types.ImageBuildOptions{} ctx := context.Background() b := &Builder{ options: opts, @@ -116,7 +114,7 @@ func TestFromScratch(t *testing.T) { err := initializeStage(sb, cmd) if runtime.GOOS == "windows" && !system.LCOWSupported() { - assert.Check(t, is.Error(err, "Windows does not support FROM scratch")) + assert.Check(t, is.Error(err, "Linux containers are not supported on this system")) return } diff --git a/components/engine/builder/dockerfile/internals.go b/components/engine/builder/dockerfile/internals.go index 5e2c286d75..1b3a5b0f03 100644 --- a/components/engine/builder/dockerfile/internals.go +++ b/components/engine/builder/dockerfile/internals.go @@ -169,7 +169,7 @@ func (b *Builder) performCopy(req dispatchRequest, inst copyInstruction) error { return err } - imageMount, err := b.imageSources.Get(state.imageID, true, req.builder.options.Platform) + imageMount, err := b.imageSources.Get(state.imageID, true, req.builder.platform) if err != nil { return errors.Wrapf(err, "failed to get destination image %q", state.imageID) } @@ -416,7 +416,9 @@ func (b *Builder) probeAndCreate(dispatchState *dispatchState, runConfig *contai func (b *Builder) create(runConfig *container.Config) (string, error) { logrus.Debugf("[BUILDER] Command to be executed: %v", runConfig.Cmd) - hostConfig := hostConfigFromOptions(b.options) + + isWCOW := runtime.GOOS == "windows" && b.platform != nil && b.platform.OS == "windows" + hostConfig := hostConfigFromOptions(b.options, isWCOW) container, err := b.containerManager.Create(runConfig, hostConfig) if err != nil { return "", err @@ -429,7 +431,7 @@ func (b *Builder) create(runConfig *container.Config) (string, error) { return container.ID, nil } -func hostConfigFromOptions(options *types.ImageBuildOptions) *container.HostConfig { +func hostConfigFromOptions(options *types.ImageBuildOptions, isWCOW bool) *container.HostConfig { resources := container.Resources{ CgroupParent: options.CgroupParent, CPUShares: options.CPUShares, @@ -457,7 +459,7 @@ func hostConfigFromOptions(options *types.ImageBuildOptions) *container.HostConf // is too small for builder scenarios where many users are // using RUN statements to install large amounts of data. // Use 127GB as that's the default size of a VHD in Hyper-V. - if runtime.GOOS == "windows" && options.Platform != nil && options.Platform.OS == "windows" { + if isWCOW { hc.StorageOpt = make(map[string]string) hc.StorageOpt["size"] = "127GB" } diff --git a/components/engine/builder/remotecontext/detect.go b/components/engine/builder/remotecontext/detect.go index aaace269e9..49b196ed7b 100644 --- a/components/engine/builder/remotecontext/detect.go +++ b/components/engine/builder/remotecontext/detect.go @@ -22,8 +22,7 @@ import ( const ClientSessionRemote = "client-session" // Detect returns a context and dockerfile from remote location or local -// archive. progressReader is only used if remoteURL is actually a URL -// (not empty, and not a Git endpoint). +// archive. func Detect(config backend.BuildConfig) (remote builder.Source, dockerfile *parser.Result, err error) { remoteURL := config.Options.RemoteContext dockerfilePath := config.Options.Dockerfile diff --git a/components/engine/client/image_build.go b/components/engine/client/image_build.go index e5013176a2..9add3c10b3 100644 --- a/components/engine/client/image_build.go +++ b/components/engine/client/image_build.go @@ -8,8 +8,8 @@ import ( "net/http" "net/url" "strconv" + "strings" - "github.com/containerd/containerd/platforms" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" ) @@ -30,12 +30,6 @@ func (cli *Client) ImageBuild(ctx context.Context, buildContext io.Reader, optio } headers.Add("X-Registry-Config", base64.URLEncoding.EncodeToString(buf)) - if options.Platform != nil { - if err := cli.NewVersionError("1.32", "platform"); err != nil { - return types.ImageBuildResponse{}, err - } - query.Set("platform", platforms.Format(*options.Platform)) - } headers.Set("Content-Type", "application/x-tar") serverResp, err := cli.postRaw(ctx, "/build", query, buildContext, headers) @@ -130,8 +124,11 @@ func (cli *Client) imageBuildOptionsToQuery(options types.ImageBuildOptions) (ur if options.SessionID != "" { query.Set("session", options.SessionID) } - if options.Platform != nil { - query.Set("platform", platforms.Format(*options.Platform)) + if options.Platform != "" { + if err := cli.NewVersionError("1.32", "platform"); err != nil { + return query, err + } + query.Set("platform", strings.ToLower(options.Platform)) } if options.BuildID != "" { query.Set("buildid", options.BuildID) diff --git a/components/engine/contrib/mkimage-alpine.sh b/components/engine/contrib/mkimage-alpine.sh index 03180e435a..db41ddc09d 100755 --- a/components/engine/contrib/mkimage-alpine.sh +++ b/components/engine/contrib/mkimage-alpine.sh @@ -29,7 +29,7 @@ getapk() { } mkbase() { - $TMP/sbin/apk.static --repository $MAINREPO --update-cache --allow-untrusted \ + $TMP/sbin/apk.static --repository $MAINREPO --no-cache --allow-untrusted \ --root $ROOTFS --initdb add alpine-base } diff --git a/components/engine/contrib/syntax/vim/README.md b/components/engine/contrib/syntax/vim/README.md index 5aa9bd825d..b73a30bea3 100644 --- a/components/engine/contrib/syntax/vim/README.md +++ b/components/engine/contrib/syntax/vim/README.md @@ -11,6 +11,10 @@ With [Vundle](https://github.com/gmarik/Vundle.vim) Plugin 'docker/docker' , {'rtp': '/contrib/syntax/vim/'} +With [vim-plug](https://github.com/junegunn/vim-plug) + + Plug 'docker/docker' , {'rtp': '/contrib/syntax/vim/'} + Features -------- diff --git a/components/engine/pkg/streamformatter/streamformatter.go b/components/engine/pkg/streamformatter/streamformatter.go index 2b5e713040..04917d49ab 100644 --- a/components/engine/pkg/streamformatter/streamformatter.go +++ b/components/engine/pkg/streamformatter/streamformatter.go @@ -139,14 +139,14 @@ type AuxFormatter struct { } // Emit emits the given interface as an aux progress message -func (sf *AuxFormatter) Emit(aux interface{}) error { +func (sf *AuxFormatter) Emit(id string, aux interface{}) error { auxJSONBytes, err := json.Marshal(aux) if err != nil { return err } auxJSON := new(json.RawMessage) *auxJSON = auxJSONBytes - msgJSON, err := json.Marshal(&jsonmessage.JSONMessage{Aux: auxJSON}) + msgJSON, err := json.Marshal(&jsonmessage.JSONMessage{ID: id, Aux: auxJSON}) if err != nil { return err } diff --git a/components/engine/pkg/streamformatter/streamformatter_test.go b/components/engine/pkg/streamformatter/streamformatter_test.go index 4399a6509b..f630699d73 100644 --- a/components/engine/pkg/streamformatter/streamformatter_test.go +++ b/components/engine/pkg/streamformatter/streamformatter_test.go @@ -106,7 +106,7 @@ func TestAuxFormatterEmit(t *testing.T) { sampleAux := &struct { Data string }{"Additional data"} - err := aux.Emit(sampleAux) + err := aux.Emit("", sampleAux) assert.NilError(t, err) assert.Check(t, is.Equal(`{"aux":{"Data":"Additional data"}}`+streamNewline, b.String())) } diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index 866df011ef..33e2dc8445 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -26,7 +26,7 @@ github.com/imdario/mergo v0.3.5 golang.org/x/sync fd80eb99c8f653c847d294a001bdf2a3a6f768f5 # buildkit -github.com/moby/buildkit cce2080ddbe4698912f2290892b247c83627efa8 +github.com/moby/buildkit 9acf51e49185b348608e0096b2903dd72907adcb github.com/tonistiigi/fsutil 8abad97ee3969cdf5e9c367f46adba2c212b3ddb github.com/grpc-ecosystem/grpc-opentracing 8e809c8a86450a29b90dcc9efbf062d0fe6d9746 github.com/opentracing/opentracing-go 1361b9cd60be79c4c3a7fa9841b3c132e40066a7 diff --git a/components/engine/vendor/github.com/moby/buildkit/README.md b/components/engine/vendor/github.com/moby/buildkit/README.md index ba8525c63d..567947634e 100644 --- a/components/engine/vendor/github.com/moby/buildkit/README.md +++ b/components/engine/vendor/github.com/moby/buildkit/README.md @@ -138,11 +138,11 @@ docker inspect myimage ##### Building a Dockerfile using [external frontend](https://hub.docker.com/r/tonistiigi/dockerfile/tags/): -During development, an external version of the Dockerfile frontend is pushed to https://hub.docker.com/r/tonistiigi/dockerfile that can be used with the gateway frontend. The source for the external frontend is currently located in `./frontend/dockerfile/cmd/dockerfile-frontend` but will move out of this repository in the future ([#163](https://github.com/moby/buildkit/issues/163)). +During development, an external version of the Dockerfile frontend is pushed to https://hub.docker.com/r/tonistiigi/dockerfile that can be used with the gateway frontend. The source for the external frontend is currently located in `./frontend/dockerfile/cmd/dockerfile-frontend` but will move out of this repository in the future ([#163](https://github.com/moby/buildkit/issues/163)). For automatic build from master branch of this repository `tonistiigi/dockerfile:master` image can be used. ``` -buildctl build --frontend=gateway.v0 --frontend-opt=source=tonistiigi/dockerfile:v0 --local context=. --local dockerfile=. -buildctl build --frontend gateway.v0 --frontend-opt=source=tonistiigi/dockerfile:v0 --frontend-opt=context=git://github.com/moby/moby --frontend-opt build-arg:APT_MIRROR=cdn-fastly.deb.debian.org +buildctl build --frontend=gateway.v0 --frontend-opt=source=tonistiigi/dockerfile --local context=. --local dockerfile=. +buildctl build --frontend gateway.v0 --frontend-opt=source=tonistiigi/dockerfile --frontend-opt=context=git://github.com/moby/moby --frontend-opt build-arg:APT_MIRROR=cdn-fastly.deb.debian.org ```` ### Exporters diff --git a/components/engine/vendor/github.com/moby/buildkit/cache/remotecache/export.go b/components/engine/vendor/github.com/moby/buildkit/cache/remotecache/export.go index 40a36759d8..0536ba2a84 100644 --- a/components/engine/vendor/github.com/moby/buildkit/cache/remotecache/export.go +++ b/components/engine/vendor/github.com/moby/buildkit/cache/remotecache/export.go @@ -9,7 +9,6 @@ import ( "github.com/containerd/containerd/content" "github.com/containerd/containerd/images" - "github.com/docker/distribution/manifest" v1 "github.com/moby/buildkit/cache/remotecache/v1" "github.com/moby/buildkit/session" "github.com/moby/buildkit/solver" @@ -17,6 +16,7 @@ import ( "github.com/moby/buildkit/util/progress" "github.com/moby/buildkit/util/push" digest "github.com/opencontainers/go-digest" + specs "github.com/opencontainers/image-spec/specs-go" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) @@ -46,7 +46,9 @@ func (ce *CacheExporter) Finalize(ctx context.Context, cc *v1.CacheChains, targe // own type because oci type can't be pushed and docker type doesn't have annotations type manifestList struct { - manifest.Versioned + specs.Versioned + + MediaType string `json:"mediaType,omitempty"` // Manifests references platform specific manifests. Manifests []ocispec.Descriptor `json:"manifests"` diff --git a/components/engine/vendor/github.com/moby/buildkit/executor/oci/user.go b/components/engine/vendor/github.com/moby/buildkit/executor/oci/user.go index ce755f18a2..ac5dbebdf2 100644 --- a/components/engine/vendor/github.com/moby/buildkit/executor/oci/user.go +++ b/components/engine/vendor/github.com/moby/buildkit/executor/oci/user.go @@ -2,27 +2,31 @@ package oci import ( "context" + "errors" "os" "strconv" "strings" + "github.com/containerd/containerd/containers" + containerdoci "github.com/containerd/containerd/oci" "github.com/containerd/continuity/fs" "github.com/opencontainers/runc/libcontainer/user" + "github.com/opencontainers/runtime-spec/specs-go" ) -func GetUser(ctx context.Context, root, username string) (uint32, uint32, error) { +func GetUser(ctx context.Context, root, username string) (uint32, uint32, []uint32, error) { // fast path from uid/gid - if uid, gid, err := ParseUser(username); err == nil { - return uid, gid, nil + if uid, gid, err := ParseUIDGID(username); err == nil { + return uid, gid, nil, nil } passwdPath, err := user.GetPasswdPath() if err != nil { - return 0, 0, err + return 0, 0, nil, err } groupPath, err := user.GetGroupPath() if err != nil { - return 0, 0, err + return 0, 0, nil, err } passwdFile, err := openUserFile(root, passwdPath) if err == nil { @@ -35,33 +39,29 @@ func GetUser(ctx context.Context, root, username string) (uint32, uint32, error) execUser, err := user.GetExecUser(username, nil, passwdFile, groupFile) if err != nil { - return 0, 0, err + return 0, 0, nil, err } - - return uint32(execUser.Uid), uint32(execUser.Gid), nil + var sgids []uint32 + for _, g := range execUser.Sgids { + sgids = append(sgids, uint32(g)) + } + return uint32(execUser.Uid), uint32(execUser.Gid), sgids, nil } -func ParseUser(str string) (uid uint32, gid uint32, err error) { +// ParseUIDGID takes the fast path to parse UID and GID if and only if they are both provided +func ParseUIDGID(str string) (uid uint32, gid uint32, err error) { if str == "" { return 0, 0, nil } parts := strings.SplitN(str, ":", 2) - for i, v := range parts { - switch i { - case 0: - uid, err = parseUID(v) - if err != nil { - return 0, 0, err - } - if len(parts) == 1 { - gid = uid - } - case 1: - gid, err = parseUID(v) - if err != nil { - return 0, 0, err - } - } + if len(parts) == 1 { + return 0, 0, errors.New("groups ID is not provided") + } + if uid, err = parseUID(parts[0]); err != nil { + return 0, 0, err + } + if gid, err = parseUID(parts[1]); err != nil { + return 0, 0, err } return } @@ -84,3 +84,24 @@ func parseUID(str string) (uint32, error) { } return uint32(uid), nil } + +// WithUIDGID allows the UID and GID for the Process to be set +// FIXME: This is a temporeray fix for the missing supplementary GIDs from containerd +// once the PR in containerd is merged we should remove this function. +func WithUIDGID(uid, gid uint32, sgids []uint32) containerdoci.SpecOpts { + return func(_ context.Context, _ containerdoci.Client, _ *containers.Container, s *containerdoci.Spec) error { + setProcess(s) + s.Process.User.UID = uid + s.Process.User.GID = gid + s.Process.User.AdditionalGids = sgids + return nil + } +} + +// setProcess sets Process to empty if unset +// FIXME: Same on this one. Need to be removed after containerd fix merged +func setProcess(s *containerdoci.Spec) { + if s.Process == nil { + s.Process = &specs.Process{} + } +} diff --git a/components/engine/vendor/github.com/moby/buildkit/executor/runcexecutor/executor.go b/components/engine/vendor/github.com/moby/buildkit/executor/runcexecutor/executor.go index edffb5bf58..97eb3430a0 100644 --- a/components/engine/vendor/github.com/moby/buildkit/executor/runcexecutor/executor.go +++ b/components/engine/vendor/github.com/moby/buildkit/executor/runcexecutor/executor.go @@ -133,7 +133,7 @@ func (w *runcExecutor) Exec(ctx context.Context, meta executor.Meta, root cache. } defer mount.Unmount(rootFSPath, 0) - uid, gid, err := oci.GetUser(ctx, rootFSPath, meta.User) + uid, gid, sgids, err := oci.GetUser(ctx, rootFSPath, meta.User) if err != nil { return err } @@ -143,7 +143,7 @@ func (w *runcExecutor) Exec(ctx context.Context, meta executor.Meta, root cache. return err } defer f.Close() - opts := []containerdoci.SpecOpts{containerdoci.WithUIDGID(uid, gid)} + opts := []containerdoci.SpecOpts{oci.WithUIDGID(uid, gid, sgids)} if system.SeccompSupported() { opts = append(opts, seccomp.WithDefaultProfile()) } @@ -170,9 +170,7 @@ func (w *runcExecutor) Exec(ctx context.Context, meta executor.Meta, root cache. } if w.rootless { - specconv.ToRootless(spec, &specconv.RootlessOpts{ - MapSubUIDGID: true, - }) + specconv.ToRootless(spec, nil) // TODO(AkihiroSuda): keep Cgroups enabled if /sys/fs/cgroup/cpuset/buildkit exists and writable spec.Linux.CgroupsPath = "" // TODO(AkihiroSuda): ToRootless removes netns, but we should readd netns here diff --git a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert.go b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert.go index 00e92b3ffa..ac70fb8aaf 100644 --- a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert.go +++ b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert.go @@ -91,8 +91,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, metaResolver = imagemetaresolver.Default() } - var allDispatchStates []*dispatchState - dispatchStatesByName := map[string]*dispatchState{} + allDispatchStates := newDispatchStates() // set base state for every image for _, st := range stages { @@ -100,6 +99,9 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, if err != nil { return nil, nil, err } + if name == "" { + return nil, nil, errors.Errorf("base name (%s) should not be blank", st.BaseName) + } st.BaseName = name ds := &dispatchState{ @@ -121,13 +123,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, ds.platform = &p } - if d, ok := dispatchStatesByName[st.BaseName]; ok { - ds.base = d - } - allDispatchStates = append(allDispatchStates, ds) - if st.Name != "" { - dispatchStatesByName[strings.ToLower(st.Name)] = ds - } + allDispatchStates.addState(ds) if opt.IgnoreCache != nil { if len(opt.IgnoreCache) == 0 { ds.ignoreCache = true @@ -143,20 +139,20 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, var target *dispatchState if opt.Target == "" { - target = allDispatchStates[len(allDispatchStates)-1] + target = allDispatchStates.lastTarget() } else { var ok bool - target, ok = dispatchStatesByName[strings.ToLower(opt.Target)] + target, ok = allDispatchStates.findStateByName(opt.Target) if !ok { return nil, nil, errors.Errorf("target stage %s could not be found", opt.Target) } } // fill dependencies to stages so unreachable ones can avoid loading image configs - for _, d := range allDispatchStates { + for _, d := range allDispatchStates.states { d.commands = make([]command, len(d.stage.Commands)) for i, cmd := range d.stage.Commands { - newCmd, err := toCommand(cmd, dispatchStatesByName, allDispatchStates) + newCmd, err := toCommand(cmd, allDispatchStates) if err != nil { return nil, nil, err } @@ -165,7 +161,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, if src != nil { d.deps[src] = struct{}{} if src.unregistered { - allDispatchStates = append(allDispatchStates, src) + allDispatchStates.addState(src) } } } @@ -173,7 +169,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, } eg, ctx := errgroup.WithContext(ctx) - for i, d := range allDispatchStates { + for i, d := range allDispatchStates.states { reachable := isReachable(target, d) // resolve image config for every stage if d.base == nil { @@ -239,7 +235,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, buildContext := &mutableOutput{} ctxPaths := map[string]struct{}{} - for _, d := range allDispatchStates { + for _, d := range allDispatchStates.states { if !isReachable(target, d) { continue } @@ -271,17 +267,16 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, } opt := dispatchOpt{ - allDispatchStates: allDispatchStates, - dispatchStatesByName: dispatchStatesByName, - metaArgs: metaArgs, - buildArgValues: opt.BuildArgs, - shlex: shlex, - sessionID: opt.SessionID, - buildContext: llb.NewState(buildContext), - proxyEnv: proxyEnv, - cacheIDNamespace: opt.CacheIDNamespace, - buildPlatforms: opt.BuildPlatforms, - targetPlatform: *opt.TargetPlatform, + allDispatchStates: allDispatchStates, + metaArgs: metaArgs, + buildArgValues: opt.BuildArgs, + shlex: shlex, + sessionID: opt.SessionID, + buildContext: llb.NewState(buildContext), + proxyEnv: proxyEnv, + cacheIDNamespace: opt.CacheIDNamespace, + buildPlatforms: opt.BuildPlatforms, + targetPlatform: *opt.TargetPlatform, } if err = dispatchOnBuild(d, d.image.Config.OnBuild, opt); err != nil { @@ -330,14 +325,14 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, return &st, &target.image, nil } -func toCommand(ic instructions.Command, dispatchStatesByName map[string]*dispatchState, allDispatchStates []*dispatchState) (command, error) { +func toCommand(ic instructions.Command, allDispatchStates *dispatchStates) (command, error) { cmd := command{Command: ic} if c, ok := ic.(*instructions.CopyCommand); ok { if c.From != "" { var stn *dispatchState index, err := strconv.Atoi(c.From) if err != nil { - stn, ok = dispatchStatesByName[strings.ToLower(c.From)] + stn, ok = allDispatchStates.findStateByName(c.From) if !ok { stn = &dispatchState{ stage: instructions.Stage{BaseName: c.From}, @@ -346,16 +341,16 @@ func toCommand(ic instructions.Command, dispatchStatesByName map[string]*dispatc } } } else { - if index < 0 || index >= len(allDispatchStates) { - return command{}, errors.Errorf("invalid stage index %d", index) + stn, err = allDispatchStates.findStateByIndex(index) + if err != nil { + return command{}, err } - stn = allDispatchStates[index] } cmd.sources = []*dispatchState{stn} } } - if ok := detectRunMount(&cmd, dispatchStatesByName, allDispatchStates); ok { + if ok := detectRunMount(&cmd, allDispatchStates); ok { return cmd, nil } @@ -363,17 +358,16 @@ func toCommand(ic instructions.Command, dispatchStatesByName map[string]*dispatc } type dispatchOpt struct { - allDispatchStates []*dispatchState - dispatchStatesByName map[string]*dispatchState - metaArgs []instructions.ArgCommand - buildArgValues map[string]string - shlex *shell.Lex - sessionID string - buildContext llb.State - proxyEnv *llb.ProxyEnv - cacheIDNamespace string - targetPlatform specs.Platform - buildPlatforms []specs.Platform + allDispatchStates *dispatchStates + metaArgs []instructions.ArgCommand + buildArgValues map[string]string + shlex *shell.Lex + sessionID string + buildContext llb.State + proxyEnv *llb.ProxyEnv + cacheIDNamespace string + targetPlatform specs.Platform + buildPlatforms []specs.Platform } func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error { @@ -456,6 +450,43 @@ type dispatchState struct { unregistered bool } +type dispatchStates struct { + states []*dispatchState + statesByName map[string]*dispatchState +} + +func newDispatchStates() *dispatchStates { + return &dispatchStates{statesByName: map[string]*dispatchState{}} +} + +func (dss *dispatchStates) addState(ds *dispatchState) { + dss.states = append(dss.states, ds) + + if d, ok := dss.statesByName[ds.stage.BaseName]; ok { + ds.base = d + } + if ds.stage.Name != "" { + dss.statesByName[strings.ToLower(ds.stage.Name)] = ds + } +} + +func (dss *dispatchStates) findStateByName(name string) (*dispatchState, bool) { + ds, ok := dss.statesByName[strings.ToLower(name)] + return ds, ok +} + +func (dss *dispatchStates) findStateByIndex(index int) (*dispatchState, error) { + if index < 0 || index >= len(dss.states) { + return nil, errors.Errorf("invalid stage index %d", index) + } + + return dss.states[index], nil +} + +func (dss *dispatchStates) lastTarget() *dispatchState { + return dss.states[len(dss.states)-1] +} + type command struct { instructions.Command sources []*dispatchState @@ -474,7 +505,7 @@ func dispatchOnBuild(d *dispatchState, triggers []string, opt dispatchOpt) error if err != nil { return err } - cmd, err := toCommand(ic, opt.dispatchStatesByName, opt.allDispatchStates) + cmd, err := toCommand(ic, opt.allDispatchStates) if err != nil { return err } @@ -570,7 +601,11 @@ func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState l for i, src := range c.Sources() { commitMessage.WriteString(" " + src) - if isAddCommand && (strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://")) { + if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") { + if !isAddCommand { + return errors.New("source can't be a URL for COPY") + } + // Resources from remote URLs are not decompressed. // https://docs.docker.com/engine/reference/builder/#add // diff --git a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_norunmount.go b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_norunmount.go index a4544e7975..d7643405aa 100644 --- a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_norunmount.go +++ b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_norunmount.go @@ -7,7 +7,7 @@ import ( "github.com/moby/buildkit/frontend/dockerfile/instructions" ) -func detectRunMount(cmd *command, dispatchStatesByName map[string]*dispatchState, allDispatchStates []*dispatchState) bool { +func detectRunMount(cmd *command, allDispatchStates *dispatchStates) bool { return false } diff --git a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_runmount.go b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_runmount.go index 61408e1ff0..aea61b3675 100644 --- a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_runmount.go +++ b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_runmount.go @@ -5,14 +5,13 @@ package dockerfile2llb import ( "path" "path/filepath" - "strings" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/frontend/dockerfile/instructions" "github.com/pkg/errors" ) -func detectRunMount(cmd *command, dispatchStatesByName map[string]*dispatchState, allDispatchStates []*dispatchState) bool { +func detectRunMount(cmd *command, allDispatchStates *dispatchStates) bool { if c, ok := cmd.Command.(*instructions.RunCommand); ok { mounts := instructions.GetMounts(c) sources := make([]*dispatchState, len(mounts)) @@ -24,7 +23,7 @@ func detectRunMount(cmd *command, dispatchStatesByName map[string]*dispatchState if from == "" || mount.Type == instructions.MountTypeTmpfs { continue } - stn, ok := dispatchStatesByName[strings.ToLower(from)] + stn, ok := allDispatchStates.findStateByName(from) if !ok { stn = &dispatchState{ stage: instructions.Stage{BaseName: from}, diff --git a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/bflag.go b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/bflag.go index e299d52323..d8bf747394 100644 --- a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/bflag.go +++ b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/bflag.go @@ -72,7 +72,7 @@ func (bf *BFlags) AddString(name string, def string) *Flag { return flag } -// AddString adds a string flag to BFlags that can match multiple values +// AddStrings adds a string flag to BFlags that can match multiple values func (bf *BFlags) AddStrings(name string) *Flag { flag := bf.addFlag(name, stringsType) if flag == nil { diff --git a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands.go b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands.go index b96010f0b1..903353e4cf 100644 --- a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands.go +++ b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands.go @@ -159,7 +159,7 @@ func (s SourcesAndDest) Dest() string { // AddCommand : ADD foo /path // -// Add the file 'foo' to '/path'. Tarball and Remote URL (git, http) handling +// Add the file 'foo' to '/path'. Tarball and Remote URL (http, https) handling // exist here. If you do not wish to have this automatic handling, use COPY. // type AddCommand struct { diff --git a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/shell/equal_env_unix.go b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/shell/equal_env_unix.go index 6e3f6b890e..36903ec58d 100644 --- a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/shell/equal_env_unix.go +++ b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/shell/equal_env_unix.go @@ -2,8 +2,9 @@ package shell -// EqualEnvKeys compare two strings and returns true if they are equal. On -// Windows this comparison is case insensitive. +// EqualEnvKeys compare two strings and returns true if they are equal. +// On Unix this comparison is case sensitive. +// On Windows this comparison is case insensitive. func EqualEnvKeys(from, to string) bool { return from == to } diff --git a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/shell/equal_env_windows.go b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/shell/equal_env_windows.go index 7780fb67e8..010569bbaa 100644 --- a/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/shell/equal_env_windows.go +++ b/components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/shell/equal_env_windows.go @@ -2,8 +2,9 @@ package shell import "strings" -// EqualEnvKeys compare two strings and returns true if they are equal. On -// Windows this comparison is case insensitive. +// EqualEnvKeys compare two strings and returns true if they are equal. +// On Unix this comparison is case sensitive. +// On Windows this comparison is case insensitive. func EqualEnvKeys(from, to string) bool { return strings.ToUpper(from) == strings.ToUpper(to) } diff --git a/components/engine/vendor/github.com/moby/buildkit/util/imageutil/config.go b/components/engine/vendor/github.com/moby/buildkit/util/imageutil/config.go index 2c2e18ba53..356ed53cea 100644 --- a/components/engine/vendor/github.com/moby/buildkit/util/imageutil/config.go +++ b/components/engine/vendor/github.com/moby/buildkit/util/imageutil/config.go @@ -141,13 +141,17 @@ func DetectManifestMediaType(ra content.ReaderAt) (string, error) { } var mfst struct { - Config json.RawMessage `json:"config"` + MediaType string `json:"mediaType"` + Config json.RawMessage `json:"config"` } if err := json.Unmarshal(p, &mfst); err != nil { return "", err } + if mfst.MediaType != "" { + return mfst.MediaType, nil + } if mfst.Config != nil { return images.MediaTypeDockerSchema2Manifest, nil }