toolshed/docker-cli
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge component 'engine' from git@github.com:docker/engine master

Browse Source
This commit is contained in:
GordonTheTurtle
2018-07-05 17:07:30 +00:00
parent 06dc7cf0b2 aaa1925293
commit 33125af849
29 changed files with 239 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files
components/engine/api/server/backend/build/backend.go
+1 -1
View File
@@ -73,7 +73,7 @@ func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string
return "", err
}
if config.ProgressWriter.AuxFormatter != nil {
if err = config.ProgressWriter.AuxFormatter.Emit(types.BuildResult{ID: imageID}); err != nil {
if err = config.ProgressWriter.AuxFormatter.Emit("moby.image.id", types.BuildResult{ID: imageID}); err != nil {
return "", err
}
}
components/engine/api/server/router/build/build_routes.go
+6 -18
View File
@@ -14,7 +14,6 @@ import (
"strings"
"sync"
"github.com/containerd/containerd/platforms"
"github.com/docker/docker/api/server/httputils"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/backend"
@@ -24,8 +23,7 @@ import (
"github.com/docker/docker/pkg/ioutils"
"github.com/docker/docker/pkg/progress"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/pkg/system"
"github.com/docker/go-units"
units "github.com/docker/go-units"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
@@ -72,17 +70,7 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui
options.Target = r.FormValue("target")
options.RemoteContext = r.FormValue("remote")
if versions.GreaterThanOrEqualTo(version, "1.32") {
apiPlatform := r.FormValue("platform")
if apiPlatform != "" {
sp, err := platforms.Parse(apiPlatform)
if err != nil {
return nil, err
}
if err := system.ValidatePlatform(sp); err != nil {
return nil, err
}
options.Platform = &sp
}
options.Platform = r.FormValue("platform")
}
if r.Form.Get("shmsize") != "" {
@@ -243,6 +231,10 @@ func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *
return errdefs.InvalidParameter(errors.New("squash is only supported with experimental mode"))
}
if buildOptions.Version == types.BuilderBuildKit && !br.daemon.HasExperimental() {
return errdefs.InvalidParameter(errors.New("buildkit is only supported with experimental mode"))
}
out := io.Writer(output)
if buildOptions.SuppressOutput {
out = notVerboseBuffer
@@ -255,10 +247,6 @@ func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *
return progress.NewProgressReader(in, progressOutput, r.ContentLength, "Downloading context", buildOptions.RemoteContext)
}
if buildOptions.Version == types.BuilderBuildKit && !br.daemon.HasExperimental() {
return errdefs.InvalidParameter(errors.New("buildkit is only supported with experimental mode"))
}
wantAux := versions.GreaterThanOrEqualTo(version, "1.30")
imgID, err := br.backend.Build(ctx, backend.BuildConfig{
components/engine/api/types/client.go
+2 -3
View File
@@ -7,8 +7,7 @@ import (
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/filters"
"github.com/docker/go-units"
specs "github.com/opencontainers/image-spec/specs-go/v1"
units "github.com/docker/go-units"
)
// CheckpointCreateOptions holds parameters to create a checkpoint from a container
@@ -181,7 +180,7 @@ type ImageBuildOptions struct {
ExtraHosts []string // List of extra hosts
Target string
SessionID string
Platform *specs.Platform
Platform string
// Version specifies the version of the unerlying builder to use
Version BuilderVersion
// BuildID is an optional identifier that can be passed together with the
components/engine/builder/builder-next/adapters/containerimage/pull.go
+5 -5
View File
@@ -644,7 +644,7 @@ func showProgress(ctx context.Context, ongoing *jobs, cs content.Store, pw progr
// featured.
type jobs struct {
name string
added map[digest.Digest]job
added map[digest.Digest]*job
mu sync.Mutex
resolved bool
}
@@ -658,7 +658,7 @@ type job struct {
func newJobs(name string) *jobs {
return &jobs{
name: name,
added: make(map[digest.Digest]job),
added: make(map[digest.Digest]*job),
}
}
@@ -669,17 +669,17 @@ func (j *jobs) add(desc ocispec.Descriptor) {
if _, ok := j.added[desc.Digest]; ok {
return
}
j.added[desc.Digest] = job{
j.added[desc.Digest] = &job{
Descriptor: desc,
started: time.Now(),
}
}
func (j *jobs) jobs() []job {
func (j *jobs) jobs() []*job {
j.mu.Lock()
defer j.mu.Unlock()
descs := make([]job, 0, len(j.added))
descs := make([]*job, 0, len(j.added))
for _, j := range j.added {
descs = append(descs, j)
}
components/engine/builder/builder-next/builder.go
+17 -22
View File
@@ -2,7 +2,6 @@ package buildkit
import (
"context"
"encoding/json"
"io"
"strings"
"sync"
@@ -14,7 +13,8 @@ import (
"github.com/docker/docker/api/types/backend"
"github.com/docker/docker/builder"
"github.com/docker/docker/daemon/images"
"github.com/docker/docker/pkg/jsonmessage"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/pkg/system"
controlapi "github.com/moby/buildkit/api/services/control"
"github.com/moby/buildkit/control"
"github.com/moby/buildkit/identity"
@@ -209,8 +209,17 @@ func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder.
frontendAttrs["no-cache"] = ""
}
if opt.Options.Platform != nil {
frontendAttrs["platform"] = platforms.Format(*opt.Options.Platform)
if opt.Options.Platform != "" {
// same as in newBuilder in builder/dockerfile.builder.go
// TODO: remove once opt.Options.Platform is of type specs.Platform
sp, err := platforms.Parse(opt.Options.Platform)
if err != nil {
return nil, err
}
if err := system.ValidatePlatform(sp); err != nil {
return nil, err
}
frontendAttrs["platform"] = opt.Options.Platform
}
exporterAttrs := map[string]string{}
@@ -228,6 +237,8 @@ func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder.
Session: opt.Options.SessionID,
}
aux := streamformatter.AuxFormatter{opt.ProgressWriter.Output}
eg, ctx := errgroup.WithContext(ctx)
eg.Go(func() error {
@@ -240,7 +251,7 @@ func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder.
return errors.Errorf("missing image id")
}
out.ImageID = id
return nil
return aux.Emit("moby.image.id", types.BuildResult{ID: id})
})
ch := make(chan *controlapi.StatusResponse)
@@ -258,25 +269,9 @@ func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder.
if err != nil {
return err
}
auxJSONBytes, err := json.Marshal(dt)
if err != nil {
if err := aux.Emit("moby.buildkit.trace", dt); err != nil {
return err
}
auxJSON := new(json.RawMessage)
*auxJSON = auxJSONBytes
msgJSON, err := json.Marshal(&jsonmessage.JSONMessage{ID: "moby.buildkit.trace", Aux: auxJSON})
if err != nil {
return err
}
msgJSON = append(msgJSON, []byte("\r\n")...)
n, err := opt.ProgressWriter.Output.Write(msgJSON)
if err != nil {
return err
}
if n != len(msgJSON) {
return io.ErrShortWrite
}
}
return nil
})
components/engine/builder/dockerfile/builder.go
+28 -5
View File
@@ -10,6 +10,7 @@ import (
"strings"
"time"
"github.com/containerd/containerd/platforms"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/backend"
"github.com/docker/docker/api/types/container"
@@ -25,6 +26,7 @@ import (
"github.com/moby/buildkit/frontend/dockerfile/parser"
"github.com/moby/buildkit/frontend/dockerfile/shell"
"github.com/moby/buildkit/session"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/sync/syncmap"
@@ -111,7 +113,11 @@ func (bm *BuildManager) Build(ctx context.Context, config backend.BuildConfig) (
PathCache: bm.pathCache,
IDMappings: bm.idMappings,
}
return newBuilder(ctx, builderOptions).build(source, dockerfile)
b, err := newBuilder(ctx, builderOptions)
if err != nil {
return nil, err
}
return b.build(source, dockerfile)
}
func (bm *BuildManager) initializeClientSession(ctx context.Context, cancel func(), options *types.ImageBuildOptions) (builder.Source, error) {
@@ -175,10 +181,11 @@ type Builder struct {
pathCache pathCache
containerManager *containerManager
imageProber ImageProber
platform *specs.Platform
}
// newBuilder creates a new Dockerfile builder from an optional dockerfile and a Options.
func newBuilder(clientCtx context.Context, options builderOptions) *Builder {
func newBuilder(clientCtx context.Context, options builderOptions) (*Builder, error) {
config := options.Options
if config == nil {
config = new(types.ImageBuildOptions)
@@ -199,7 +206,20 @@ func newBuilder(clientCtx context.Context, options builderOptions) *Builder {
containerManager: newContainerManager(options.Backend),
}
return b
// same as in Builder.Build in builder/builder-next/builder.go
// TODO: remove once config.Platform is of type specs.Platform
if config.Platform != "" {
sp, err := platforms.Parse(config.Platform)
if err != nil {
return nil, err
}
if err := system.ValidatePlatform(sp); err != nil {
return nil, err
}
b.platform = &sp
}
return b, nil
}
// Build 'LABEL' command(s) from '--label' options and add to the last stage
@@ -257,7 +277,7 @@ func emitImageID(aux *streamformatter.AuxFormatter, state *dispatchState) error
if aux == nil || state.imageID == "" {
return nil
}
return aux.Emit(types.BuildResult{ID: state.imageID})
return aux.Emit("", types.BuildResult{ID: state.imageID})
}
func processMetaArg(meta instructions.ArgCommand, shlex *shell.Lex, args *BuildArgs) error {
@@ -365,9 +385,12 @@ func BuildFromConfig(config *container.Config, changes []string, os string) (*co
return nil, errdefs.InvalidParameter(err)
}
b := newBuilder(context.Background(), builderOptions{
b, err := newBuilder(context.Background(), builderOptions{
Options: &types.ImageBuildOptions{NoCache: true},
})
if err != nil {
return nil, err
}
// ensure that the commands are valid
for _, n := range dockerfile.AST.Children {
components/engine/builder/dockerfile/copy.go
+1 -1
View File
@@ -87,7 +87,7 @@ func copierFromDispatchRequest(req dispatchRequest, download sourceDownloader, i
pathCache: req.builder.pathCache,
download: download,
imageSource: imageSource,
platform: req.builder.options.Platform,
platform: req.builder.platform,
}
}
components/engine/builder/dockerfile/dispatchers.go
+2 -2
View File
@@ -146,7 +146,7 @@ func (d *dispatchRequest) getImageMount(imageRefOrID string) (*imageMount, error
imageRefOrID = stage.Image
localOnly = true
}
return d.builder.imageSources.Get(imageRefOrID, localOnly, d.builder.options.Platform)
return d.builder.imageSources.Get(imageRefOrID, localOnly, d.builder.platform)
}
// FROM [--platform=platform] imagename[:tag | @digest] [AS build-stage-name]
@@ -238,7 +238,7 @@ func (d *dispatchRequest) getImageOrStage(name string, platform *specs.Platform)
}
if platform == nil {
platform = d.builder.options.Platform
platform = d.builder.platform
}
// Windows cannot support a container with no base image unless it is LCOW.
components/engine/builder/dockerfile/dispatchers_test.go
+2 -4
View File
@@ -6,7 +6,6 @@ import (
"runtime"
"testing"
"github.com/containerd/containerd/platforms"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/backend"
"github.com/docker/docker/api/types/container"
@@ -23,8 +22,7 @@ import (
func newBuilderWithMockBackend() *Builder {
mockBackend := &MockBackend{}
defaultPlatform := platforms.DefaultSpec()
opts := &types.ImageBuildOptions{Platform: &defaultPlatform}
opts := &types.ImageBuildOptions{}
ctx := context.Background()
b := &Builder{
options: opts,
@@ -116,7 +114,7 @@ func TestFromScratch(t *testing.T) {
err := initializeStage(sb, cmd)
if runtime.GOOS == "windows" && !system.LCOWSupported() {
assert.Check(t, is.Error(err, "Windows does not support FROM scratch"))
assert.Check(t, is.Error(err, "Linux containers are not supported on this system"))
return
}
components/engine/builder/dockerfile/internals.go
+6 -4
View File
@@ -169,7 +169,7 @@ func (b *Builder) performCopy(req dispatchRequest, inst copyInstruction) error {
return err
}
imageMount, err := b.imageSources.Get(state.imageID, true, req.builder.options.Platform)
imageMount, err := b.imageSources.Get(state.imageID, true, req.builder.platform)
if err != nil {
return errors.Wrapf(err, "failed to get destination image %q", state.imageID)
}
@@ -416,7 +416,9 @@ func (b *Builder) probeAndCreate(dispatchState *dispatchState, runConfig *contai
func (b *Builder) create(runConfig *container.Config) (string, error) {
logrus.Debugf("[BUILDER] Command to be executed: %v", runConfig.Cmd)
hostConfig := hostConfigFromOptions(b.options)
isWCOW := runtime.GOOS == "windows" && b.platform != nil && b.platform.OS == "windows"
hostConfig := hostConfigFromOptions(b.options, isWCOW)
container, err := b.containerManager.Create(runConfig, hostConfig)
if err != nil {
return "", err
@@ -429,7 +431,7 @@ func (b *Builder) create(runConfig *container.Config) (string, error) {
return container.ID, nil
}
func hostConfigFromOptions(options *types.ImageBuildOptions) *container.HostConfig {
func hostConfigFromOptions(options *types.ImageBuildOptions, isWCOW bool) *container.HostConfig {
resources := container.Resources{
CgroupParent: options.CgroupParent,
CPUShares: options.CPUShares,
@@ -457,7 +459,7 @@ func hostConfigFromOptions(options *types.ImageBuildOptions) *container.HostConf
// is too small for builder scenarios where many users are
// using RUN statements to install large amounts of data.
// Use 127GB as that's the default size of a VHD in Hyper-V.
if runtime.GOOS == "windows" && options.Platform != nil && options.Platform.OS == "windows" {
if isWCOW {
hc.StorageOpt = make(map[string]string)
hc.StorageOpt["size"] = "127GB"
}
components/engine/builder/remotecontext/detect.go
+1 -2
View File
@@ -22,8 +22,7 @@ import (
const ClientSessionRemote = "client-session"
// Detect returns a context and dockerfile from remote location or local
// archive. progressReader is only used if remoteURL is actually a URL
// (not empty, and not a Git endpoint).
// archive.
func Detect(config backend.BuildConfig) (remote builder.Source, dockerfile *parser.Result, err error) {
remoteURL := config.Options.RemoteContext
dockerfilePath := config.Options.Dockerfile
components/engine/client/image_build.go
+6 -9
View File
@@ -8,8 +8,8 @@ import (
"net/http"
"net/url"
"strconv"
"strings"
"github.com/containerd/containerd/platforms"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
)
@@ -30,12 +30,6 @@ func (cli *Client) ImageBuild(ctx context.Context, buildContext io.Reader, optio
}
headers.Add("X-Registry-Config", base64.URLEncoding.EncodeToString(buf))
if options.Platform != nil {
if err := cli.NewVersionError("1.32", "platform"); err != nil {
return types.ImageBuildResponse{}, err
}
query.Set("platform", platforms.Format(*options.Platform))
}
headers.Set("Content-Type", "application/x-tar")
serverResp, err := cli.postRaw(ctx, "/build", query, buildContext, headers)
@@ -130,8 +124,11 @@ func (cli *Client) imageBuildOptionsToQuery(options types.ImageBuildOptions) (ur
if options.SessionID != "" {
query.Set("session", options.SessionID)
}
if options.Platform != nil {
query.Set("platform", platforms.Format(*options.Platform))
if options.Platform != "" {
if err := cli.NewVersionError("1.32", "platform"); err != nil {
return query, err
}
query.Set("platform", strings.ToLower(options.Platform))
}
if options.BuildID != "" {
query.Set("buildid", options.BuildID)
components/engine/contrib/mkimage-alpine.sh
+1 -1
View File
@@ -29,7 +29,7 @@ getapk() {
}
mkbase() {
$TMP/sbin/apk.static --repository $MAINREPO --update-cache --allow-untrusted \
$TMP/sbin/apk.static --repository $MAINREPO --no-cache --allow-untrusted \
--root $ROOTFS --initdb add alpine-base
}
components/engine/contrib/syntax/vim/README.md
+4
View File
@@ -11,6 +11,10 @@ With [Vundle](https://github.com/gmarik/Vundle.vim)
Plugin 'docker/docker' , {'rtp': '/contrib/syntax/vim/'}
With [vim-plug](https://github.com/junegunn/vim-plug)
Plug 'docker/docker' , {'rtp': '/contrib/syntax/vim/'}
Features
--------
components/engine/pkg/streamformatter/streamformatter.go
+2 -2
View File
@@ -139,14 +139,14 @@ type AuxFormatter struct {
}
// Emit emits the given interface as an aux progress message
func (sf *AuxFormatter) Emit(aux interface{}) error {
func (sf *AuxFormatter) Emit(id string, aux interface{}) error {
auxJSONBytes, err := json.Marshal(aux)
if err != nil {
return err
}
auxJSON := new(json.RawMessage)
*auxJSON = auxJSONBytes
msgJSON, err := json.Marshal(&jsonmessage.JSONMessage{Aux: auxJSON})
msgJSON, err := json.Marshal(&jsonmessage.JSONMessage{ID: id, Aux: auxJSON})
if err != nil {
return err
}
components/engine/pkg/streamformatter/streamformatter_test.go
+1 -1
View File
@@ -106,7 +106,7 @@ func TestAuxFormatterEmit(t *testing.T) {
sampleAux := &struct {
Data string
}{"Additional data"}
err := aux.Emit(sampleAux)
err := aux.Emit("", sampleAux)
assert.NilError(t, err)
assert.Check(t, is.Equal(`{"aux":{"Data":"Additional data"}}`+streamNewline, b.String()))
}
components/engine/vendor.conf
+1 -1
View File
@@ -26,7 +26,7 @@ github.com/imdario/mergo v0.3.5
golang.org/x/sync fd80eb99c8f653c847d294a001bdf2a3a6f768f5
# buildkit
github.com/moby/buildkit cce2080ddbe4698912f2290892b247c83627efa8
github.com/moby/buildkit 9acf51e49185b348608e0096b2903dd72907adcb
github.com/tonistiigi/fsutil 8abad97ee3969cdf5e9c367f46adba2c212b3ddb
github.com/grpc-ecosystem/grpc-opentracing 8e809c8a86450a29b90dcc9efbf062d0fe6d9746
github.com/opentracing/opentracing-go 1361b9cd60be79c4c3a7fa9841b3c132e40066a7
components/engine/vendor/github.com/moby/buildkit/README.md
Generated Vendored
+3 -3
View File
@@ -138,11 +138,11 @@ docker inspect myimage
##### Building a Dockerfile using [external frontend](https://hub.docker.com/r/tonistiigi/dockerfile/tags/):
During development, an external version of the Dockerfile frontend is pushed to https://hub.docker.com/r/tonistiigi/dockerfile that can be used with the gateway frontend. The source for the external frontend is currently located in `./frontend/dockerfile/cmd/dockerfile-frontend` but will move out of this repository in the future ([#163](https://github.com/moby/buildkit/issues/163)).
During development, an external version of the Dockerfile frontend is pushed to https://hub.docker.com/r/tonistiigi/dockerfile that can be used with the gateway frontend. The source for the external frontend is currently located in `./frontend/dockerfile/cmd/dockerfile-frontend` but will move out of this repository in the future ([#163](https://github.com/moby/buildkit/issues/163)). For automatic build from master branch of this repository `tonistiigi/dockerfile:master` image can be used.
```
buildctl build --frontend=gateway.v0 --frontend-opt=source=tonistiigi/dockerfile:v0 --local context=. --local dockerfile=.
buildctl build --frontend gateway.v0 --frontend-opt=source=tonistiigi/dockerfile:v0 --frontend-opt=context=git://github.com/moby/moby --frontend-opt build-arg:APT_MIRROR=cdn-fastly.deb.debian.org
buildctl build --frontend=gateway.v0 --frontend-opt=source=tonistiigi/dockerfile --local context=. --local dockerfile=.
buildctl build --frontend gateway.v0 --frontend-opt=source=tonistiigi/dockerfile --frontend-opt=context=git://github.com/moby/moby --frontend-opt build-arg:APT_MIRROR=cdn-fastly.deb.debian.org
````
### Exporters
components/engine/vendor/github.com/moby/buildkit/cache/remotecache/export.go
Generated Vendored
+4 -2
View File
@@ -9,7 +9,6 @@ import (
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/images"
"github.com/docker/distribution/manifest"
v1 "github.com/moby/buildkit/cache/remotecache/v1"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/solver"
@@ -17,6 +16,7 @@ import (
"github.com/moby/buildkit/util/progress"
"github.com/moby/buildkit/util/push"
digest "github.com/opencontainers/go-digest"
specs "github.com/opencontainers/image-spec/specs-go"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
@@ -46,7 +46,9 @@ func (ce *CacheExporter) Finalize(ctx context.Context, cc *v1.CacheChains, targe
// own type because oci type can't be pushed and docker type doesn't have annotations
type manifestList struct {
manifest.Versioned
specs.Versioned
MediaType string `json:"mediaType,omitempty"`
// Manifests references platform specific manifests.
Manifests []ocispec.Descriptor `json:"manifests"`
components/engine/vendor/github.com/moby/buildkit/executor/oci/user.go
Generated Vendored
+46 -25
View File
@@ -2,27 +2,31 @@ package oci
import (
"context"
"errors"
"os"
"strconv"
"strings"
"github.com/containerd/containerd/containers"
containerdoci "github.com/containerd/containerd/oci"
"github.com/containerd/continuity/fs"
"github.com/opencontainers/runc/libcontainer/user"
"github.com/opencontainers/runtime-spec/specs-go"
)
func GetUser(ctx context.Context, root, username string) (uint32, uint32, error) {
func GetUser(ctx context.Context, root, username string) (uint32, uint32, []uint32, error) {
// fast path from uid/gid
if uid, gid, err := ParseUser(username); err == nil {
return uid, gid, nil
if uid, gid, err := ParseUIDGID(username); err == nil {
return uid, gid, nil, nil
}
passwdPath, err := user.GetPasswdPath()
if err != nil {
return 0, 0, err
return 0, 0, nil, err
}
groupPath, err := user.GetGroupPath()
if err != nil {
return 0, 0, err
return 0, 0, nil, err
}
passwdFile, err := openUserFile(root, passwdPath)
if err == nil {
@@ -35,33 +39,29 @@ func GetUser(ctx context.Context, root, username string) (uint32, uint32, error)
execUser, err := user.GetExecUser(username, nil, passwdFile, groupFile)
if err != nil {
return 0, 0, err
return 0, 0, nil, err
}
return uint32(execUser.Uid), uint32(execUser.Gid), nil
var sgids []uint32
for _, g := range execUser.Sgids {
sgids = append(sgids, uint32(g))
}
return uint32(execUser.Uid), uint32(execUser.Gid), sgids, nil
}
func ParseUser(str string) (uid uint32, gid uint32, err error) {
// ParseUIDGID takes the fast path to parse UID and GID if and only if they are both provided
func ParseUIDGID(str string) (uid uint32, gid uint32, err error) {
if str == "" {
return 0, 0, nil
}
parts := strings.SplitN(str, ":", 2)
for i, v := range parts {
switch i {
case 0:
uid, err = parseUID(v)
if err != nil {
return 0, 0, err
}
if len(parts) == 1 {
gid = uid
}
case 1:
gid, err = parseUID(v)
if err != nil {
return 0, 0, err
}
}
if len(parts) == 1 {
return 0, 0, errors.New("groups ID is not provided")
}
if uid, err = parseUID(parts[0]); err != nil {
return 0, 0, err
}
if gid, err = parseUID(parts[1]); err != nil {
return 0, 0, err
}
return
}
@@ -84,3 +84,24 @@ func parseUID(str string) (uint32, error) {
}
return uint32(uid), nil
}
// WithUIDGID allows the UID and GID for the Process to be set
// FIXME: This is a temporeray fix for the missing supplementary GIDs from containerd
// once the PR in containerd is merged we should remove this function.
func WithUIDGID(uid, gid uint32, sgids []uint32) containerdoci.SpecOpts {
return func(_ context.Context, _ containerdoci.Client, _ *containers.Container, s *containerdoci.Spec) error {
setProcess(s)
s.Process.User.UID = uid
s.Process.User.GID = gid
s.Process.User.AdditionalGids = sgids
return nil
}
}
// setProcess sets Process to empty if unset
// FIXME: Same on this one. Need to be removed after containerd fix merged
func setProcess(s *containerdoci.Spec) {
if s.Process == nil {
s.Process = &specs.Process{}
}
}
components/engine/vendor/github.com/moby/buildkit/executor/runcexecutor/executor.go
Generated Vendored
+3 -5
View File
@@ -133,7 +133,7 @@ func (w *runcExecutor) Exec(ctx context.Context, meta executor.Meta, root cache.
}
defer mount.Unmount(rootFSPath, 0)
uid, gid, err := oci.GetUser(ctx, rootFSPath, meta.User)
uid, gid, sgids, err := oci.GetUser(ctx, rootFSPath, meta.User)
if err != nil {
return err
}
@@ -143,7 +143,7 @@ func (w *runcExecutor) Exec(ctx context.Context, meta executor.Meta, root cache.
return err
}
defer f.Close()
opts := []containerdoci.SpecOpts{containerdoci.WithUIDGID(uid, gid)}
opts := []containerdoci.SpecOpts{oci.WithUIDGID(uid, gid, sgids)}
if system.SeccompSupported() {
opts = append(opts, seccomp.WithDefaultProfile())
}
@@ -170,9 +170,7 @@ func (w *runcExecutor) Exec(ctx context.Context, meta executor.Meta, root cache.
}
if w.rootless {
specconv.ToRootless(spec, &specconv.RootlessOpts{
MapSubUIDGID: true,
})
specconv.ToRootless(spec, nil)
// TODO(AkihiroSuda): keep Cgroups enabled if /sys/fs/cgroup/cpuset/buildkit exists and writable
spec.Linux.CgroupsPath = ""
// TODO(AkihiroSuda): ToRootless removes netns, but we should readd netns here
components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert.go
Generated Vendored
+81 -46
View File
@@ -91,8 +91,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
metaResolver = imagemetaresolver.Default()
}
var allDispatchStates []*dispatchState
dispatchStatesByName := map[string]*dispatchState{}
allDispatchStates := newDispatchStates()
// set base state for every image
for _, st := range stages {
@@ -100,6 +99,9 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
if err != nil {
return nil, nil, err
}
if name == "" {
return nil, nil, errors.Errorf("base name (%s) should not be blank", st.BaseName)
}
st.BaseName = name
ds := &dispatchState{
@@ -121,13 +123,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
ds.platform = &p
}
if d, ok := dispatchStatesByName[st.BaseName]; ok {
ds.base = d
}
allDispatchStates = append(allDispatchStates, ds)
if st.Name != "" {
dispatchStatesByName[strings.ToLower(st.Name)] = ds
}
allDispatchStates.addState(ds)
if opt.IgnoreCache != nil {
if len(opt.IgnoreCache) == 0 {
ds.ignoreCache = true
@@ -143,20 +139,20 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
var target *dispatchState
if opt.Target == "" {
target = allDispatchStates[len(allDispatchStates)-1]
target = allDispatchStates.lastTarget()
} else {
var ok bool
target, ok = dispatchStatesByName[strings.ToLower(opt.Target)]
target, ok = allDispatchStates.findStateByName(opt.Target)
if !ok {
return nil, nil, errors.Errorf("target stage %s could not be found", opt.Target)
}
}
// fill dependencies to stages so unreachable ones can avoid loading image configs
for _, d := range allDispatchStates {
for _, d := range allDispatchStates.states {
d.commands = make([]command, len(d.stage.Commands))
for i, cmd := range d.stage.Commands {
newCmd, err := toCommand(cmd, dispatchStatesByName, allDispatchStates)
newCmd, err := toCommand(cmd, allDispatchStates)
if err != nil {
return nil, nil, err
}
@@ -165,7 +161,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
if src != nil {
d.deps[src] = struct{}{}
if src.unregistered {
allDispatchStates = append(allDispatchStates, src)
allDispatchStates.addState(src)
}
}
}
@@ -173,7 +169,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
}
eg, ctx := errgroup.WithContext(ctx)
for i, d := range allDispatchStates {
for i, d := range allDispatchStates.states {
reachable := isReachable(target, d)
// resolve image config for every stage
if d.base == nil {
@@ -239,7 +235,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
buildContext := &mutableOutput{}
ctxPaths := map[string]struct{}{}
for _, d := range allDispatchStates {
for _, d := range allDispatchStates.states {
if !isReachable(target, d) {
continue
}
@@ -271,17 +267,16 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
}
opt := dispatchOpt{
allDispatchStates: allDispatchStates,
dispatchStatesByName: dispatchStatesByName,
metaArgs: metaArgs,
buildArgValues: opt.BuildArgs,
shlex: shlex,
sessionID: opt.SessionID,
buildContext: llb.NewState(buildContext),
proxyEnv: proxyEnv,
cacheIDNamespace: opt.CacheIDNamespace,
buildPlatforms: opt.BuildPlatforms,
targetPlatform: *opt.TargetPlatform,
allDispatchStates: allDispatchStates,
metaArgs: metaArgs,
buildArgValues: opt.BuildArgs,
shlex: shlex,
sessionID: opt.SessionID,
buildContext: llb.NewState(buildContext),
proxyEnv: proxyEnv,
cacheIDNamespace: opt.CacheIDNamespace,
buildPlatforms: opt.BuildPlatforms,
targetPlatform: *opt.TargetPlatform,
}
if err = dispatchOnBuild(d, d.image.Config.OnBuild, opt); err != nil {
@@ -330,14 +325,14 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
return &st, &target.image, nil
}
func toCommand(ic instructions.Command, dispatchStatesByName map[string]*dispatchState, allDispatchStates []*dispatchState) (command, error) {
func toCommand(ic instructions.Command, allDispatchStates *dispatchStates) (command, error) {
cmd := command{Command: ic}
if c, ok := ic.(*instructions.CopyCommand); ok {
if c.From != "" {
var stn *dispatchState
index, err := strconv.Atoi(c.From)
if err != nil {
stn, ok = dispatchStatesByName[strings.ToLower(c.From)]
stn, ok = allDispatchStates.findStateByName(c.From)
if !ok {
stn = &dispatchState{
stage: instructions.Stage{BaseName: c.From},
@@ -346,16 +341,16 @@ func toCommand(ic instructions.Command, dispatchStatesByName map[string]*dispatc
}
}
} else {
if index < 0 || index >= len(allDispatchStates) {
return command{}, errors.Errorf("invalid stage index %d", index)
stn, err = allDispatchStates.findStateByIndex(index)
if err != nil {
return command{}, err
}
stn = allDispatchStates[index]
}
cmd.sources = []*dispatchState{stn}
}
}
if ok := detectRunMount(&cmd, dispatchStatesByName, allDispatchStates); ok {
if ok := detectRunMount(&cmd, allDispatchStates); ok {
return cmd, nil
}
@@ -363,17 +358,16 @@ func toCommand(ic instructions.Command, dispatchStatesByName map[string]*dispatc
}
type dispatchOpt struct {
allDispatchStates []*dispatchState
dispatchStatesByName map[string]*dispatchState
metaArgs []instructions.ArgCommand
buildArgValues map[string]string
shlex *shell.Lex
sessionID string
buildContext llb.State
proxyEnv *llb.ProxyEnv
cacheIDNamespace string
targetPlatform specs.Platform
buildPlatforms []specs.Platform
allDispatchStates *dispatchStates
metaArgs []instructions.ArgCommand
buildArgValues map[string]string
shlex *shell.Lex
sessionID string
buildContext llb.State
proxyEnv *llb.ProxyEnv
cacheIDNamespace string
targetPlatform specs.Platform
buildPlatforms []specs.Platform
}
func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
@@ -456,6 +450,43 @@ type dispatchState struct {
unregistered bool
}
type dispatchStates struct {
states []*dispatchState
statesByName map[string]*dispatchState
}
func newDispatchStates() *dispatchStates {
return &dispatchStates{statesByName: map[string]*dispatchState{}}
}
func (dss *dispatchStates) addState(ds *dispatchState) {
dss.states = append(dss.states, ds)
if d, ok := dss.statesByName[ds.stage.BaseName]; ok {
ds.base = d
}
if ds.stage.Name != "" {
dss.statesByName[strings.ToLower(ds.stage.Name)] = ds
}
}
func (dss *dispatchStates) findStateByName(name string) (*dispatchState, bool) {
ds, ok := dss.statesByName[strings.ToLower(name)]
return ds, ok
}
func (dss *dispatchStates) findStateByIndex(index int) (*dispatchState, error) {
if index < 0 || index >= len(dss.states) {
return nil, errors.Errorf("invalid stage index %d", index)
}
return dss.states[index], nil
}
func (dss *dispatchStates) lastTarget() *dispatchState {
return dss.states[len(dss.states)-1]
}
type command struct {
instructions.Command
sources []*dispatchState
@@ -474,7 +505,7 @@ func dispatchOnBuild(d *dispatchState, triggers []string, opt dispatchOpt) error
if err != nil {
return err
}
cmd, err := toCommand(ic, opt.dispatchStatesByName, opt.allDispatchStates)
cmd, err := toCommand(ic, opt.allDispatchStates)
if err != nil {
return err
}
@@ -570,7 +601,11 @@ func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState l
for i, src := range c.Sources() {
commitMessage.WriteString(" " + src)
if isAddCommand && (strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://")) {
if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
if !isAddCommand {
return errors.New("source can't be a URL for COPY")
}
// Resources from remote URLs are not decompressed.
// https://docs.docker.com/engine/reference/builder/#add
//
components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_norunmount.go
Generated Vendored
+1 -1
View File
@@ -7,7 +7,7 @@ import (
"github.com/moby/buildkit/frontend/dockerfile/instructions"
)
func detectRunMount(cmd *command, dispatchStatesByName map[string]*dispatchState, allDispatchStates []*dispatchState) bool {
func detectRunMount(cmd *command, allDispatchStates *dispatchStates) bool {
return false
}
components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_runmount.go
Generated Vendored
+2 -3
View File
@@ -5,14 +5,13 @@ package dockerfile2llb
import (
"path"
"path/filepath"
"strings"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/frontend/dockerfile/instructions"
"github.com/pkg/errors"
)
func detectRunMount(cmd *command, dispatchStatesByName map[string]*dispatchState, allDispatchStates []*dispatchState) bool {
func detectRunMount(cmd *command, allDispatchStates *dispatchStates) bool {
if c, ok := cmd.Command.(*instructions.RunCommand); ok {
mounts := instructions.GetMounts(c)
sources := make([]*dispatchState, len(mounts))
@@ -24,7 +23,7 @@ func detectRunMount(cmd *command, dispatchStatesByName map[string]*dispatchState
if from == "" || mount.Type == instructions.MountTypeTmpfs {
continue
}
stn, ok := dispatchStatesByName[strings.ToLower(from)]
stn, ok := allDispatchStates.findStateByName(from)
if !ok {
stn = &dispatchState{
stage: instructions.Stage{BaseName: from},
components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/bflag.go
Generated Vendored
+1 -1
View File
@@ -72,7 +72,7 @@ func (bf *BFlags) AddString(name string, def string) *Flag {
return flag
}
// AddString adds a string flag to BFlags that can match multiple values
// AddStrings adds a string flag to BFlags that can match multiple values
func (bf *BFlags) AddStrings(name string) *Flag {
flag := bf.addFlag(name, stringsType)
if flag == nil {
components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands.go
Generated Vendored
+1 -1
View File
@@ -159,7 +159,7 @@ func (s SourcesAndDest) Dest() string {
// AddCommand : ADD foo /path
//
// Add the file 'foo' to '/path'. Tarball and Remote URL (git, http) handling
// Add the file 'foo' to '/path'. Tarball and Remote URL (http, https) handling
// exist here. If you do not wish to have this automatic handling, use COPY.
//
type AddCommand struct {
components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/shell/equal_env_unix.go
Generated Vendored
+3 -2
View File
@@ -2,8 +2,9 @@
package shell
// EqualEnvKeys compare two strings and returns true if they are equal. On
// Windows this comparison is case insensitive.
// EqualEnvKeys compare two strings and returns true if they are equal.
// On Unix this comparison is case sensitive.
// On Windows this comparison is case insensitive.
func EqualEnvKeys(from, to string) bool {
return from == to
}
components/engine/vendor/github.com/moby/buildkit/frontend/dockerfile/shell/equal_env_windows.go
Generated Vendored
+3 -2
View File
@@ -2,8 +2,9 @@ package shell
import "strings"
// EqualEnvKeys compare two strings and returns true if they are equal. On
// Windows this comparison is case insensitive.
// EqualEnvKeys compare two strings and returns true if they are equal.
// On Unix this comparison is case sensitive.
// On Windows this comparison is case insensitive.
func EqualEnvKeys(from, to string) bool {
return strings.ToUpper(from) == strings.ToUpper(to)
}
components/engine/vendor/github.com/moby/buildkit/util/imageutil/config.go
Generated Vendored
+5 -1
View File
@@ -141,13 +141,17 @@ func DetectManifestMediaType(ra content.ReaderAt) (string, error) {
}
var mfst struct {
Config json.RawMessage `json:"config"`
MediaType string `json:"mediaType"`
Config json.RawMessage `json:"config"`
}
if err := json.Unmarshal(p, &mfst); err != nil {
return "", err
}
if mfst.MediaType != "" {
return mfst.MediaType, nil
}
if mfst.Config != nil {
return images.MediaTypeDockerSchema2Manifest, nil
}