diff --git a/components/engine/CONTRIBUTING.md b/components/engine/CONTRIBUTING.md index abf6c45a3c..917214cd1b 100644 --- a/components/engine/CONTRIBUTING.md +++ b/components/engine/CONTRIBUTING.md @@ -39,7 +39,7 @@ A great way to contribute to the project is to send a detailed report when you encounter an issue. We always appreciate a well-written, thorough bug report, and will thank you for it! -Check that [our issue database](https://github.com/docker/docker/issues) +Check that [our issue database](https://github.com/moby/moby/issues) doesn't already include that problem or suggestion before submitting an issue. If you find a match, you can use the "subscribe" button to get notified on updates. Do *not* leave random "+1" or "I have this too" comments, as they @@ -66,7 +66,7 @@ This section gives the experienced contributor some tips and guidelines. Not sure if that typo is worth a pull request? Found a bug and know how to fix it? Do it! We will appreciate it. Any significant improvement should be -documented as [a GitHub issue](https://github.com/docker/docker/issues) before +documented as [a GitHub issue](https://github.com/moby/moby/issues) before anybody starts working on it. We are always thrilled to receive pull requests. We do our best to process them diff --git a/components/engine/Makefile b/components/engine/Makefile index 526d6405ea..0d99606cc2 100644 --- a/components/engine/Makefile +++ b/components/engine/Makefile @@ -1,4 +1,4 @@ -.PHONY: all binary build cross deb help init-go-pkg-cache install manpages rpm run shell test test-docker-py test-integration-cli test-unit tgz validate win +.PHONY: all binary dynbinary build cross deb help init-go-pkg-cache install manpages rpm run shell test test-docker-py test-integration-cli test-unit tgz validate win # set the graph driver as the current graphdriver if not set DOCKER_GRAPHDRIVER := $(if $(DOCKER_GRAPHDRIVER),$(DOCKER_GRAPHDRIVER),$(shell docker info 2>&1 | grep "Storage Driver" | sed 's/.*: //')) @@ -106,6 +106,9 @@ all: build ## validate all checks, build linux binaries, run all tests\ncross bu binary: build ## build the linux binaries $(DOCKER_RUN_DOCKER) hack/make.sh binary +dynbinary: build ## build the linux dynbinaries + $(DOCKER_RUN_DOCKER) hack/make.sh dynbinary + build: bundles init-go-pkg-cache $(warning The docker client CLI has moved to github.com/docker/cli. By default, it is built from the git sha specified in hack/dockerfile/binaries-commits. For a dev-test cycle involving the CLI, run:${\n} DOCKER_CLI_PATH=/host/path/to/cli/binary make shell ${\n} then change the cli and compile into a binary at the same location.${\n}) docker build ${BUILD_APT_MIRROR} ${DOCKER_BUILD_ARGS} -t "$(DOCKER_IMAGE)" -f "$(DOCKERFILE)" . diff --git a/components/engine/api/server/backend/build/backend.go b/components/engine/api/server/backend/build/backend.go index fe98c087b8..b5a758c387 100644 --- a/components/engine/api/server/backend/build/backend.go +++ b/components/engine/api/server/backend/build/backend.go @@ -55,6 +55,11 @@ func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string if imageID, err = squashBuild(build, b.imageComponent); err != nil { return "", err } + if config.ProgressWriter.AuxFormatter != nil { + if err = config.ProgressWriter.AuxFormatter.Emit(types.BuildResult{ID: imageID}); err != nil { + return "", err + } + } } stdout := config.ProgressWriter.StdoutFormatter diff --git a/components/engine/api/server/backend/build/tag.go b/components/engine/api/server/backend/build/tag.go index e2de4c13bb..7bd5dcdeb2 100644 --- a/components/engine/api/server/backend/build/tag.go +++ b/components/engine/api/server/backend/build/tag.go @@ -37,7 +37,7 @@ func (bt *Tagger) TagImages(imageID image.ID) error { for _, rt := range bt.repoAndTags { // TODO @jhowardmsft LCOW support. Will need revisiting. platform := runtime.GOOS - if platform == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { platform = "linux" } if err := bt.imageComponent.TagImageWithReference(imageID, platform, rt); err != nil { diff --git a/components/engine/api/server/router/image/image_routes.go b/components/engine/api/server/router/image/image_routes.go index 232d795683..9b99a585f3 100644 --- a/components/engine/api/server/router/image/image_routes.go +++ b/components/engine/api/server/router/image/image_routes.go @@ -101,7 +101,7 @@ func (s *imageRouter) postImagesCreate(ctx context.Context, w http.ResponseWrite // } // valid := []string{runtime.GOOS} // - // if runtime.GOOS == "windows" && system.LCOWSupported() { + // if system.LCOWSupported() { // valid = append(valid, "linux") // } // @@ -118,7 +118,7 @@ func (s *imageRouter) postImagesCreate(ctx context.Context, w http.ResponseWrite // return err // } platform := runtime.GOOS - if platform == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { platform = "linux" } diff --git a/components/engine/builder/dockerfile/copy.go b/components/engine/builder/dockerfile/copy.go index 0026bdc158..c7db943f5c 100644 --- a/components/engine/builder/dockerfile/copy.go +++ b/components/engine/builder/dockerfile/copy.go @@ -32,9 +32,10 @@ type pathCache interface { // copyInfo is a data object which stores the metadata about each source file in // a copyInstruction type copyInfo struct { - root string - path string - hash string + root string + path string + hash string + noDecompress bool } func (c copyInfo) fullPath() (string, error) { @@ -124,7 +125,9 @@ func (o *copier) getCopyInfoForSourcePath(orig string) ([]copyInfo, error) { o.tmpPaths = append(o.tmpPaths, remote.Root()) hash, err := remote.Hash(path) - return newCopyInfos(newCopyInfoFromSource(remote, path, hash)), err + ci := newCopyInfoFromSource(remote, path, hash) + ci.noDecompress = true // data from http shouldn't be extracted even on ADD + return newCopyInfos(ci), err } // Cleanup removes any temporary directories created as part of downloading @@ -387,7 +390,7 @@ func performCopyForInfo(dest copyInfo, source copyInfo, options copyFileOptions) if src.IsDir() { return copyDirectory(archiver, srcPath, destPath) } - if options.decompress && archive.IsArchivePath(srcPath) { + if options.decompress && archive.IsArchivePath(srcPath) && !source.noDecompress { return archiver.UntarPath(srcPath, destPath) } diff --git a/components/engine/builder/dockerfile/evaluator.go b/components/engine/builder/dockerfile/evaluator.go index 69e3f6d0f2..ba4315940a 100644 --- a/components/engine/builder/dockerfile/evaluator.go +++ b/components/engine/builder/dockerfile/evaluator.go @@ -233,7 +233,7 @@ func (s *dispatchState) beginStage(stageName string, image builder.Image) { func (s *dispatchState) setDefaultPath() { // TODO @jhowardmsft LCOW Support - This will need revisiting later platform := runtime.GOOS - if platform == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { platform = "linux" } if system.DefaultPathEnv(platform) == "" { diff --git a/components/engine/builder/dockerfile/parser/parser.go b/components/engine/builder/dockerfile/parser/parser.go index 9766fbfdfe..7f07ff2150 100644 --- a/components/engine/builder/dockerfile/parser/parser.go +++ b/components/engine/builder/dockerfile/parser/parser.go @@ -91,8 +91,8 @@ var ( // DefaultEscapeToken is the default escape token const DefaultEscapeToken = '\\' -// DefaultPlatformToken is the platform assumed for the build if not explicitly provided -var DefaultPlatformToken = runtime.GOOS +// defaultPlatformToken is the platform assumed for the build if not explicitly provided +var defaultPlatformToken = runtime.GOOS // Directive is the structure used during a build run to hold the state of // parsing directives. @@ -119,7 +119,7 @@ func (d *Directive) setEscapeToken(s string) error { func (d *Directive) setPlatformToken(s string) error { s = strings.ToLower(s) valid := []string{runtime.GOOS} - if runtime.GOOS == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { valid = append(valid, "linux") } for _, item := range valid { @@ -154,7 +154,7 @@ func (d *Directive) possibleParserDirective(line string) error { // TODO @jhowardmsft LCOW Support: Eventually this check can be removed, // but only recognise a platform token if running in LCOW mode. - if runtime.GOOS == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { tpcMatch := tokenPlatformCommand.FindStringSubmatch(strings.ToLower(line)) if len(tpcMatch) != 0 { for i, n := range tokenPlatformCommand.SubexpNames() { @@ -177,7 +177,7 @@ func (d *Directive) possibleParserDirective(line string) error { func NewDefaultDirective() *Directive { directive := Directive{} directive.setEscapeToken(string(DefaultEscapeToken)) - directive.setPlatformToken(runtime.GOOS) + directive.setPlatformToken(defaultPlatformToken) return &directive } diff --git a/components/engine/builder/remotecontext/detect.go b/components/engine/builder/remotecontext/detect.go index 22ad5b7c9f..4345736805 100644 --- a/components/engine/builder/remotecontext/detect.go +++ b/components/engine/builder/remotecontext/detect.go @@ -131,6 +131,7 @@ func removeDockerfile(c modifiableContext, filesToRemove ...string) error { } excludes, err := dockerignore.ReadAll(f) if err != nil { + f.Close() return err } f.Close() diff --git a/components/engine/client/client.go b/components/engine/client/client.go index 222aa50f11..62a3db360a 100644 --- a/components/engine/client/client.go +++ b/components/engine/client/client.go @@ -260,8 +260,13 @@ func (cli *Client) NegotiateAPIVersionPing(p types.Ping) { p.APIVersion = "1.24" } - // if server version is lower than the current cli, downgrade - if versions.LessThan(p.APIVersion, cli.ClientVersion()) { + // if the client is not initialized with a version, start with the latest supported version + if cli.version == "" { + cli.version = api.DefaultVersion + } + + // if server version is lower than the maximum version supported by the Client, downgrade + if versions.LessThan(p.APIVersion, api.DefaultVersion) { cli.version = p.APIVersion } } diff --git a/components/engine/client/ping.go b/components/engine/client/ping.go index 631ed02005..a4c2e2c4dd 100644 --- a/components/engine/client/ping.go +++ b/components/engine/client/ping.go @@ -18,13 +18,15 @@ func (cli *Client) Ping(ctx context.Context) (types.Ping, error) { } defer ensureReaderClosed(serverResp) - ping.APIVersion = serverResp.header.Get("API-Version") + if serverResp.header != nil { + ping.APIVersion = serverResp.header.Get("API-Version") - if serverResp.header.Get("Docker-Experimental") == "true" { - ping.Experimental = true + if serverResp.header.Get("Docker-Experimental") == "true" { + ping.Experimental = true + } + ping.OSType = serverResp.header.Get("OSType") } - ping.OSType = serverResp.header.Get("OSType") - - return ping, nil + err = cli.checkResponseErr(serverResp) + return ping, err } diff --git a/components/engine/client/ping_test.go b/components/engine/client/ping_test.go new file mode 100644 index 0000000000..7a4a1a9024 --- /dev/null +++ b/components/engine/client/ping_test.go @@ -0,0 +1,82 @@ +package client + +import ( + "errors" + "io/ioutil" + "net/http" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "golang.org/x/net/context" +) + +// TestPingFail tests that when a server sends a non-successful response that we +// can still grab API details, when set. +// Some of this is just excercising the code paths to make sure there are no +// panics. +func TestPingFail(t *testing.T) { + var withHeader bool + client := &Client{ + client: newMockClient(func(req *http.Request) (*http.Response, error) { + resp := &http.Response{StatusCode: http.StatusInternalServerError} + if withHeader { + resp.Header = http.Header{} + resp.Header.Set("API-Version", "awesome") + resp.Header.Set("Docker-Experimental", "true") + } + resp.Body = ioutil.NopCloser(strings.NewReader("some error with the server")) + return resp, nil + }), + } + + ping, err := client.Ping(context.Background()) + assert.Error(t, err) + assert.Equal(t, false, ping.Experimental) + assert.Equal(t, "", ping.APIVersion) + + withHeader = true + ping2, err := client.Ping(context.Background()) + assert.Error(t, err) + assert.Equal(t, true, ping2.Experimental) + assert.Equal(t, "awesome", ping2.APIVersion) +} + +// TestPingWithError tests the case where there is a protocol error in the ping. +// This test is mostly just testing that there are no panics in this code path. +func TestPingWithError(t *testing.T) { + client := &Client{ + client: newMockClient(func(req *http.Request) (*http.Response, error) { + resp := &http.Response{StatusCode: http.StatusInternalServerError} + resp.Header = http.Header{} + resp.Header.Set("API-Version", "awesome") + resp.Header.Set("Docker-Experimental", "true") + resp.Body = ioutil.NopCloser(strings.NewReader("some error with the server")) + return resp, errors.New("some error") + }), + } + + ping, err := client.Ping(context.Background()) + assert.Error(t, err) + assert.Equal(t, false, ping.Experimental) + assert.Equal(t, "", ping.APIVersion) +} + +// TestPingSuccess tests that we are able to get the expected API headers/ping +// details on success. +func TestPingSuccess(t *testing.T) { + client := &Client{ + client: newMockClient(func(req *http.Request) (*http.Response, error) { + resp := &http.Response{StatusCode: http.StatusInternalServerError} + resp.Header = http.Header{} + resp.Header.Set("API-Version", "awesome") + resp.Header.Set("Docker-Experimental", "true") + resp.Body = ioutil.NopCloser(strings.NewReader("some error with the server")) + return resp, nil + }), + } + ping, err := client.Ping(context.Background()) + assert.Error(t, err) + assert.Equal(t, true, ping.Experimental) + assert.Equal(t, "awesome", ping.APIVersion) +} diff --git a/components/engine/client/request.go b/components/engine/client/request.go index 6457b316a3..3e7d43feac 100644 --- a/components/engine/client/request.go +++ b/components/engine/client/request.go @@ -24,6 +24,7 @@ type serverResponse struct { body io.ReadCloser header http.Header statusCode int + reqURL *url.URL } // head sends an http request to the docker API using the method HEAD. @@ -118,11 +119,18 @@ func (cli *Client) sendRequest(ctx context.Context, method, path string, query u if err != nil { return serverResponse{}, err } - return cli.doRequest(ctx, req) + resp, err := cli.doRequest(ctx, req) + if err != nil { + return resp, err + } + if err := cli.checkResponseErr(resp); err != nil { + return resp, err + } + return resp, nil } func (cli *Client) doRequest(ctx context.Context, req *http.Request) (serverResponse, error) { - serverResp := serverResponse{statusCode: -1} + serverResp := serverResponse{statusCode: -1, reqURL: req.URL} resp, err := ctxhttp.Do(ctx, cli.client, req) if err != nil { @@ -179,37 +187,44 @@ func (cli *Client) doRequest(ctx context.Context, req *http.Request) (serverResp if resp != nil { serverResp.statusCode = resp.StatusCode + serverResp.body = resp.Body + serverResp.header = resp.Header } - - if serverResp.statusCode < 200 || serverResp.statusCode >= 400 { - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return serverResp, err - } - if len(body) == 0 { - return serverResp, fmt.Errorf("Error: request returned %s for API route and version %s, check if the server supports the requested API version", http.StatusText(serverResp.statusCode), req.URL) - } - - var errorMessage string - if (cli.version == "" || versions.GreaterThan(cli.version, "1.23")) && - resp.Header.Get("Content-Type") == "application/json" { - var errorResponse types.ErrorResponse - if err := json.Unmarshal(body, &errorResponse); err != nil { - return serverResp, fmt.Errorf("Error reading JSON: %v", err) - } - errorMessage = errorResponse.Message - } else { - errorMessage = string(body) - } - - return serverResp, fmt.Errorf("Error response from daemon: %s", strings.TrimSpace(errorMessage)) - } - - serverResp.body = resp.Body - serverResp.header = resp.Header return serverResp, nil } +func (cli *Client) checkResponseErr(serverResp serverResponse) error { + if serverResp.statusCode >= 200 && serverResp.statusCode < 400 { + return nil + } + + body, err := ioutil.ReadAll(serverResp.body) + if err != nil { + return err + } + if len(body) == 0 { + return fmt.Errorf("Error: request returned %s for API route and version %s, check if the server supports the requested API version", http.StatusText(serverResp.statusCode), serverResp.reqURL) + } + + var ct string + if serverResp.header != nil { + ct = serverResp.header.Get("Content-Type") + } + + var errorMessage string + if (cli.version == "" || versions.GreaterThan(cli.version, "1.23")) && ct == "application/json" { + var errorResponse types.ErrorResponse + if err := json.Unmarshal(body, &errorResponse); err != nil { + return fmt.Errorf("Error reading JSON: %v", err) + } + errorMessage = errorResponse.Message + } else { + errorMessage = string(body) + } + + return fmt.Errorf("Error response from daemon: %s", strings.TrimSpace(errorMessage)) +} + func (cli *Client) addHeaders(req *http.Request, headers headers) *http.Request { // Add CLI Config's HTTP Headers BEFORE we set the Docker headers // then the user can't change OUR headers @@ -239,9 +254,9 @@ func encodeData(data interface{}) (*bytes.Buffer, error) { } func ensureReaderClosed(response serverResponse) { - if body := response.body; body != nil { + if response.body != nil { // Drain up to 512 bytes and close the body to let the Transport reuse the connection - io.CopyN(ioutil.Discard, body, 512) + io.CopyN(ioutil.Discard, response.body, 512) response.body.Close() } } diff --git a/components/engine/client/service_create.go b/components/engine/client/service_create.go index 71a7583e86..c2fc2776a8 100644 --- a/components/engine/client/service_create.go +++ b/components/engine/client/service_create.go @@ -39,7 +39,7 @@ func (cli *Client) ServiceCreate(ctx context.Context, service swarm.ServiceSpec, service.TaskTemplate.ContainerSpec.Image = img } // add platforms that are compatible with the service - service.TaskTemplate.Placement = updateServicePlatforms(service.TaskTemplate.Placement, distributionInspect) + service.TaskTemplate.Placement = setServicePlatforms(service.TaskTemplate.Placement, distributionInspect) } } var response types.ServiceCreateResponse @@ -86,13 +86,15 @@ func imageWithTagString(image string) string { return "" } -// updateServicePlatforms updates the Platforms in swarm.Placement to list -// all compatible platforms for the service, as found in distributionInspect -// and returns a pointer to the new or updated swarm.Placement struct -func updateServicePlatforms(placement *swarm.Placement, distributionInspect registrytypes.DistributionInspect) *swarm.Placement { +// setServicePlatforms sets Platforms in swarm.Placement to list all +// compatible platforms for the service, as found in distributionInspect +// and returns a pointer to the new or updated swarm.Placement struct. +func setServicePlatforms(placement *swarm.Placement, distributionInspect registrytypes.DistributionInspect) *swarm.Placement { if placement == nil { placement = &swarm.Placement{} } + // reset any existing listed platforms + placement.Platforms = []swarm.Platform{} for _, p := range distributionInspect.Platforms { placement.Platforms = append(placement.Platforms, swarm.Platform{ Architecture: p.Architecture, diff --git a/components/engine/client/service_update.go b/components/engine/client/service_update.go index 412790b5dd..a72adc997c 100644 --- a/components/engine/client/service_update.go +++ b/components/engine/client/service_update.go @@ -51,7 +51,7 @@ func (cli *Client) ServiceUpdate(ctx context.Context, serviceID string, version service.TaskTemplate.ContainerSpec.Image = img } // add platforms that are compatible with the service - service.TaskTemplate.Placement = updateServicePlatforms(service.TaskTemplate.Placement, distributionInspect) + service.TaskTemplate.Placement = setServicePlatforms(service.TaskTemplate.Placement, distributionInspect) } } diff --git a/components/engine/container/container.go b/components/engine/container/container.go index fe050bcd17..854850f4fb 100644 --- a/components/engine/container/container.go +++ b/components/engine/container/container.go @@ -1038,7 +1038,7 @@ func (container *Container) CreateDaemonEnvironment(tty bool, linkedEnv []string platform = runtime.GOOS } env := []string{} - if runtime.GOOS != "windows" || (runtime.GOOS == "windows" && system.LCOWSupported() && platform == "linux") { + if runtime.GOOS != "windows" || (system.LCOWSupported() && platform == "linux") { env = []string{ "PATH=" + system.DefaultPathEnv(platform), "HOSTNAME=" + container.Config.Hostname, @@ -1052,7 +1052,6 @@ func (container *Container) CreateDaemonEnvironment(tty bool, linkedEnv []string // because the env on the container can override certain default values // we need to replace the 'env' keys where they match and append anything // else. - //return ReplaceOrAppendEnvValues(linkedEnv, container.Config.Env) - foo := ReplaceOrAppendEnvValues(env, container.Config.Env) - return foo + env = ReplaceOrAppendEnvValues(env, container.Config.Env) + return env } diff --git a/components/engine/container/view.go b/components/engine/container/view.go index 13c7161852..f605b4f483 100644 --- a/components/engine/container/view.go +++ b/components/engine/container/view.go @@ -73,6 +73,17 @@ type memDB struct { store *memdb.MemDB } +// NoSuchContainerError indicates that the container wasn't found in the +// database. +type NoSuchContainerError struct { + id string +} + +// Error satisfies the error interface. +func (e NoSuchContainerError) Error() string { + return "no such container " + e.id +} + // NewViewDB provides the default implementation, with the default schema func NewViewDB() (ViewDB, error) { store, err := memdb.NewMemDB(schema) @@ -134,6 +145,9 @@ func (v *memdbView) Get(id string) (*Snapshot, error) { if err != nil { return nil, err } + if s == nil { + return nil, NoSuchContainerError{id: id} + } return v.transform(s.(*Container)), nil } diff --git a/components/engine/daemon/cluster/executor/container/adapter.go b/components/engine/daemon/cluster/executor/container/adapter.go index 75a085ef1d..92e4947e65 100644 --- a/components/engine/daemon/cluster/executor/container/adapter.go +++ b/components/engine/daemon/cluster/executor/container/adapter.go @@ -93,7 +93,7 @@ func (c *containerAdapter) pullImage(ctx context.Context) error { // TODO @jhowardmsft LCOW Support: This will need revisiting as // the stack is built up to include LCOW support for swarm. platform := runtime.GOOS - if platform == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { platform = "linux" } err := c.backend.PullImage(ctx, c.container.image(), "", platform, metaHeaders, authConfig, pw) diff --git a/components/engine/daemon/create.go b/components/engine/daemon/create.go index d47de31fa8..78070fd29d 100644 --- a/components/engine/daemon/create.go +++ b/components/engine/daemon/create.go @@ -82,7 +82,7 @@ func (daemon *Daemon) create(params types.ContainerCreateConfig, managed bool) ( if params.Platform == "" { params.Platform = runtime.GOOS } - if params.Platform == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { params.Platform = "linux" } @@ -106,7 +106,7 @@ func (daemon *Daemon) create(params types.ContainerCreateConfig, managed bool) ( if img != nil { if params.Platform != img.Platform() { // Ignore this in LCOW mode. @jhowardmsft TODO - This will need revisiting later. - if !(runtime.GOOS == "windows" && system.LCOWSupported()) { + if !system.LCOWSupported() { return nil, fmt.Errorf("cannot create a %s container from a %s image", params.Platform, img.Platform()) } } diff --git a/components/engine/daemon/graphdriver/lcow/lcow.go b/components/engine/daemon/graphdriver/lcow/lcow.go index 6bf491574a..079252ecb8 100644 --- a/components/engine/daemon/graphdriver/lcow/lcow.go +++ b/components/engine/daemon/graphdriver/lcow/lcow.go @@ -16,6 +16,7 @@ import ( "sync" "time" + "github.com/Microsoft/hcsshim" "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/graphdriver" "github.com/docker/docker/pkg/archive" @@ -32,6 +33,9 @@ func init() { const ( // sandboxFilename is the name of the file containing a layers sandbox (read-write layer) sandboxFilename = "sandbox.vhdx" + + // svmScratchFilename is the name of the scratch-space used by an SVM to avoid running out of memory + svmScratchFilename = "scratch.vhdx" ) // cacheType is our internal structure representing an item in our local cache @@ -51,8 +55,13 @@ type Driver struct { cachedSandboxFile string // options are the graphdriver options we are initialised with options []string - // JJH LIFETIME TODO - Remove this and move up to daemon. For now, a global service utility-VM + // config is the representation of the SVM. + // @jhowardmsft LIFETIME TODO - For now, a global service utility-VM config client.Config + // svmScratchSpaceFile is a host location for a dedicated scratch space + // that the SVM utilities can use as a scratch-space to avoid OOMs + // @jhowardmsft LIFETIME TODO - For now, a global service utility-VM + svmScratchSpaceFile string // it is safe for windows to use a cache here because it does not support // restoring containers when the daemon dies. @@ -69,10 +78,11 @@ func InitLCOW(home string, options []string, uidMaps, gidMaps []idtools.IDMap) ( logrus.Debugf("%s %s", title, home) d := &Driver{ - homeDir: home, - options: options, - cachedSandboxFile: filepath.Join(home, "cache", sandboxFilename), - cache: make(map[string]cacheType), + homeDir: home, + options: options, + cachedSandboxFile: filepath.Join(home, "cache", sandboxFilename), + svmScratchSpaceFile: filepath.Join(home, "svmscratch", svmScratchFilename), + cache: make(map[string]cacheType), } if err := idtools.MkdirAllAs(home, 0700, 0, 0); err != nil { @@ -84,6 +94,11 @@ func InitLCOW(home string, options []string, uidMaps, gidMaps []idtools.IDMap) ( return nil, fmt.Errorf("%s failed to create '%s': %v", title, home, err) } + // Location for the SVM scratch + if err := idtools.MkdirAllAs(filepath.Dir(d.svmScratchSpaceFile), 0700, 0, 0); err != nil { + return nil, fmt.Errorf("%s failed to create '%s': %v", title, home, err) + } + return d, nil } @@ -92,6 +107,8 @@ func InitLCOW(home string, options []string, uidMaps, gidMaps []idtools.IDMap) ( // service VM globally to a service VM per container (or offline operation). However, // for the initial bring-up of LCOW, this is acceptable. func (d *Driver) startUvm(context string) error { + const toolsScratchPath = "/mnt/gcs/LinuxServiceVM/scratch" + // Nothing to do if it's already running if d.config.Uvm != nil { return nil @@ -102,10 +119,36 @@ func (d *Driver) startUvm(context string) error { return fmt.Errorf("failed to generate default gogcs configuration (%s): %s", context, err) } + scratchAttached := false + if _, err := os.Stat(d.svmScratchSpaceFile); err == nil { + // We have a scratch space already, so just attach it as a mapped virtual disk + logrus.Debugf("lcowdriver: startuvm: (%s) attaching pre-existing scratch", context) + mvd := hcsshim.MappedVirtualDisk{ + HostPath: d.svmScratchSpaceFile, + ContainerPath: toolsScratchPath, + CreateInUtilityVM: true, + } + d.config.MappedVirtualDisks = append(d.config.MappedVirtualDisks, mvd) + scratchAttached = true + } + d.config.Name = "LinuxServiceVM" // TODO @jhowardmsft - This requires an in-flight platform change. Can't hard code it to this longer term if err := d.config.Create(); err != nil { return fmt.Errorf("failed to start utility VM (%s): %s", context, err) } + + // Hot-add the scratch-space if not already attached + if !scratchAttached { + logrus.Debugf("lcowdriver: startuvm: (%s) creating an SVM scratch", context) + if err := d.config.CreateSandbox(d.svmScratchSpaceFile, client.DefaultSandboxSizeMB, d.cachedSandboxFile); err != nil { + return fmt.Errorf("failed to create SVM scratch VHDX (%s): %s", context, err) + } + logrus.Debugf("lcowdriver: startuvm: (%s) hot-adding an SVM scratch", context) + if err := d.config.HotAddVhd(d.svmScratchSpaceFile, toolsScratchPath); err != nil { + return fmt.Errorf("failed to hot-add %s failed: %s", d.svmScratchSpaceFile, err) + } + } + logrus.Debugf("lcowdriver: startuvm: (%s) successful", context) return nil } @@ -118,7 +161,7 @@ func (d *Driver) terminateUvm(context string) error { // FIXME: @jhowardmsft // This isn't thread-safe yet, but will change anyway with the lifetime - // changes and multiple instances. Defering that work for now. + // changes and multiple instances. Deferring that work for now. uvm := d.config.Uvm d.config.Uvm = nil @@ -190,7 +233,6 @@ func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error { // Make sure layers are created with the correct ACL so that VMs can access them. layerPath := d.dir(id) logrus.Debugf("lcowdriver: create: id %s: creating layerPath %s", id, layerPath) - // Make sure the layers are created with the correct ACL so that VMs can access them. if err := system.MkdirAllWithACL(layerPath, 755, system.SddlNtvmAdministratorsLocalSystem); err != nil { return err } diff --git a/components/engine/daemon/graphdriver/overlay2/overlay.go b/components/engine/daemon/graphdriver/overlay2/overlay.go index ccb2c3669d..deea3e0176 100644 --- a/components/engine/daemon/graphdriver/overlay2/overlay.go +++ b/components/engine/daemon/graphdriver/overlay2/overlay.go @@ -194,6 +194,7 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap ctr: graphdriver.NewRefCounter(graphdriver.NewFsChecker(graphdriver.FsMagicOverlay)), supportsDType: supportsDType, locker: locker.New(), + options: *opts, } d.naiveDiff = graphdriver.NewNaiveDiffDriver(d, uidMaps, gidMaps) @@ -202,7 +203,12 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap // Try to enable project quota support over xfs. if d.quotaCtl, err = quota.NewControl(home); err == nil { projectQuotaSupported = true + } else if opts.quota.Size > 0 { + return nil, fmt.Errorf("Storage option overlay2.size not supported. Filesystem does not support Project Quota: %v", err) } + } else if opts.quota.Size > 0 { + // if xfs is not the backing fs then error out if the storage-opt overlay2.size is used. + return nil, fmt.Errorf("Storage Option overlay2.size only supported for backingFS XFS. Found %v", backingFs) } logrus.Debugf("backingFs=%s, projectQuotaSupported=%v", backingFs, projectQuotaSupported) @@ -224,9 +230,14 @@ func parseOptions(options []string) (*overlayOptions, error) { if err != nil { return nil, err } - + case "overlay2.size": + size, err := units.RAMInBytes(val) + if err != nil { + return nil, err + } + o.quota.Size = uint64(size) default: - return nil, fmt.Errorf("overlay2: Unknown option %s\n", key) + return nil, fmt.Errorf("overlay2: unknown option %s", key) } } return o, nil @@ -312,17 +323,38 @@ func (d *Driver) Cleanup() error { // CreateReadWrite creates a layer that is writable for use as a container // file system. func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error { - return d.Create(id, parent, opts) + if opts != nil && len(opts.StorageOpt) != 0 && !projectQuotaSupported { + return fmt.Errorf("--storage-opt is supported only for overlay over xfs with 'pquota' mount option") + } + + if opts == nil { + opts = &graphdriver.CreateOpts{ + StorageOpt: map[string]string{}, + } + } + + if _, ok := opts.StorageOpt["size"]; !ok { + if opts.StorageOpt == nil { + opts.StorageOpt = map[string]string{} + } + opts.StorageOpt["size"] = strconv.FormatUint(d.options.quota.Size, 10) + } + + return d.create(id, parent, opts) } // Create is used to create the upper, lower, and merge directories required for overlay fs for a given id. // The parent filesystem is used to configure these directories for the overlay. func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) (retErr error) { - - if opts != nil && len(opts.StorageOpt) != 0 && !projectQuotaSupported { - return fmt.Errorf("--storage-opt is supported only for overlay over xfs with 'pquota' mount option") + if opts != nil && len(opts.StorageOpt) != 0 { + if _, ok := opts.StorageOpt["size"]; ok { + return fmt.Errorf("--storage-opt size is only supported for ReadWrite Layers") + } } + return d.create(id, parent, opts) +} +func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts) (retErr error) { dir := d.dir(id) rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps) diff --git a/components/engine/daemon/graphdriver/windows/windows.go b/components/engine/daemon/graphdriver/windows/windows.go index a722078e35..514650aa7c 100644 --- a/components/engine/daemon/graphdriver/windows/windows.go +++ b/components/engine/daemon/graphdriver/windows/windows.go @@ -154,7 +154,7 @@ func (d *Driver) Status() [][2]string { } // panicIfUsedByLcow does exactly what it says. -// TODO @jhowardmsft - this is an temporary measure for the bring-up of +// TODO @jhowardmsft - this is a temporary measure for the bring-up of // Linux containers on Windows. It is a failsafe to ensure that the right // graphdriver is used. func panicIfUsedByLcow() { diff --git a/components/engine/daemon/image_exporter.go b/components/engine/daemon/image_exporter.go index edac775d43..a7b0be64c2 100644 --- a/components/engine/daemon/image_exporter.go +++ b/components/engine/daemon/image_exporter.go @@ -16,7 +16,7 @@ import ( func (daemon *Daemon) ExportImage(names []string, outStream io.Writer) error { // TODO @jhowardmsft LCOW. This will need revisiting later. platform := runtime.GOOS - if platform == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { platform = "linux" } imageExporter := tarexport.NewTarExporter(daemon.stores[platform].imageStore, daemon.stores[platform].layerStore, daemon.stores[platform].referenceStore, daemon) @@ -29,7 +29,7 @@ func (daemon *Daemon) ExportImage(names []string, outStream io.Writer) error { func (daemon *Daemon) LoadImage(inTar io.ReadCloser, outStream io.Writer, quiet bool) error { // TODO @jhowardmsft LCOW. This will need revisiting later. platform := runtime.GOOS - if platform == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { platform = "linux" } imageExporter := tarexport.NewTarExporter(daemon.stores[platform].imageStore, daemon.stores[platform].layerStore, daemon.stores[platform].referenceStore, daemon) diff --git a/components/engine/daemon/image_push.go b/components/engine/daemon/image_push.go index b1d15454fb..c2e5967b19 100644 --- a/components/engine/daemon/image_push.go +++ b/components/engine/daemon/image_push.go @@ -43,7 +43,7 @@ func (daemon *Daemon) PushImage(ctx context.Context, image, tag string, metaHead // TODO @jhowardmsft LCOW Support. This will require revisiting. For now, hard-code. platform := runtime.GOOS - if platform == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { platform = "linux" } diff --git a/components/engine/daemon/images.go b/components/engine/daemon/images.go index 6e29cae4d0..4baf703715 100644 --- a/components/engine/daemon/images.go +++ b/components/engine/daemon/images.go @@ -38,7 +38,7 @@ func (r byCreated) Less(i, j int) bool { return r[i].Created < r[j].Created } func (daemon *Daemon) Map() map[image.ID]*image.Image { // TODO @jhowardmsft LCOW. This will need work to enumerate the stores for all platforms. platform := runtime.GOOS - if platform == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { platform = "linux" } return daemon.stores[platform].imageStore.Map() @@ -53,7 +53,7 @@ func (daemon *Daemon) Images(imageFilters filters.Args, all bool, withExtraAttrs // TODO @jhowardmsft LCOW. This will need work to enumerate the stores for all platforms. platform := runtime.GOOS - if platform == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { platform = "linux" } diff --git a/components/engine/daemon/info.go b/components/engine/daemon/info.go index a11775c2e9..2cb3e84798 100644 --- a/components/engine/daemon/info.go +++ b/components/engine/daemon/info.go @@ -90,7 +90,7 @@ func (daemon *Daemon) SystemInfo() (*types.Info, error) { // TODO @jhowardmsft LCOW support. For now, hard-code the platform shown for the driver status p := runtime.GOOS - if p == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { p = "linux" } diff --git a/components/engine/daemon/list.go b/components/engine/daemon/list.go index 870a7bda2d..b854be7549 100644 --- a/components/engine/daemon/list.go +++ b/components/engine/daemon/list.go @@ -162,11 +162,13 @@ func (daemon *Daemon) filterByNameIDMatches(view container.View, ctx *listContex cntrs := make([]container.Snapshot, 0, len(matches)) for id := range matches { c, err := view.Get(id) - if err != nil { - return nil, err - } - if c != nil { + switch err.(type) { + case nil: cntrs = append(cntrs, *c) + case container.NoSuchContainerError: + // ignore error + default: + return nil, err } } diff --git a/components/engine/daemon/logger/loggerutils/rotatefilewriter.go b/components/engine/daemon/logger/loggerutils/rotatefilewriter.go index ecb461fcec..457a39b5a3 100644 --- a/components/engine/daemon/logger/loggerutils/rotatefilewriter.go +++ b/components/engine/daemon/logger/loggerutils/rotatefilewriter.go @@ -74,7 +74,7 @@ func (w *RotateFileWriter) checkCapacityAndRotate() error { if err := rotate(name, w.maxFiles); err != nil { return err } - file, err := os.OpenFile(name, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 06400) + file, err := os.OpenFile(name, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0640) if err != nil { return err } diff --git a/components/engine/daemon/prune.go b/components/engine/daemon/prune.go index 561ac1e01e..1d8686b36f 100644 --- a/components/engine/daemon/prune.go +++ b/components/engine/daemon/prune.go @@ -161,7 +161,7 @@ func (daemon *Daemon) VolumesPrune(ctx context.Context, pruneFilters filters.Arg func (daemon *Daemon) ImagesPrune(ctx context.Context, pruneFilters filters.Args) (*types.ImagesPruneReport, error) { // TODO @jhowardmsft LCOW Support: This will need revisiting later. platform := runtime.GOOS - if platform == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { platform = "linux" } diff --git a/components/engine/distribution/pull_v2.go b/components/engine/distribution/pull_v2.go index 0f3f31cf52..50257f5cb4 100644 --- a/components/engine/distribution/pull_v2.go +++ b/components/engine/distribution/pull_v2.go @@ -491,7 +491,7 @@ func (p *v2Puller) pullSchema1(ctx context.Context, ref reference.Named, unverif // the history does, but unfortunately that's a string, so search through // all the history until hopefully we find one which indicates the os. platform := runtime.GOOS - if runtime.GOOS == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { type config struct { Os string `json:"os,omitempty"` } diff --git a/components/engine/hack/make/test-unit b/components/engine/hack/make/test-unit index d3f182c094..85eef5b5b6 100644 --- a/components/engine/hack/make/test-unit +++ b/components/engine/hack/make/test-unit @@ -26,6 +26,8 @@ bundle_test_unit() { TEST_PATH=./${TESTDIRS} fi + source "${MAKEDIR}/.go-autogen" + if [ "$(go env GOHOSTOS)" = 'solaris' ]; then pkg_list=$(go list -e \ -f '{{if ne .Name "github.com/docker/docker"}} diff --git a/components/engine/image/store.go b/components/engine/image/store.go index b215d977f8..c85f8d6830 100644 --- a/components/engine/image/store.go +++ b/components/engine/image/store.go @@ -3,7 +3,6 @@ package image import ( "encoding/json" "fmt" - "runtime" "strings" "sync" "time" @@ -119,8 +118,9 @@ func (is *store) Create(config []byte) (ID, error) { return "", err } + // TODO @jhowardmsft - LCOW Support. This will need revisiting. // Integrity check - ensure we are creating something for the correct platform - if runtime.GOOS == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { if strings.ToLower(img.Platform()) != strings.ToLower(is.platform) { return "", fmt.Errorf("cannot create entry for platform %q in image store for platform %q", img.Platform(), is.platform) } diff --git a/components/engine/image/tarexport/load.go b/components/engine/image/tarexport/load.go index adf4574da8..af8cefc6ad 100644 --- a/components/engine/image/tarexport/load.go +++ b/components/engine/image/tarexport/load.go @@ -79,6 +79,9 @@ func (l *tarexporter) Load(inTar io.ReadCloser, outStream io.Writer, quiet bool) if err != nil { return err } + if err := checkCompatibleOS(img.OS); err != nil { + return err + } var rootFS image.RootFS rootFS = *img.RootFS rootFS.DiffIDs = nil @@ -292,11 +295,18 @@ func (l *tarexporter) legacyLoadImage(oldID, sourceDir string, loadedMap map[str return err } - var img struct{ Parent string } + var img struct { + OS string + Parent string + } if err := json.Unmarshal(imageJSON, &img); err != nil { return err } + if err := checkCompatibleOS(img.OS); err != nil { + return err + } + var parentID image.ID if img.Parent != "" { for { @@ -402,3 +412,20 @@ func checkValidParent(img, parent *image.Image) bool { } return true } + +func checkCompatibleOS(os string) error { + // TODO @jhowardmsft LCOW - revisit for simultaneous platforms + platform := runtime.GOOS + if system.LCOWSupported() { + platform = "linux" + } + // always compatible if the OS matches; also match an empty OS + if os == platform || os == "" { + return nil + } + // for compatibility, only fail if the image or runtime OS is Windows + if os == "windows" || platform == "windows" { + return fmt.Errorf("cannot load %s image on %s", os, platform) + } + return nil +} diff --git a/components/engine/integration-cli/docker_api_build_test.go b/components/engine/integration-cli/docker_api_build_test.go index a5e19f658b..c1ab7661e0 100644 --- a/components/engine/integration-cli/docker_api_build_test.go +++ b/components/engine/integration-cli/docker_api_build_test.go @@ -366,6 +366,50 @@ func (s *DockerRegistrySuite) TestBuildCopyFromForcePull(c *check.C) { assert.Contains(c, string(out), "Successfully built") } +func (s *DockerSuite) TestBuildAddRemoteNoDecompress(c *check.C) { + buffer := new(bytes.Buffer) + tw := tar.NewWriter(buffer) + dt := []byte("contents") + err := tw.WriteHeader(&tar.Header{ + Name: "foo", + Size: int64(len(dt)), + Mode: 0600, + Typeflag: tar.TypeReg, + }) + require.NoError(c, err) + _, err = tw.Write(dt) + require.NoError(c, err) + err = tw.Close() + require.NoError(c, err) + + server := fakestorage.New(c, "", fakecontext.WithBinaryFiles(map[string]*bytes.Buffer{ + "test.tar": buffer, + })) + defer server.Close() + + dockerfile := fmt.Sprintf(` + FROM busybox + ADD %s/test.tar / + RUN [ -f test.tar ] + `, server.URL()) + + ctx := fakecontext.New(c, "", + fakecontext.WithDockerfile(dockerfile), + ) + defer ctx.Close() + + res, body, err := request.Post( + "/build", + request.RawContent(ctx.AsTarReader(c)), + request.ContentType("application/x-tar")) + require.NoError(c, err) + assert.Equal(c, http.StatusOK, res.StatusCode) + + out, err := testutil.ReadBody(body) + require.NoError(c, err) + assert.Contains(c, string(out), "Successfully built") +} + func (s *DockerSuite) TestBuildWithSession(c *check.C) { testRequires(c, ExperimentalDaemon) diff --git a/components/engine/integration-cli/docker_cli_build_test.go b/components/engine/integration-cli/docker_cli_build_test.go index b9a5dff789..77b151e89f 100644 --- a/components/engine/integration-cli/docker_cli_build_test.go +++ b/components/engine/integration-cli/docker_cli_build_test.go @@ -6459,3 +6459,32 @@ func (s *DockerSuite) TestBuildIidFileCleanupOnFail(c *check.C) { c.Assert(err, check.NotNil) c.Assert(os.IsNotExist(err), check.Equals, true) } + +func (s *DockerSuite) TestBuildIidFileSquash(c *check.C) { + testRequires(c, ExperimentalDaemon) + tmpDir, err := ioutil.TempDir("", "TestBuildIidFileSquash") + if err != nil { + c.Fatal(err) + } + defer os.RemoveAll(tmpDir) + tmpIidFile := filepath.Join(tmpDir, "iidsquash") + + name := "testbuildiidfilesquash" + // Use a Dockerfile with multiple stages to ensure we get the last one + cli.BuildCmd(c, name, + // This could be minimalBaseImage except + // https://github.com/moby/moby/issues/33823 requires + // `touch` to workaround. + build.WithDockerfile(`FROM busybox +ENV FOO FOO +ENV BAR BAR +RUN touch /foop +`), + cli.WithFlags("--iidfile", tmpIidFile, "--squash")) + + id, err := ioutil.ReadFile(tmpIidFile) + c.Assert(err, check.IsNil) + d, err := digest.Parse(string(id)) + c.Assert(err, check.IsNil) + c.Assert(d.String(), checker.Equals, getIDByName(c, name)) +} diff --git a/components/engine/integration-cli/docker_cli_daemon_test.go b/components/engine/integration-cli/docker_cli_daemon_test.go index 9046fe6ea6..b551ad4509 100644 --- a/components/engine/integration-cli/docker_cli_daemon_test.go +++ b/components/engine/integration-cli/docker_cli_daemon_test.go @@ -2828,7 +2828,7 @@ func (s *DockerDaemonSuite) TestExecWithUserAfterLiveRestore(c *check.C) { out2, err := s.d.Cmd("exec", "-u", "test", "top", "id") c.Assert(err, check.IsNil, check.Commentf("Output: %s", out2)) - c.Assert(out1, check.Equals, out2, check.Commentf("Output: before restart '%s', after restart '%s'", out1, out2)) + c.Assert(out2, check.Equals, out1, check.Commentf("Output: before restart '%s', after restart '%s'", out1, out2)) out, err = s.d.Cmd("stop", "top") c.Assert(err, check.IsNil, check.Commentf("Output: %s", out)) diff --git a/components/engine/layer/layer_store.go b/components/engine/layer/layer_store.go index e7c424aa92..75ac1e4f48 100644 --- a/components/engine/layer/layer_store.go +++ b/components/engine/layer/layer_store.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "io/ioutil" - "runtime" "strings" "sync" @@ -273,7 +272,7 @@ func (ls *layerStore) registerWithDescriptor(ts io.Reader, parent ChainID, platf var p *roLayer // Integrity check - ensure we are creating something for the correct platform - if runtime.GOOS == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { if strings.ToLower(ls.platform) != strings.ToLower(string(platform)) { return nil, fmt.Errorf("cannot create entry for platform %q in layer store for platform %q", platform, ls.platform) } diff --git a/components/engine/plugin/manager.go b/components/engine/plugin/manager.go index aa090f5a90..02d372741f 100644 --- a/components/engine/plugin/manager.go +++ b/components/engine/plugin/manager.go @@ -353,7 +353,7 @@ func isEqualPrivilege(a, b types.PluginPrivilege) bool { func configToRootFS(c []byte) (*image.RootFS, layer.Platform, error) { // TODO @jhowardmsft LCOW - Will need to revisit this. For now, calculate the platform. platform := layer.Platform(runtime.GOOS) - if platform == "windows" && system.LCOWSupported() { + if system.LCOWSupported() { platform = "linux" } var pluginConfig types.PluginConfig diff --git a/components/engine/reports/2017-06-26.md b/components/engine/reports/2017-06-26.md new file mode 100644 index 0000000000..e12533ae46 --- /dev/null +++ b/components/engine/reports/2017-06-26.md @@ -0,0 +1,120 @@ +# Development Report for June 26, 2017 + +## Moby Summit + +The Moby Summit held in San Francisco was very active and well attended ([blog](http://mobyproject.org/blog/2017/06/26/moby-summit-recap/) / [linuxkit table notes](https://github.com/linuxkit/linuxkit/blob/master/reports/2017-06-19-summit.md) [#2090](https://github.com/linuxkit/linuxkit/pull/2090) [#2033](https://github.com/linuxkit/linuxkit/pull/2033) [@mgoelzer] [@justincormack]). + +## Container Engine + +Thanks to @fabiokung there is no container locks anymore on `docker ps` [#31273](https://github.com/moby/moby/pull/31273) + +## BuildKit + +[Repo](https://github.com/moby/buildkit) +[Proposal](https://github.com/moby/moby/issues/32925) + +New development repo is open at https://github.com/moby/buildkit + +The readme file provides examples how to get started. You can see an example of building BuildKit with BuildKit. + +There are lots of new issues opened as well to track the missing functionality. You are welcomed to help on any of them or discuss the design there. + +Last week most of the work was done on improving the `llb` client library for more complicated use cases and providing traces and interactive progress of executed build jobs. + +The `llb` client package is a go library that helps you to generate the build definition graph. It uses chained methods to make it easy to describe what steps need to be running. Mounts can be added to the execution steps for defining multiple inputs or outputs. To prepare the graph, you just have to call `Marshal()` on a leaf node that will generate the protobuf definition for everything required to build that node. + +### Typed Dockerfile parsing + +[PR](https://github.com/moby/moby/pull/33492) + +This PR that enables parsing Dockerfiles into typed structures so they can be preprocessed to eliminate unnecessary build stages and reused with different kinds of dispatchers(eg. BuildKit). + +The PR had some review and updates in last week. Should be ready to code review soon. + +### Merged: Long running session & incremental file sending + +[PR](https://github.com/moby/moby/pull/32677) + +Incremental context sending PR was merged and is expected to land in `v17.07`. + +This feature experimental feature lets you skip sending the build context to the daemon on repeated builder invocations during development. Currently, this feature requires a CLI flag `--stream=true`. If this flag is used, one first builder invocation full build context is sent to the daemon. On a second attempt, only the changed files are transferred. + +Previous build context is saved in the build cache, and you can see how much space it takes form `docker system df`. Build cache will be automatically garbage collected and can also be manually cleared with `docker prune`. + +### Quality: Dependency interface switch + +[Move file copying from the daemon to the builder](https://github.com/moby/moby/pull/33454) PR was merged. + + +### Proposals for new Dockerfile features that need design feedback: + +[Add IMPORT/EXPORT commands to Dockerfile](https://github.com/moby/moby/issues/32100) + +[Add `DOCKEROS/DOCKERARCH` default ARG to Dockerfile](https://github.com/moby/moby/issues/32487) + +[Add support for `RUN --mount`](https://github.com/moby/moby/issues/32507) + +[DAG image builder](https://github.com/moby/moby/issues/32550) + +[Option to export the hash of the build context](https://github.com/moby/moby/issues/32963) (new) + +[Allow --cache-from=*](https://github.com/moby/moby/issues/33002#issuecomment-299041162) (new) + +[Provide advanced .dockeringore use-cases](https://github.com/moby/moby/issues/12886) [2](https://github.com/moby/moby/issues/12886#issuecomment-306247989) + +If you are interested in implementing any of them, leave a comment on the specific issues. + +### Other builder PRs merged last week + +[Warn/deprecate continuing on empty lines in `Dockerfile`](https://github.com/moby/moby/pull/29161) + +[Fix behavior of absolute paths in .dockerignore](https://github.com/moby/moby/pull/32088) + +[fix copy —from conflict with force pull](https://github.com/moby/moby/pull/33735) + +### Builder features currently in code-review: + +[Fix handling of remote "git@" notation](https://github.com/moby/moby/pull/33696) + +[builder: Emit a BuildResult after squashing.](https://github.com/moby/moby/pull/33824) + +[Fix shallow git clone in docker-build](https://github.com/moby/moby/pull/33704) + +### Backlog + +[Build secrets](https://github.com/moby/moby/issues/33343) has not got much traction. If you want this feature to become a reality, please make yourself heard. + +## LinuxKit + +* **Kernel GPG verification:** The kernel compilation containers now verify the GPG and SHA256 + checksums before building the binaries. ([#2062](https://github.com/linuxkit/linuxkit/issues/2062) [#2083](https://github.com/linuxkit/linuxkit/issues/2083) [@mscribe] [@justincormack] [@rn] [@riyazdf]). + The base Alpine build image now includes `gnupg` to support this feature ([#2091](https://github.com/linuxkit/linuxkit/issues/2091) [@riyazdf] [@rn]). + +* **Security SIG on Landlock:** The third Moby Security SIG focussed on the [Landlock](https://github.com/landlock-lsm) security module that provides unprivileged fine-grained sandboxing to applications. There are videos and forum links ([#2087](https://github.com/linuxkit/linuxkit/issues/2087) [#2089](https://github.com/linuxkit/linuxkit/issues/2089) [#2073](https://github.com/linuxkit/linuxkit/issues/2073) [@riyazdf]). + +* **Networking drivers now modules:** The kernels have been updated to 4.11.6/4.9.33/4.4.73, and many drivers are now loaded as modules to speed up boot-time ([#2095](https://github.com/linuxkit/linuxkit/issues/2095) [#2061](https://github.com/linuxkit/linuxkit/issues/2061) [@rn] [@justincormack] [@tych0]) + +- **Whaley important update:** The ASCII logo was updated and we fondly wave goodbye to the waves. ([#2084](https://github.com/linuxkit/linuxkit/issues/2084) [@thaJeztah] [@rn]) + +- **Containerised getty and sshd:** The login services now run in their own mount namespace, which was confusing people since they were expecting it to be on the host filesystem. This is now being addressed via a reminder in the `motd` upon login ([#2078](https://github.com/linuxkit/linuxkit/issues/2078) [#2097](https://github.com/linuxkit/linuxkit/issues/2097) [@deitch] [@ijc] [@justincormack] [@riyazdf] [@rn]) + +- **Hardened user copying:** The RFC on ensuring that we use a hardened kernel/userspace copying system was closed, as it is enabled by default on all our modern kernels and a regression test is included by default ([#2086](https://github.com/linuxkit/linuxkit/issues/2086) [@fntlnz] [@riyazdf]). + +- **Vultr provider:** There is an ongoing effort to add a metadata provider for [Vultr](http://vultr.com) ([#2101](https://github.com/linuxkit/linuxkit/issues/2101) [@furious-luke] [@justincormack]). + +### Packages and Projects + +- Simplified Makefiles for packages ([#2080](https://github.com/linuxkit/linuxkit/issues/2080) [@justincormack] [@rn]) +- The MirageOS SDK is integrating many upstream changes from dependent libraries, for the DHCP client ([#2070](https://github.com/linuxkit/linuxkit/issues/2070) [#2072](https://github.com/linuxkit/linuxkit/issues/2072) [@samoht] [@talex5] [@avsm]). + +### Documentation and Tests + +- A comprehensive test suite for containerd is now integrated into LinuxKit tests ([#2062](https://github.com/linuxkit/linuxkit/issues/2062) [@AkihiroSuda] [@justincormack] [@rn]) +- Fix documentation links ([#2074](https://github.com/linuxkit/linuxkit/issues/2074) [@ndauten] [@justincormack]) +- Update RTF version ([#2077](https://github.com/linuxkit/linuxkit/issues/2077) [@justincormack]) +- tests: add build test for Docker for Mac blueprint ([#2093](https://github.com/linuxkit/linuxkit/issues/2093) [@riyazdf] [@MagnusS]) +- Disable Qemu EFI ISO test for now ([#2100](https://github.com/linuxkit/linuxkit/issues/2100) [@justincormack]) +- The CI whitelists and ACLs were updated ([linuxkit-ci#11](https://github.com/linuxkit/linuxkit-ce/issues/11) [linuxkit-ci#15](https://github.com/linuxkit/linuxkit-ce/issues/15) [linuxkit/linuxkit-ci#10](https://github.com/linuxkit/linuxkit-ce/issues/10) [@rn] [@justincormack]) +- Fix spelling errors ([#2079](https://github.com/linuxkit/linuxkit/issues/2079) [@ndauten]) +- Fix typo in dev report ([#2094](https://github.com/linuxkit/linuxkit/issues/2094) [@justincormack]) +- Fix dead Link to VMWare File ([#2082](https://github.com/linuxkit/linuxkit/issues/2082) [@davefreitag]) \ No newline at end of file diff --git a/components/engine/reports/builder/2017-06-26.md b/components/engine/reports/builder/2017-06-26.md new file mode 100644 index 0000000000..e0ba95a7a5 --- /dev/null +++ b/components/engine/reports/builder/2017-06-26.md @@ -0,0 +1,78 @@ +# Development Report for June 26, 2017 + + +### BuildKit + +[Repo](https://github.com/moby/buildkit) +[Proposal](https://github.com/moby/moby/issues/32925) + +New development repo is open at https://github.com/moby/buildkit + +The readme file provides examples how to get started. You can see an example of building BuildKit with BuildKit. + +There are lots of new issues opened as well to track the missing functionality. You are welcomed to help on any of them or discuss the design there. + +Last week most of the work was done on improving the `llb` client library for more complicated use cases and providing traces and interactive progress of executed build jobs. + +The `llb` client package is a go library that helps you to generate the build definition graph. It uses chained methods to make it easy to describe what steps need to be running. Mounts can be added to the execution steps for defining multiple inputs or outputs. To prepare the graph, you just have to call `Marshal()` on a leaf node that will generate the protobuf definition for everything required to build that node. + +### Typed Dockerfile parsing + +[PR](https://github.com/moby/moby/pull/33492) + +This PR that enables parsing Dockerfiles into typed structures so they can be preprocessed to eliminate unnecessary build stages and reused with different kinds of dispatchers(eg. BuildKit). + +The PR had some review and updates in last week. Should be ready to code review soon. + +### Merged: Long running session & incremental file sending + +[PR](https://github.com/moby/moby/pull/32677) + +Incremental context sending PR was merged and is expected to land in `v17.07`. + +This feature experimental feature lets you skip sending the build context to the daemon on repeated builder invocations during development. Currently, this feature requires a CLI flag `--stream=true`. If this flag is used, one first builder invocation full build context is sent to the daemon. On a second attempt, only the changed files are transferred. + +Previous build context is saved in the build cache, and you can see how much space it takes form `docker system df`. Build cache will be automatically garbage collected and can also be manually cleared with `docker prune`. + +### Quality: Dependency interface switch + +[Move file copying from the daemon to the builder](https://github.com/moby/moby/pull/33454) PR was merged. + + +### Proposals for new Dockerfile features that need design feedback: + +[Add IMPORT/EXPORT commands to Dockerfile](https://github.com/moby/moby/issues/32100) + +[Add `DOCKEROS/DOCKERARCH` default ARG to Dockerfile](https://github.com/moby/moby/issues/32487) + +[Add support for `RUN --mount`](https://github.com/moby/moby/issues/32507) + +[DAG image builder](https://github.com/moby/moby/issues/32550) + +[Option to export the hash of the build context](https://github.com/moby/moby/issues/32963) (new) + +[Allow --cache-from=*](https://github.com/moby/moby/issues/33002#issuecomment-299041162) (new) + +[Provide advanced .dockeringore use-cases](https://github.com/moby/moby/issues/12886) [2](https://github.com/moby/moby/issues/12886#issuecomment-306247989) + +If you are interested in implementing any of them, leave a comment on the specific issues. + +### Other builder PRs merged last week + +[Warn/deprecate continuing on empty lines in `Dockerfile`](https://github.com/moby/moby/pull/29161) + +[Fix behavior of absolute paths in .dockerignore](https://github.com/moby/moby/pull/32088) + +[fix copy —from conflict with force pull](https://github.com/moby/moby/pull/33735) + +### Builder features currently in code-review: + +[Fix handling of remote "git@" notation](https://github.com/moby/moby/pull/33696) + +[builder: Emit a BuildResult after squashing.](https://github.com/moby/moby/pull/33824) + +[Fix shallow git clone in docker-build](https://github.com/moby/moby/pull/33704) + +### Backlog + +[Build secrets](https://github.com/moby/moby/issues/33343) has not got much traction. If you want this feature to become a reality, please make yourself heard. \ No newline at end of file diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index bad511c4d4..20460d5cc9 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -13,7 +13,6 @@ github.com/kr/pty 5cf931ef8f github.com/mattn/go-shellwords v1.0.3 github.com/tchap/go-patricia v2.2.6 github.com/vdemeester/shakers 24d7f1d6a71aa5d9cbe7390e4afb66b7eef9e1b3 -# forked golang.org/x/net package includes a patch for lazy loading trace templates golang.org/x/net 7dcfb8076726a3fdd9353b6b8a1f1b6be6811bd6 golang.org/x/sys 8f0908ab3b2457e2e15403d3697c9ef5cb4b57a9 github.com/docker/go-units 9e638d38cf6977a37a8ea0078f3ee75a7cdb2dd1 diff --git a/components/engine/volume/volume.go b/components/engine/volume/volume.go index 2abfcc7b58..2de248aa5f 100644 --- a/components/engine/volume/volume.go +++ b/components/engine/volume/volume.go @@ -153,16 +153,18 @@ func (m *MountPoint) Cleanup() error { // before creating the source directory on the host. func (m *MountPoint) Setup(mountLabel string, rootIDs idtools.IDPair, checkFun func(m *MountPoint) error) (path string, err error) { defer func() { - if err == nil { - if label.RelabelNeeded(m.Mode) { - if err = label.Relabel(m.Source, mountLabel, label.IsShared(m.Mode)); err != nil { - path = "" - err = errors.Wrapf(err, "error setting label on mount source '%s'", m.Source) - return - } - } + if err != nil || !label.RelabelNeeded(m.Mode) { + return + } + + err = label.Relabel(m.Source, mountLabel, label.IsShared(m.Mode)) + if err == syscall.ENOTSUP { + err = nil + } + if err != nil { + path = "" + err = errors.Wrapf(err, "error setting label on mount source '%s'", m.Source) } - return }() if m.Volume != nil {