Merge component 'engine' from git@github.com:moby/moby master

components/engine/CONTRIBUTING.md

@@ -39,7 +39,7 @@ A great way to contribute to the project is to send a detailed report when you
 encounter an issue. We always appreciate a well-written, thorough bug report,
 and will thank you for it!
 
-Check that [our issue database](https://github.com/docker/docker/issues)
+Check that [our issue database](https://github.com/moby/moby/issues)
 doesn't already include that problem or suggestion before submitting an issue.
 If you find a match, you can use the "subscribe" button to get notified on
 updates. Do *not* leave random "+1" or "I have this too" comments, as they
@@ -66,7 +66,7 @@ This section gives the experienced contributor some tips and guidelines.
 
 Not sure if that typo is worth a pull request? Found a bug and know how to fix
 it? Do it! We will appreciate it. Any significant improvement should be
-documented as [a GitHub issue](https://github.com/docker/docker/issues) before
+documented as [a GitHub issue](https://github.com/moby/moby/issues) before
 anybody starts working on it.
 
 We are always thrilled to receive pull requests. We do our best to process them

components/engine/Makefile

@@ -1,4 +1,4 @@
-.PHONY: all binary build cross deb help init-go-pkg-cache install manpages rpm run shell test test-docker-py test-integration-cli test-unit tgz validate win
+.PHONY: all binary dynbinary build cross deb help init-go-pkg-cache install manpages rpm run shell test test-docker-py test-integration-cli test-unit tgz validate win
 
 # set the graph driver as the current graphdriver if not set
 DOCKER_GRAPHDRIVER := $(if $(DOCKER_GRAPHDRIVER),$(DOCKER_GRAPHDRIVER),$(shell docker info 2>&1 | grep "Storage Driver" | sed 's/.*: //'))
@@ -106,6 +106,9 @@ all: build ## validate all checks, build linux binaries, run all tests\ncross bu
 binary: build ## build the linux binaries
 	$(DOCKER_RUN_DOCKER) hack/make.sh binary
 
+dynbinary: build ## build the linux dynbinaries
+	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary
+
 build: bundles init-go-pkg-cache
 	$(warning The docker client CLI has moved to github.com/docker/cli. By default, it is built from the git sha specified in hack/dockerfile/binaries-commits. For a dev-test cycle involving the CLI, run:${\n} DOCKER_CLI_PATH=/host/path/to/cli/binary make shell ${\n} then change the cli and compile into a binary at the same location.${\n})
 	docker build ${BUILD_APT_MIRROR} ${DOCKER_BUILD_ARGS} -t "$(DOCKER_IMAGE)" -f "$(DOCKERFILE)" .

components/engine/api/server/backend/build/backend.go

@@ -55,6 +55,11 @@ func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string
 		if imageID, err = squashBuild(build, b.imageComponent); err != nil {
 			return "", err
 		}
+		if config.ProgressWriter.AuxFormatter != nil {
+			if err = config.ProgressWriter.AuxFormatter.Emit(types.BuildResult{ID: imageID}); err != nil {
+				return "", err
+			}
+		}
 	}
 
 	stdout := config.ProgressWriter.StdoutFormatter

components/engine/api/server/backend/build/tag.go

@@ -37,7 +37,7 @@ func (bt *Tagger) TagImages(imageID image.ID) error {
 	for _, rt := range bt.repoAndTags {
 		// TODO @jhowardmsft LCOW support. Will need revisiting.
 		platform := runtime.GOOS
-		if platform == "windows" && system.LCOWSupported() {
+		if system.LCOWSupported() {
 			platform = "linux"
 		}
 		if err := bt.imageComponent.TagImageWithReference(imageID, platform, rt); err != nil {

components/engine/api/server/router/image/image_routes.go

@@ -101,7 +101,7 @@ func (s *imageRouter) postImagesCreate(ctx context.Context, w http.ResponseWrite
 	//	}
 	//	valid := []string{runtime.GOOS}
 	//
-	//	if runtime.GOOS == "windows" && system.LCOWSupported() {
+	//	if system.LCOWSupported() {
 	//		valid = append(valid, "linux")
 	//	}
 	//
@@ -118,7 +118,7 @@ func (s *imageRouter) postImagesCreate(ctx context.Context, w http.ResponseWrite
 	//		return err
 	//	}
 	platform := runtime.GOOS
-	if platform == "windows" && system.LCOWSupported() {
+	if system.LCOWSupported() {
 		platform = "linux"
 	}
 

components/engine/builder/dockerfile/copy.go

@@ -32,9 +32,10 @@ type pathCache interface {
 // copyInfo is a data object which stores the metadata about each source file in
 // a copyInstruction
 type copyInfo struct {
-	root string
-	path string
-	hash string
+	root         string
+	path         string
+	hash         string
+	noDecompress bool
 }
 
 func (c copyInfo) fullPath() (string, error) {
@@ -124,7 +125,9 @@ func (o *copier) getCopyInfoForSourcePath(orig string) ([]copyInfo, error) {
 	o.tmpPaths = append(o.tmpPaths, remote.Root())
 
 	hash, err := remote.Hash(path)
-	return newCopyInfos(newCopyInfoFromSource(remote, path, hash)), err
+	ci := newCopyInfoFromSource(remote, path, hash)
+	ci.noDecompress = true // data from http shouldn't be extracted even on ADD
+	return newCopyInfos(ci), err
 }
 
 // Cleanup removes any temporary directories created as part of downloading
@@ -387,7 +390,7 @@ func performCopyForInfo(dest copyInfo, source copyInfo, options copyFileOptions)
 	if src.IsDir() {
 		return copyDirectory(archiver, srcPath, destPath)
 	}
-	if options.decompress && archive.IsArchivePath(srcPath) {
+	if options.decompress && archive.IsArchivePath(srcPath) && !source.noDecompress {
 		return archiver.UntarPath(srcPath, destPath)
 	}
 

components/engine/builder/dockerfile/evaluator.go

@@ -233,7 +233,7 @@ func (s *dispatchState) beginStage(stageName string, image builder.Image) {
 func (s *dispatchState) setDefaultPath() {
 	// TODO @jhowardmsft LCOW Support - This will need revisiting later
 	platform := runtime.GOOS
-	if platform == "windows" && system.LCOWSupported() {
+	if system.LCOWSupported() {
 		platform = "linux"
 	}
 	if system.DefaultPathEnv(platform) == "" {

components/engine/builder/dockerfile/parser/parser.go

@@ -91,8 +91,8 @@ var (
 // DefaultEscapeToken is the default escape token
 const DefaultEscapeToken = '\\'
 
-// DefaultPlatformToken is the platform assumed for the build if not explicitly provided
-var DefaultPlatformToken = runtime.GOOS
+// defaultPlatformToken is the platform assumed for the build if not explicitly provided
+var defaultPlatformToken = runtime.GOOS
 
 // Directive is the structure used during a build run to hold the state of
 // parsing directives.
@@ -119,7 +119,7 @@ func (d *Directive) setEscapeToken(s string) error {
 func (d *Directive) setPlatformToken(s string) error {
 	s = strings.ToLower(s)
 	valid := []string{runtime.GOOS}
-	if runtime.GOOS == "windows" && system.LCOWSupported() {
+	if system.LCOWSupported() {
 		valid = append(valid, "linux")
 	}
 	for _, item := range valid {
@@ -154,7 +154,7 @@ func (d *Directive) possibleParserDirective(line string) error {
 
 	// TODO @jhowardmsft LCOW Support: Eventually this check can be removed,
 	// but only recognise a platform token if running in LCOW mode.
-	if runtime.GOOS == "windows" && system.LCOWSupported() {
+	if system.LCOWSupported() {
 		tpcMatch := tokenPlatformCommand.FindStringSubmatch(strings.ToLower(line))
 		if len(tpcMatch) != 0 {
 			for i, n := range tokenPlatformCommand.SubexpNames() {
@@ -177,7 +177,7 @@ func (d *Directive) possibleParserDirective(line string) error {
 func NewDefaultDirective() *Directive {
 	directive := Directive{}
 	directive.setEscapeToken(string(DefaultEscapeToken))
-	directive.setPlatformToken(runtime.GOOS)
+	directive.setPlatformToken(defaultPlatformToken)
 	return &directive
 }
 

components/engine/builder/remotecontext/detect.go

@@ -131,6 +131,7 @@ func removeDockerfile(c modifiableContext, filesToRemove ...string) error {
 	}
 	excludes, err := dockerignore.ReadAll(f)
 	if err != nil {
+		f.Close()
 		return err
 	}
 	f.Close()

components/engine/client/client.go

@@ -260,8 +260,13 @@ func (cli *Client) NegotiateAPIVersionPing(p types.Ping) {
 		p.APIVersion = "1.24"
 	}
 
-	// if server version is lower than the current cli, downgrade
-	if versions.LessThan(p.APIVersion, cli.ClientVersion()) {
+	// if the client is not initialized with a version, start with the latest supported version
+	if cli.version == "" {
+		cli.version = api.DefaultVersion
+	}
+
+	// if server version is lower than the maximum version supported by the Client, downgrade
+	if versions.LessThan(p.APIVersion, api.DefaultVersion) {
 		cli.version = p.APIVersion
 	}
 }

components/engine/client/ping.go

@@ -18,13 +18,15 @@ func (cli *Client) Ping(ctx context.Context) (types.Ping, error) {
 	}
 	defer ensureReaderClosed(serverResp)
 
-	ping.APIVersion = serverResp.header.Get("API-Version")
+	if serverResp.header != nil {
+		ping.APIVersion = serverResp.header.Get("API-Version")
 
-	if serverResp.header.Get("Docker-Experimental") == "true" {
-		ping.Experimental = true
+		if serverResp.header.Get("Docker-Experimental") == "true" {
+			ping.Experimental = true
+		}
+		ping.OSType = serverResp.header.Get("OSType")
 	}
 
-	ping.OSType = serverResp.header.Get("OSType")
-
-	return ping, nil
+	err = cli.checkResponseErr(serverResp)
+	return ping, err
 }

components/engine/client/ping_test.go

@@ -0,0 +1,82 @@
+package client
+
+import (
+	"errors"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/net/context"
+)
+
+// TestPingFail tests that when a server sends a non-successful response that we
+// can still grab API details, when set.
+// Some of this is just excercising the code paths to make sure there are no
+// panics.
+func TestPingFail(t *testing.T) {
+	var withHeader bool
+	client := &Client{
+		client: newMockClient(func(req *http.Request) (*http.Response, error) {
+			resp := &http.Response{StatusCode: http.StatusInternalServerError}
+			if withHeader {
+				resp.Header = http.Header{}
+				resp.Header.Set("API-Version", "awesome")
+				resp.Header.Set("Docker-Experimental", "true")
+			}
+			resp.Body = ioutil.NopCloser(strings.NewReader("some error with the server"))
+			return resp, nil
+		}),
+	}
+
+	ping, err := client.Ping(context.Background())
+	assert.Error(t, err)
+	assert.Equal(t, false, ping.Experimental)
+	assert.Equal(t, "", ping.APIVersion)
+
+	withHeader = true
+	ping2, err := client.Ping(context.Background())
+	assert.Error(t, err)
+	assert.Equal(t, true, ping2.Experimental)
+	assert.Equal(t, "awesome", ping2.APIVersion)
+}
+
+// TestPingWithError tests the case where there is a protocol error in the ping.
+// This test is mostly just testing that there are no panics in this code path.
+func TestPingWithError(t *testing.T) {
+	client := &Client{
+		client: newMockClient(func(req *http.Request) (*http.Response, error) {
+			resp := &http.Response{StatusCode: http.StatusInternalServerError}
+			resp.Header = http.Header{}
+			resp.Header.Set("API-Version", "awesome")
+			resp.Header.Set("Docker-Experimental", "true")
+			resp.Body = ioutil.NopCloser(strings.NewReader("some error with the server"))
+			return resp, errors.New("some error")
+		}),
+	}
+
+	ping, err := client.Ping(context.Background())
+	assert.Error(t, err)
+	assert.Equal(t, false, ping.Experimental)
+	assert.Equal(t, "", ping.APIVersion)
+}
+
+// TestPingSuccess tests that we are able to get the expected API headers/ping
+// details on success.
+func TestPingSuccess(t *testing.T) {
+	client := &Client{
+		client: newMockClient(func(req *http.Request) (*http.Response, error) {
+			resp := &http.Response{StatusCode: http.StatusInternalServerError}
+			resp.Header = http.Header{}
+			resp.Header.Set("API-Version", "awesome")
+			resp.Header.Set("Docker-Experimental", "true")
+			resp.Body = ioutil.NopCloser(strings.NewReader("some error with the server"))
+			return resp, nil
+		}),
+	}
+	ping, err := client.Ping(context.Background())
+	assert.Error(t, err)
+	assert.Equal(t, true, ping.Experimental)
+	assert.Equal(t, "awesome", ping.APIVersion)
+}

components/engine/client/request.go

@@ -24,6 +24,7 @@ type serverResponse struct {
 	body       io.ReadCloser
 	header     http.Header
 	statusCode int
+	reqURL     *url.URL
 }
 
 // head sends an http request to the docker API using the method HEAD.
@@ -118,11 +119,18 @@ func (cli *Client) sendRequest(ctx context.Context, method, path string, query u
 	if err != nil {
 		return serverResponse{}, err
 	}
-	return cli.doRequest(ctx, req)
+	resp, err := cli.doRequest(ctx, req)
+	if err != nil {
+		return resp, err
+	}
+	if err := cli.checkResponseErr(resp); err != nil {
+		return resp, err
+	}
+	return resp, nil
 }
 
 func (cli *Client) doRequest(ctx context.Context, req *http.Request) (serverResponse, error) {
-	serverResp := serverResponse{statusCode: -1}
+	serverResp := serverResponse{statusCode: -1, reqURL: req.URL}
 
 	resp, err := ctxhttp.Do(ctx, cli.client, req)
 	if err != nil {
@@ -179,37 +187,44 @@ func (cli *Client) doRequest(ctx context.Context, req *http.Request) (serverResp
 
 	if resp != nil {
 		serverResp.statusCode = resp.StatusCode
+		serverResp.body = resp.Body
+		serverResp.header = resp.Header
 	}
-
-	if serverResp.statusCode < 200 || serverResp.statusCode >= 400 {
-		body, err := ioutil.ReadAll(resp.Body)
-		if err != nil {
-			return serverResp, err
-		}
-		if len(body) == 0 {
-			return serverResp, fmt.Errorf("Error: request returned %s for API route and version %s, check if the server supports the requested API version", http.StatusText(serverResp.statusCode), req.URL)
-		}
-
-		var errorMessage string
-		if (cli.version == "" || versions.GreaterThan(cli.version, "1.23")) &&
-			resp.Header.Get("Content-Type") == "application/json" {
-			var errorResponse types.ErrorResponse
-			if err := json.Unmarshal(body, &errorResponse); err != nil {
-				return serverResp, fmt.Errorf("Error reading JSON: %v", err)
-			}
-			errorMessage = errorResponse.Message
-		} else {
-			errorMessage = string(body)
-		}
-
-		return serverResp, fmt.Errorf("Error response from daemon: %s", strings.TrimSpace(errorMessage))
-	}
-
-	serverResp.body = resp.Body
-	serverResp.header = resp.Header
 	return serverResp, nil
 }
 
+func (cli *Client) checkResponseErr(serverResp serverResponse) error {
+	if serverResp.statusCode >= 200 && serverResp.statusCode < 400 {
+		return nil
+	}
+
+	body, err := ioutil.ReadAll(serverResp.body)
+	if err != nil {
+		return err
+	}
+	if len(body) == 0 {
+		return fmt.Errorf("Error: request returned %s for API route and version %s, check if the server supports the requested API version", http.StatusText(serverResp.statusCode), serverResp.reqURL)
+	}
+
+	var ct string
+	if serverResp.header != nil {
+		ct = serverResp.header.Get("Content-Type")
+	}
+
+	var errorMessage string
+	if (cli.version == "" || versions.GreaterThan(cli.version, "1.23")) && ct == "application/json" {
+		var errorResponse types.ErrorResponse
+		if err := json.Unmarshal(body, &errorResponse); err != nil {
+			return fmt.Errorf("Error reading JSON: %v", err)
+		}
+		errorMessage = errorResponse.Message
+	} else {
+		errorMessage = string(body)
+	}
+
+	return fmt.Errorf("Error response from daemon: %s", strings.TrimSpace(errorMessage))
+}
+
 func (cli *Client) addHeaders(req *http.Request, headers headers) *http.Request {
 	// Add CLI Config's HTTP Headers BEFORE we set the Docker headers
 	// then the user can't change OUR headers
@@ -239,9 +254,9 @@ func encodeData(data interface{}) (*bytes.Buffer, error) {
 }
 
 func ensureReaderClosed(response serverResponse) {
-	if body := response.body; body != nil {
+	if response.body != nil {
 		// Drain up to 512 bytes and close the body to let the Transport reuse the connection
-		io.CopyN(ioutil.Discard, body, 512)
+		io.CopyN(ioutil.Discard, response.body, 512)
 		response.body.Close()
 	}
 }

components/engine/client/service_create.go

@@ -39,7 +39,7 @@ func (cli *Client) ServiceCreate(ctx context.Context, service swarm.ServiceSpec,
 				service.TaskTemplate.ContainerSpec.Image = img
 			}
 			// add platforms that are compatible with the service
-			service.TaskTemplate.Placement = updateServicePlatforms(service.TaskTemplate.Placement, distributionInspect)
+			service.TaskTemplate.Placement = setServicePlatforms(service.TaskTemplate.Placement, distributionInspect)
 		}
 	}
 	var response types.ServiceCreateResponse
@@ -86,13 +86,15 @@ func imageWithTagString(image string) string {
 	return ""
 }
 
-// updateServicePlatforms updates the Platforms in swarm.Placement to list
-// all compatible platforms for the service, as found in distributionInspect
-// and returns a pointer to the new or updated swarm.Placement struct
-func updateServicePlatforms(placement *swarm.Placement, distributionInspect registrytypes.DistributionInspect) *swarm.Placement {
+// setServicePlatforms sets Platforms in swarm.Placement to list all
+// compatible platforms for the service, as found in distributionInspect
+// and returns a pointer to the new or updated swarm.Placement struct.
+func setServicePlatforms(placement *swarm.Placement, distributionInspect registrytypes.DistributionInspect) *swarm.Placement {
 	if placement == nil {
 		placement = &swarm.Placement{}
 	}
+	// reset any existing listed platforms
+	placement.Platforms = []swarm.Platform{}
 	for _, p := range distributionInspect.Platforms {
 		placement.Platforms = append(placement.Platforms, swarm.Platform{
 			Architecture: p.Architecture,

components/engine/client/service_update.go

@@ -51,7 +51,7 @@ func (cli *Client) ServiceUpdate(ctx context.Context, serviceID string, version
 				service.TaskTemplate.ContainerSpec.Image = img
 			}
 			// add platforms that are compatible with the service
-			service.TaskTemplate.Placement = updateServicePlatforms(service.TaskTemplate.Placement, distributionInspect)
+			service.TaskTemplate.Placement = setServicePlatforms(service.TaskTemplate.Placement, distributionInspect)
 		}
 	}
 

components/engine/container/container.go

@@ -1038,7 +1038,7 @@ func (container *Container) CreateDaemonEnvironment(tty bool, linkedEnv []string
 		platform = runtime.GOOS
 	}
 	env := []string{}
-	if runtime.GOOS != "windows" || (runtime.GOOS == "windows" && system.LCOWSupported() && platform == "linux") {
+	if runtime.GOOS != "windows" || (system.LCOWSupported() && platform == "linux") {
 		env = []string{
 			"PATH=" + system.DefaultPathEnv(platform),
 			"HOSTNAME=" + container.Config.Hostname,
@@ -1052,7 +1052,6 @@ func (container *Container) CreateDaemonEnvironment(tty bool, linkedEnv []string
 	// because the env on the container can override certain default values
 	// we need to replace the 'env' keys where they match and append anything
 	// else.
-	//return ReplaceOrAppendEnvValues(linkedEnv, container.Config.Env)
-	foo := ReplaceOrAppendEnvValues(env, container.Config.Env)
-	return foo
+	env = ReplaceOrAppendEnvValues(env, container.Config.Env)
+	return env
 }

components/engine/container/view.go

@@ -73,6 +73,17 @@ type memDB struct {
 	store *memdb.MemDB
 }
 
+// NoSuchContainerError indicates that the container wasn't found in the
+// database.
+type NoSuchContainerError struct {
+	id string
+}
+
+// Error satisfies the error interface.
+func (e NoSuchContainerError) Error() string {
+	return "no such container " + e.id
+}
+
 // NewViewDB provides the default implementation, with the default schema
 func NewViewDB() (ViewDB, error) {
 	store, err := memdb.NewMemDB(schema)
@@ -134,6 +145,9 @@ func (v *memdbView) Get(id string) (*Snapshot, error) {
 	if err != nil {
 		return nil, err
 	}
+	if s == nil {
+		return nil, NoSuchContainerError{id: id}
+	}
 	return v.transform(s.(*Container)), nil
 }
 

components/engine/daemon/cluster/executor/container/adapter.go

@@ -93,7 +93,7 @@ func (c *containerAdapter) pullImage(ctx context.Context) error {
 		// TODO @jhowardmsft LCOW Support: This will need revisiting as
 		// the stack is built up to include LCOW support for swarm.
 		platform := runtime.GOOS
-		if platform == "windows" && system.LCOWSupported() {
+		if system.LCOWSupported() {
 			platform = "linux"
 		}
 		err := c.backend.PullImage(ctx, c.container.image(), "", platform, metaHeaders, authConfig, pw)

components/engine/daemon/create.go

@@ -82,7 +82,7 @@ func (daemon *Daemon) create(params types.ContainerCreateConfig, managed bool) (
 	if params.Platform == "" {
 		params.Platform = runtime.GOOS
 	}
-	if params.Platform == "windows" && system.LCOWSupported() {
+	if system.LCOWSupported() {
 		params.Platform = "linux"
 	}
 
@@ -106,7 +106,7 @@ func (daemon *Daemon) create(params types.ContainerCreateConfig, managed bool) (
 	if img != nil {
 		if params.Platform != img.Platform() {
 			// Ignore this in LCOW mode. @jhowardmsft TODO - This will need revisiting later.
-			if !(runtime.GOOS == "windows" && system.LCOWSupported()) {
+			if !system.LCOWSupported() {
 				return nil, fmt.Errorf("cannot create a %s container from a %s image", params.Platform, img.Platform())
 			}
 		}

components/engine/daemon/graphdriver/lcow/lcow.go

@@ -16,6 +16,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/Microsoft/hcsshim"
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/daemon/graphdriver"
 	"github.com/docker/docker/pkg/archive"
@@ -32,6 +33,9 @@ func init() {
 const (
 	// sandboxFilename is the name of the file containing a layers sandbox (read-write layer)
 	sandboxFilename = "sandbox.vhdx"
+
+	// svmScratchFilename is the name of the scratch-space used by an SVM to avoid running out of memory
+	svmScratchFilename = "scratch.vhdx"
 )
 
 // cacheType is our internal structure representing an item in our local cache
@@ -51,8 +55,13 @@ type Driver struct {
 	cachedSandboxFile string
 	// options are the graphdriver options we are initialised with
 	options []string
-	// JJH LIFETIME TODO - Remove this and move up to daemon. For now, a global service utility-VM
+	// config is the representation of the SVM.
+	// @jhowardmsft LIFETIME TODO - For now, a global service utility-VM
 	config client.Config
+	// svmScratchSpaceFile is a host location for a dedicated scratch space
+	// that the SVM utilities can use as a scratch-space to avoid OOMs
+	// @jhowardmsft LIFETIME TODO - For now, a global service utility-VM
+	svmScratchSpaceFile string
 
 	// it is safe for windows to use a cache here because it does not support
 	// restoring containers when the daemon dies.
@@ -69,10 +78,11 @@ func InitLCOW(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (
 	logrus.Debugf("%s %s", title, home)
 
 	d := &Driver{
-		homeDir:           home,
-		options:           options,
-		cachedSandboxFile: filepath.Join(home, "cache", sandboxFilename),
-		cache:             make(map[string]cacheType),
+		homeDir:             home,
+		options:             options,
+		cachedSandboxFile:   filepath.Join(home, "cache", sandboxFilename),
+		svmScratchSpaceFile: filepath.Join(home, "svmscratch", svmScratchFilename),
+		cache:               make(map[string]cacheType),
 	}
 
 	if err := idtools.MkdirAllAs(home, 0700, 0, 0); err != nil {
@@ -84,6 +94,11 @@ func InitLCOW(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (
 		return nil, fmt.Errorf("%s failed to create '%s': %v", title, home, err)
 	}
 
+	// Location for the SVM scratch
+	if err := idtools.MkdirAllAs(filepath.Dir(d.svmScratchSpaceFile), 0700, 0, 0); err != nil {
+		return nil, fmt.Errorf("%s failed to create '%s': %v", title, home, err)
+	}
+
 	return d, nil
 }
 
@@ -92,6 +107,8 @@ func InitLCOW(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (
 // service VM globally to a service VM per container (or offline operation). However,
 // for the initial bring-up of LCOW, this is acceptable.
 func (d *Driver) startUvm(context string) error {
+	const toolsScratchPath = "/mnt/gcs/LinuxServiceVM/scratch"
+
 	// Nothing to do if it's already running
 	if d.config.Uvm != nil {
 		return nil
@@ -102,10 +119,36 @@ func (d *Driver) startUvm(context string) error {
 		return fmt.Errorf("failed to generate default gogcs configuration (%s): %s", context, err)
 	}
 
+	scratchAttached := false
+	if _, err := os.Stat(d.svmScratchSpaceFile); err == nil {
+		// We have a scratch space already, so just attach it as a mapped virtual disk
+		logrus.Debugf("lcowdriver: startuvm: (%s) attaching pre-existing scratch", context)
+		mvd := hcsshim.MappedVirtualDisk{
+			HostPath:          d.svmScratchSpaceFile,
+			ContainerPath:     toolsScratchPath,
+			CreateInUtilityVM: true,
+		}
+		d.config.MappedVirtualDisks = append(d.config.MappedVirtualDisks, mvd)
+		scratchAttached = true
+	}
+
 	d.config.Name = "LinuxServiceVM" // TODO @jhowardmsft - This requires an in-flight platform change. Can't hard code it to this longer term
 	if err := d.config.Create(); err != nil {
 		return fmt.Errorf("failed to start utility VM (%s): %s", context, err)
 	}
+
+	// Hot-add the scratch-space if not already attached
+	if !scratchAttached {
+		logrus.Debugf("lcowdriver: startuvm: (%s) creating an SVM scratch", context)
+		if err := d.config.CreateSandbox(d.svmScratchSpaceFile, client.DefaultSandboxSizeMB, d.cachedSandboxFile); err != nil {
+			return fmt.Errorf("failed to create SVM scratch VHDX (%s): %s", context, err)
+		}
+		logrus.Debugf("lcowdriver: startuvm: (%s) hot-adding an SVM scratch", context)
+		if err := d.config.HotAddVhd(d.svmScratchSpaceFile, toolsScratchPath); err != nil {
+			return fmt.Errorf("failed to hot-add %s failed: %s", d.svmScratchSpaceFile, err)
+		}
+	}
+	logrus.Debugf("lcowdriver: startuvm: (%s) successful", context)
 	return nil
 }
 
@@ -118,7 +161,7 @@ func (d *Driver) terminateUvm(context string) error {
 
 	// FIXME: @jhowardmsft
 	// This isn't thread-safe yet, but will change anyway with the lifetime
-	// changes and multiple instances. Defering that work for now.
+	// changes and multiple instances. Deferring that work for now.
 	uvm := d.config.Uvm
 	d.config.Uvm = nil
 
@@ -190,7 +233,6 @@ func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error {
 	// Make sure layers are created with the correct ACL so that VMs can access them.
 	layerPath := d.dir(id)
 	logrus.Debugf("lcowdriver: create: id %s: creating layerPath %s", id, layerPath)
-	// Make sure the layers are created with the correct ACL so that VMs can access them.
 	if err := system.MkdirAllWithACL(layerPath, 755, system.SddlNtvmAdministratorsLocalSystem); err != nil {
 		return err
 	}

components/engine/daemon/graphdriver/overlay2/overlay.go

@@ -194,6 +194,7 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		ctr:           graphdriver.NewRefCounter(graphdriver.NewFsChecker(graphdriver.FsMagicOverlay)),
 		supportsDType: supportsDType,
 		locker:        locker.New(),
+		options:       *opts,
 	}
 
 	d.naiveDiff = graphdriver.NewNaiveDiffDriver(d, uidMaps, gidMaps)
@@ -202,7 +203,12 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		// Try to enable project quota support over xfs.
 		if d.quotaCtl, err = quota.NewControl(home); err == nil {
 			projectQuotaSupported = true
+		} else if opts.quota.Size > 0 {
+			return nil, fmt.Errorf("Storage option overlay2.size not supported. Filesystem does not support Project Quota: %v", err)
 		}
+	} else if opts.quota.Size > 0 {
+		// if xfs is not the backing fs then error out if the storage-opt overlay2.size is used.
+		return nil, fmt.Errorf("Storage Option overlay2.size only supported for backingFS XFS. Found %v", backingFs)
 	}
 
 	logrus.Debugf("backingFs=%s,  projectQuotaSupported=%v", backingFs, projectQuotaSupported)
@@ -224,9 +230,14 @@ func parseOptions(options []string) (*overlayOptions, error) {
 			if err != nil {
 				return nil, err
 			}
-
+		case "overlay2.size":
+			size, err := units.RAMInBytes(val)
+			if err != nil {
+				return nil, err
+			}
+			o.quota.Size = uint64(size)
 		default:
-			return nil, fmt.Errorf("overlay2: Unknown option %s\n", key)
+			return nil, fmt.Errorf("overlay2: unknown option %s", key)
 		}
 	}
 	return o, nil
@@ -312,17 +323,38 @@ func (d *Driver) Cleanup() error {
 // CreateReadWrite creates a layer that is writable for use as a container
 // file system.
 func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error {
-	return d.Create(id, parent, opts)
+	if opts != nil && len(opts.StorageOpt) != 0 && !projectQuotaSupported {
+		return fmt.Errorf("--storage-opt is supported only for overlay over xfs with 'pquota' mount option")
+	}
+
+	if opts == nil {
+		opts = &graphdriver.CreateOpts{
+			StorageOpt: map[string]string{},
+		}
+	}
+
+	if _, ok := opts.StorageOpt["size"]; !ok {
+		if opts.StorageOpt == nil {
+			opts.StorageOpt = map[string]string{}
+		}
+		opts.StorageOpt["size"] = strconv.FormatUint(d.options.quota.Size, 10)
+	}
+
+	return d.create(id, parent, opts)
 }
 
 // Create is used to create the upper, lower, and merge directories required for overlay fs for a given id.
 // The parent filesystem is used to configure these directories for the overlay.
 func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) (retErr error) {
-
-	if opts != nil && len(opts.StorageOpt) != 0 && !projectQuotaSupported {
-		return fmt.Errorf("--storage-opt is supported only for overlay over xfs with 'pquota' mount option")
+	if opts != nil && len(opts.StorageOpt) != 0 {
+		if _, ok := opts.StorageOpt["size"]; ok {
+			return fmt.Errorf("--storage-opt size is only supported for ReadWrite Layers")
+		}
 	}
+	return d.create(id, parent, opts)
+}
 
+func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts) (retErr error) {
 	dir := d.dir(id)
 
 	rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)

components/engine/daemon/graphdriver/windows/windows.go

@@ -154,7 +154,7 @@ func (d *Driver) Status() [][2]string {
 }
 
 // panicIfUsedByLcow does exactly what it says.
-// TODO @jhowardmsft - this is an temporary measure for the bring-up of
+// TODO @jhowardmsft - this is a temporary measure for the bring-up of
 // Linux containers on Windows. It is a failsafe to ensure that the right
 // graphdriver is used.
 func panicIfUsedByLcow() {

components/engine/daemon/image_exporter.go

@@ -16,7 +16,7 @@ import (
 func (daemon *Daemon) ExportImage(names []string, outStream io.Writer) error {
 	// TODO @jhowardmsft LCOW. This will need revisiting later.
 	platform := runtime.GOOS
-	if platform == "windows" && system.LCOWSupported() {
+	if system.LCOWSupported() {
 		platform = "linux"
 	}
 	imageExporter := tarexport.NewTarExporter(daemon.stores[platform].imageStore, daemon.stores[platform].layerStore, daemon.stores[platform].referenceStore, daemon)
@@ -29,7 +29,7 @@ func (daemon *Daemon) ExportImage(names []string, outStream io.Writer) error {
 func (daemon *Daemon) LoadImage(inTar io.ReadCloser, outStream io.Writer, quiet bool) error {
 	// TODO @jhowardmsft LCOW. This will need revisiting later.
 	platform := runtime.GOOS
-	if platform == "windows" && system.LCOWSupported() {
+	if system.LCOWSupported() {
 		platform = "linux"
 	}
 	imageExporter := tarexport.NewTarExporter(daemon.stores[platform].imageStore, daemon.stores[platform].layerStore, daemon.stores[platform].referenceStore, daemon)

components/engine/daemon/image_push.go

@@ -43,7 +43,7 @@ func (daemon *Daemon) PushImage(ctx context.Context, image, tag string, metaHead
 
 	// TODO @jhowardmsft LCOW Support. This will require revisiting. For now, hard-code.
 	platform := runtime.GOOS
-	if platform == "windows" && system.LCOWSupported() {
+	if system.LCOWSupported() {
 		platform = "linux"
 	}
 

components/engine/daemon/images.go

@@ -38,7 +38,7 @@ func (r byCreated) Less(i, j int) bool { return r[i].Created < r[j].Created }
 func (daemon *Daemon) Map() map[image.ID]*image.Image {
 	// TODO @jhowardmsft LCOW. This will need  work to enumerate the stores for all platforms.
 	platform := runtime.GOOS
-	if platform == "windows" && system.LCOWSupported() {
+	if system.LCOWSupported() {
 		platform = "linux"
 	}
 	return daemon.stores[platform].imageStore.Map()
@@ -53,7 +53,7 @@ func (daemon *Daemon) Images(imageFilters filters.Args, all bool, withExtraAttrs
 
 	// TODO @jhowardmsft LCOW. This will need  work to enumerate the stores for all platforms.
 	platform := runtime.GOOS
-	if platform == "windows" && system.LCOWSupported() {
+	if system.LCOWSupported() {
 		platform = "linux"
 	}
 

components/engine/daemon/info.go

@@ -90,7 +90,7 @@ func (daemon *Daemon) SystemInfo() (*types.Info, error) {
 
 	// TODO @jhowardmsft LCOW support. For now, hard-code the platform shown for the driver status
 	p := runtime.GOOS
-	if p == "windows" && system.LCOWSupported() {
+	if system.LCOWSupported() {
 		p = "linux"
 	}
 

components/engine/daemon/list.go

@@ -162,11 +162,13 @@ func (daemon *Daemon) filterByNameIDMatches(view container.View, ctx *listContex
 	cntrs := make([]container.Snapshot, 0, len(matches))
 	for id := range matches {
 		c, err := view.Get(id)
-		if err != nil {
-			return nil, err
-		}
-		if c != nil {
+		switch err.(type) {
+		case nil:
 			cntrs = append(cntrs, *c)
+		case container.NoSuchContainerError:
+			// ignore error
+		default:
+			return nil, err
 		}
 	}
 

components/engine/daemon/logger/loggerutils/rotatefilewriter.go

@@ -74,7 +74,7 @@ func (w *RotateFileWriter) checkCapacityAndRotate() error {
 		if err := rotate(name, w.maxFiles); err != nil {
 			return err
 		}
-		file, err := os.OpenFile(name, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 06400)
+		file, err := os.OpenFile(name, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0640)
 		if err != nil {
 			return err
 		}

components/engine/daemon/prune.go

@@ -161,7 +161,7 @@ func (daemon *Daemon) VolumesPrune(ctx context.Context, pruneFilters filters.Arg
 func (daemon *Daemon) ImagesPrune(ctx context.Context, pruneFilters filters.Args) (*types.ImagesPruneReport, error) {
 	// TODO @jhowardmsft LCOW Support: This will need revisiting later.
 	platform := runtime.GOOS
-	if platform == "windows" && system.LCOWSupported() {
+	if system.LCOWSupported() {
 		platform = "linux"
 	}
 

components/engine/distribution/pull_v2.go

@@ -491,7 +491,7 @@ func (p *v2Puller) pullSchema1(ctx context.Context, ref reference.Named, unverif
 	// the history does, but unfortunately that's a string, so search through
 	// all the history until hopefully we find one which indicates the os.
 	platform := runtime.GOOS
-	if runtime.GOOS == "windows" && system.LCOWSupported() {
+	if system.LCOWSupported() {
 		type config struct {
 			Os string `json:"os,omitempty"`
 		}

components/engine/hack/make/test-unit

@@ -26,6 +26,8 @@ bundle_test_unit() {
 		TEST_PATH=./${TESTDIRS}
 	fi
 
+	source "${MAKEDIR}/.go-autogen"
+
 	if [ "$(go env GOHOSTOS)" = 'solaris' ]; then
 		pkg_list=$(go list -e \
 			-f '{{if ne .Name "github.com/docker/docker"}}

components/engine/image/store.go

@@ -3,7 +3,6 @@ package image
 import (
 	"encoding/json"
 	"fmt"
-	"runtime"
 	"strings"
 	"sync"
 	"time"
@@ -119,8 +118,9 @@ func (is *store) Create(config []byte) (ID, error) {
 		return "", err
 	}
 
+	// TODO @jhowardmsft - LCOW Support. This will need revisiting.
 	// Integrity check - ensure we are creating something for the correct platform
-	if runtime.GOOS == "windows" && system.LCOWSupported() {
+	if system.LCOWSupported() {
 		if strings.ToLower(img.Platform()) != strings.ToLower(is.platform) {
 			return "", fmt.Errorf("cannot create entry for platform %q in image store for platform %q", img.Platform(), is.platform)
 		}

components/engine/image/tarexport/load.go

@@ -79,6 +79,9 @@ func (l *tarexporter) Load(inTar io.ReadCloser, outStream io.Writer, quiet bool)
 		if err != nil {
 			return err
 		}
+		if err := checkCompatibleOS(img.OS); err != nil {
+			return err
+		}
 		var rootFS image.RootFS
 		rootFS = *img.RootFS
 		rootFS.DiffIDs = nil
@@ -292,11 +295,18 @@ func (l *tarexporter) legacyLoadImage(oldID, sourceDir string, loadedMap map[str
 		return err
 	}
 
-	var img struct{ Parent string }
+	var img struct {
+		OS     string
+		Parent string
+	}
 	if err := json.Unmarshal(imageJSON, &img); err != nil {
 		return err
 	}
 
+	if err := checkCompatibleOS(img.OS); err != nil {
+		return err
+	}
+
 	var parentID image.ID
 	if img.Parent != "" {
 		for {
@@ -402,3 +412,20 @@ func checkValidParent(img, parent *image.Image) bool {
 	}
 	return true
 }
+
+func checkCompatibleOS(os string) error {
+	// TODO @jhowardmsft LCOW - revisit for simultaneous platforms
+	platform := runtime.GOOS
+	if system.LCOWSupported() {
+		platform = "linux"
+	}
+	// always compatible if the OS matches; also match an empty OS
+	if os == platform || os == "" {
+		return nil
+	}
+	// for compatibility, only fail if the image or runtime OS is Windows
+	if os == "windows" || platform == "windows" {
+		return fmt.Errorf("cannot load %s image on %s", os, platform)
+	}
+	return nil
+}

components/engine/integration-cli/docker_api_build_test.go

@@ -366,6 +366,50 @@ func (s *DockerRegistrySuite) TestBuildCopyFromForcePull(c *check.C) {
 	assert.Contains(c, string(out), "Successfully built")
 }
 
+func (s *DockerSuite) TestBuildAddRemoteNoDecompress(c *check.C) {
+	buffer := new(bytes.Buffer)
+	tw := tar.NewWriter(buffer)
+	dt := []byte("contents")
+	err := tw.WriteHeader(&tar.Header{
+		Name:     "foo",
+		Size:     int64(len(dt)),
+		Mode:     0600,
+		Typeflag: tar.TypeReg,
+	})
+	require.NoError(c, err)
+	_, err = tw.Write(dt)
+	require.NoError(c, err)
+	err = tw.Close()
+	require.NoError(c, err)
+
+	server := fakestorage.New(c, "", fakecontext.WithBinaryFiles(map[string]*bytes.Buffer{
+		"test.tar": buffer,
+	}))
+	defer server.Close()
+
+	dockerfile := fmt.Sprintf(`
+		FROM busybox
+		ADD %s/test.tar /
+		RUN [ -f test.tar ]
+		`, server.URL())
+
+	ctx := fakecontext.New(c, "",
+		fakecontext.WithDockerfile(dockerfile),
+	)
+	defer ctx.Close()
+
+	res, body, err := request.Post(
+		"/build",
+		request.RawContent(ctx.AsTarReader(c)),
+		request.ContentType("application/x-tar"))
+	require.NoError(c, err)
+	assert.Equal(c, http.StatusOK, res.StatusCode)
+
+	out, err := testutil.ReadBody(body)
+	require.NoError(c, err)
+	assert.Contains(c, string(out), "Successfully built")
+}
+
 func (s *DockerSuite) TestBuildWithSession(c *check.C) {
 	testRequires(c, ExperimentalDaemon)
 

components/engine/integration-cli/docker_cli_build_test.go

@@ -6459,3 +6459,32 @@ func (s *DockerSuite) TestBuildIidFileCleanupOnFail(c *check.C) {
 	c.Assert(err, check.NotNil)
 	c.Assert(os.IsNotExist(err), check.Equals, true)
 }
+
+func (s *DockerSuite) TestBuildIidFileSquash(c *check.C) {
+	testRequires(c, ExperimentalDaemon)
+	tmpDir, err := ioutil.TempDir("", "TestBuildIidFileSquash")
+	if err != nil {
+		c.Fatal(err)
+	}
+	defer os.RemoveAll(tmpDir)
+	tmpIidFile := filepath.Join(tmpDir, "iidsquash")
+
+	name := "testbuildiidfilesquash"
+	// Use a Dockerfile with multiple stages to ensure we get the last one
+	cli.BuildCmd(c, name,
+		// This could be minimalBaseImage except
+		// https://github.com/moby/moby/issues/33823 requires
+		// `touch` to workaround.
+		build.WithDockerfile(`FROM busybox
+ENV FOO FOO
+ENV BAR BAR
+RUN touch /foop
+`),
+		cli.WithFlags("--iidfile", tmpIidFile, "--squash"))
+
+	id, err := ioutil.ReadFile(tmpIidFile)
+	c.Assert(err, check.IsNil)
+	d, err := digest.Parse(string(id))
+	c.Assert(err, check.IsNil)
+	c.Assert(d.String(), checker.Equals, getIDByName(c, name))
+}

components/engine/integration-cli/docker_cli_daemon_test.go

@@ -2828,7 +2828,7 @@ func (s *DockerDaemonSuite) TestExecWithUserAfterLiveRestore(c *check.C) {
 
 	out2, err := s.d.Cmd("exec", "-u", "test", "top", "id")
 	c.Assert(err, check.IsNil, check.Commentf("Output: %s", out2))
-	c.Assert(out1, check.Equals, out2, check.Commentf("Output: before restart '%s', after restart '%s'", out1, out2))
+	c.Assert(out2, check.Equals, out1, check.Commentf("Output: before restart '%s', after restart '%s'", out1, out2))
 
 	out, err = s.d.Cmd("stop", "top")
 	c.Assert(err, check.IsNil, check.Commentf("Output: %s", out))

components/engine/layer/layer_store.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"runtime"
 	"strings"
 	"sync"
 
@@ -273,7 +272,7 @@ func (ls *layerStore) registerWithDescriptor(ts io.Reader, parent ChainID, platf
 	var p *roLayer
 
 	// Integrity check - ensure we are creating something for the correct platform
-	if runtime.GOOS == "windows" && system.LCOWSupported() {
+	if system.LCOWSupported() {
 		if strings.ToLower(ls.platform) != strings.ToLower(string(platform)) {
 			return nil, fmt.Errorf("cannot create entry for platform %q in layer store for platform %q", platform, ls.platform)
 		}

components/engine/plugin/manager.go

@@ -353,7 +353,7 @@ func isEqualPrivilege(a, b types.PluginPrivilege) bool {
 func configToRootFS(c []byte) (*image.RootFS, layer.Platform, error) {
 	// TODO @jhowardmsft LCOW - Will need to revisit this. For now, calculate the platform.
 	platform := layer.Platform(runtime.GOOS)
-	if platform == "windows" && system.LCOWSupported() {
+	if system.LCOWSupported() {
 		platform = "linux"
 	}
 	var pluginConfig types.PluginConfig

components/engine/reports/2017-06-26.md

@@ -0,0 +1,120 @@
+# Development Report for June 26, 2017
+
+## Moby Summit
+
+The Moby Summit held in San Francisco was very active and well attended ([blog](http://mobyproject.org/blog/2017/06/26/moby-summit-recap/) / [linuxkit table notes](https://github.com/linuxkit/linuxkit/blob/master/reports/2017-06-19-summit.md) [#2090](https://github.com/linuxkit/linuxkit/pull/2090) [#2033](https://github.com/linuxkit/linuxkit/pull/2033) [@mgoelzer] [@justincormack]). 
+
+## Container Engine
+
+Thanks to @fabiokung there is no container locks anymore on `docker ps` [#31273](https://github.com/moby/moby/pull/31273)
+
+## BuildKit
+
+[Repo](https://github.com/moby/buildkit)
+[Proposal](https://github.com/moby/moby/issues/32925)
+
+New development repo is open at https://github.com/moby/buildkit
+
+The readme file provides examples how to get started. You can see an example of building BuildKit with BuildKit.
+
+There are lots of new issues opened as well to track the missing functionality. You are welcomed to help on any of them or discuss the design there.
+
+Last week most of the work was done on improving the `llb` client library for more complicated use cases and providing traces and interactive progress of executed build jobs.
+
+The `llb` client package is a go library that helps you to generate the build definition graph. It uses chained methods to make it easy to describe what steps need to be running. Mounts can be added to the execution steps for defining multiple inputs or outputs. To prepare the graph, you just have to call `Marshal()` on a leaf node that will generate the protobuf definition for everything required to build that node.
+
+### Typed Dockerfile parsing
+
+[PR](https://github.com/moby/moby/pull/33492)
+
+This PR that enables parsing Dockerfiles into typed structures so they can be preprocessed to eliminate unnecessary build stages and reused with different kinds of dispatchers(eg. BuildKit).
+
+The PR had some review and updates in last week. Should be ready to code review soon.
+
+### Merged: Long running session & incremental file sending
+
+[PR](https://github.com/moby/moby/pull/32677) 
+
+Incremental context sending PR was merged and is expected to land in `v17.07`.
+
+This feature experimental feature lets you skip sending the build context to the daemon on repeated builder invocations during development. Currently, this feature requires a CLI flag `--stream=true`. If this flag is used, one first builder invocation full build context is sent to the daemon. On a second attempt, only the changed files are transferred.
+
+Previous build context is saved in the build cache, and you can see how much space it takes form `docker system df`. Build cache will be automatically garbage collected and can also be manually cleared with `docker prune`.
+
+### Quality: Dependency interface switch
+
+[Move file copying from the daemon to the builder](https://github.com/moby/moby/pull/33454) PR was merged.
+
+
+### Proposals for new Dockerfile features that need design feedback:
+
+[Add IMPORT/EXPORT commands to Dockerfile](https://github.com/moby/moby/issues/32100)
+
+[Add `DOCKEROS/DOCKERARCH` default ARG to Dockerfile](https://github.com/moby/moby/issues/32487)
+
+[Add support for `RUN --mount`](https://github.com/moby/moby/issues/32507)
+
+[DAG image builder](https://github.com/moby/moby/issues/32550)
+
+[Option to export the hash of the build context](https://github.com/moby/moby/issues/32963) (new)
+
+[Allow --cache-from=*](https://github.com/moby/moby/issues/33002#issuecomment-299041162) (new)
+
+[Provide advanced .dockeringore use-cases](https://github.com/moby/moby/issues/12886) [2](https://github.com/moby/moby/issues/12886#issuecomment-306247989)
+
+If you are interested in implementing any of them, leave a comment on the specific issues.
+
+### Other builder PRs merged last week
+
+[Warn/deprecate continuing on empty lines in `Dockerfile`](https://github.com/moby/moby/pull/29161)
+
+[Fix behavior of absolute paths in .dockerignore](https://github.com/moby/moby/pull/32088)
+
+[fix copy —from conflict with force pull](https://github.com/moby/moby/pull/33735)
+
+### Builder features currently in code-review:
+
+[Fix handling of remote "git@" notation](https://github.com/moby/moby/pull/33696)
+
+[builder: Emit a BuildResult after squashing.](https://github.com/moby/moby/pull/33824)
+
+[Fix shallow git clone in docker-build](https://github.com/moby/moby/pull/33704)
+
+### Backlog
+
+[Build secrets](https://github.com/moby/moby/issues/33343) has not got much traction. If you want this feature to become a reality, please make yourself heard.
+
+## LinuxKit
+
+* **Kernel GPG verification:** The kernel compilation containers now verify the GPG and SHA256
+  checksums before building the binaries. ([#2062](https://github.com/linuxkit/linuxkit/issues/2062) [#2083](https://github.com/linuxkit/linuxkit/issues/2083) [@mscribe] [@justincormack] [@rn] [@riyazdf]).
+  The base Alpine build image now includes `gnupg` to support this feature ([#2091](https://github.com/linuxkit/linuxkit/issues/2091) [@riyazdf] [@rn]).
+
+* **Security SIG on Landlock:** The third Moby Security SIG focussed on the [Landlock](https://github.com/landlock-lsm) security module that provides unprivileged fine-grained sandboxing to applications.  There are videos and forum links ([#2087](https://github.com/linuxkit/linuxkit/issues/2087) [#2089](https://github.com/linuxkit/linuxkit/issues/2089) [#2073](https://github.com/linuxkit/linuxkit/issues/2073) [@riyazdf]).
+
+* **Networking drivers now modules:** The kernels have been updated to 4.11.6/4.9.33/4.4.73, and many drivers are now loaded as modules to speed up boot-time ([#2095](https://github.com/linuxkit/linuxkit/issues/2095) [#2061](https://github.com/linuxkit/linuxkit/issues/2061) [@rn] [@justincormack] [@tych0])
+
+- **Whaley important update:** The ASCII logo was updated and we fondly wave goodbye to the waves. ([#2084](https://github.com/linuxkit/linuxkit/issues/2084) [@thaJeztah] [@rn])
+
+- **Containerised getty and sshd:** The login services now run in their own mount namespace, which was confusing people since they were expecting it to be on the host filesystem.  This is now being addressed via a reminder in the `motd` upon login ([#2078](https://github.com/linuxkit/linuxkit/issues/2078) [#2097](https://github.com/linuxkit/linuxkit/issues/2097) [@deitch] [@ijc] [@justincormack] [@riyazdf] [@rn])
+
+- **Hardened user copying:** The RFC on ensuring that we use a hardened kernel/userspace copying system was closed, as it is enabled by default on all our modern kernels and a regression test is included by default ([#2086](https://github.com/linuxkit/linuxkit/issues/2086) [@fntlnz] [@riyazdf]).
+
+- **Vultr provider:** There is an ongoing effort to add a metadata provider for [Vultr](http://vultr.com) ([#2101](https://github.com/linuxkit/linuxkit/issues/2101) [@furious-luke] [@justincormack]).
+
+### Packages and Projects
+
+- Simplified Makefiles for packages ([#2080](https://github.com/linuxkit/linuxkit/issues/2080) [@justincormack] [@rn])
+- The MirageOS SDK is integrating many upstream changes from dependent libraries, for the DHCP client ([#2070](https://github.com/linuxkit/linuxkit/issues/2070) [#2072](https://github.com/linuxkit/linuxkit/issues/2072) [@samoht] [@talex5] [@avsm]).
+
+### Documentation and Tests
+
+- A comprehensive test suite for containerd is now integrated into LinuxKit tests ([#2062](https://github.com/linuxkit/linuxkit/issues/2062) [@AkihiroSuda] [@justincormack] [@rn])
+- Fix documentation links ([#2074](https://github.com/linuxkit/linuxkit/issues/2074) [@ndauten] [@justincormack])
+- Update RTF version ([#2077](https://github.com/linuxkit/linuxkit/issues/2077) [@justincormack])
+- tests: add build test for Docker for Mac blueprint ([#2093](https://github.com/linuxkit/linuxkit/issues/2093) [@riyazdf] [@MagnusS])
+- Disable Qemu EFI ISO test for now ([#2100](https://github.com/linuxkit/linuxkit/issues/2100) [@justincormack])
+- The CI whitelists and ACLs were updated ([linuxkit-ci#11](https://github.com/linuxkit/linuxkit-ce/issues/11) [linuxkit-ci#15](https://github.com/linuxkit/linuxkit-ce/issues/15) [linuxkit/linuxkit-ci#10](https://github.com/linuxkit/linuxkit-ce/issues/10) [@rn] [@justincormack])
+- Fix spelling errors ([#2079](https://github.com/linuxkit/linuxkit/issues/2079) [@ndauten])
+- Fix typo in dev report ([#2094](https://github.com/linuxkit/linuxkit/issues/2094) [@justincormack])
+- Fix dead Link to VMWare File ([#2082](https://github.com/linuxkit/linuxkit/issues/2082) [@davefreitag])

components/engine/reports/builder/2017-06-26.md

@@ -0,0 +1,78 @@
+# Development Report for June 26, 2017
+
+
+### BuildKit
+
+[Repo](https://github.com/moby/buildkit)
+[Proposal](https://github.com/moby/moby/issues/32925)
+
+New development repo is open at https://github.com/moby/buildkit
+
+The readme file provides examples how to get started. You can see an example of building BuildKit with BuildKit.
+
+There are lots of new issues opened as well to track the missing functionality. You are welcomed to help on any of them or discuss the design there.
+
+Last week most of the work was done on improving the `llb` client library for more complicated use cases and providing traces and interactive progress of executed build jobs.
+
+The `llb` client package is a go library that helps you to generate the build definition graph. It uses chained methods to make it easy to describe what steps need to be running. Mounts can be added to the execution steps for defining multiple inputs or outputs. To prepare the graph, you just have to call `Marshal()` on a leaf node that will generate the protobuf definition for everything required to build that node.
+
+### Typed Dockerfile parsing
+
+[PR](https://github.com/moby/moby/pull/33492)
+
+This PR that enables parsing Dockerfiles into typed structures so they can be preprocessed to eliminate unnecessary build stages and reused with different kinds of dispatchers(eg. BuildKit).
+
+The PR had some review and updates in last week. Should be ready to code review soon.
+
+### Merged: Long running session & incremental file sending
+
+[PR](https://github.com/moby/moby/pull/32677) 
+
+Incremental context sending PR was merged and is expected to land in `v17.07`.
+
+This feature experimental feature lets you skip sending the build context to the daemon on repeated builder invocations during development. Currently, this feature requires a CLI flag `--stream=true`. If this flag is used, one first builder invocation full build context is sent to the daemon. On a second attempt, only the changed files are transferred.
+
+Previous build context is saved in the build cache, and you can see how much space it takes form `docker system df`. Build cache will be automatically garbage collected and can also be manually cleared with `docker prune`.
+
+### Quality: Dependency interface switch
+
+[Move file copying from the daemon to the builder](https://github.com/moby/moby/pull/33454) PR was merged.
+
+
+### Proposals for new Dockerfile features that need design feedback:
+
+[Add IMPORT/EXPORT commands to Dockerfile](https://github.com/moby/moby/issues/32100)
+
+[Add `DOCKEROS/DOCKERARCH` default ARG to Dockerfile](https://github.com/moby/moby/issues/32487)
+
+[Add support for `RUN --mount`](https://github.com/moby/moby/issues/32507)
+
+[DAG image builder](https://github.com/moby/moby/issues/32550)
+
+[Option to export the hash of the build context](https://github.com/moby/moby/issues/32963) (new)
+
+[Allow --cache-from=*](https://github.com/moby/moby/issues/33002#issuecomment-299041162) (new)
+
+[Provide advanced .dockeringore use-cases](https://github.com/moby/moby/issues/12886) [2](https://github.com/moby/moby/issues/12886#issuecomment-306247989)
+
+If you are interested in implementing any of them, leave a comment on the specific issues.
+
+### Other builder PRs merged last week
+
+[Warn/deprecate continuing on empty lines in `Dockerfile`](https://github.com/moby/moby/pull/29161)
+
+[Fix behavior of absolute paths in .dockerignore](https://github.com/moby/moby/pull/32088)
+
+[fix copy —from conflict with force pull](https://github.com/moby/moby/pull/33735)
+
+### Builder features currently in code-review:
+
+[Fix handling of remote "git@" notation](https://github.com/moby/moby/pull/33696)
+
+[builder: Emit a BuildResult after squashing.](https://github.com/moby/moby/pull/33824)
+
+[Fix shallow git clone in docker-build](https://github.com/moby/moby/pull/33704)
+
+### Backlog
+
+[Build secrets](https://github.com/moby/moby/issues/33343) has not got much traction. If you want this feature to become a reality, please make yourself heard.

components/engine/vendor.conf

@@ -13,7 +13,6 @@ github.com/kr/pty 5cf931ef8f
 github.com/mattn/go-shellwords v1.0.3
 github.com/tchap/go-patricia v2.2.6
 github.com/vdemeester/shakers 24d7f1d6a71aa5d9cbe7390e4afb66b7eef9e1b3
-# forked golang.org/x/net package includes a patch for lazy loading trace templates
 golang.org/x/net 7dcfb8076726a3fdd9353b6b8a1f1b6be6811bd6
 golang.org/x/sys 8f0908ab3b2457e2e15403d3697c9ef5cb4b57a9
 github.com/docker/go-units 9e638d38cf6977a37a8ea0078f3ee75a7cdb2dd1

components/engine/volume/volume.go

@@ -153,16 +153,18 @@ func (m *MountPoint) Cleanup() error {
 // before creating the source directory on the host.
 func (m *MountPoint) Setup(mountLabel string, rootIDs idtools.IDPair, checkFun func(m *MountPoint) error) (path string, err error) {
 	defer func() {
-		if err == nil {
-			if label.RelabelNeeded(m.Mode) {
-				if err = label.Relabel(m.Source, mountLabel, label.IsShared(m.Mode)); err != nil {
-					path = ""
-					err = errors.Wrapf(err, "error setting label on mount source '%s'", m.Source)
-					return
-				}
-			}
+		if err != nil || !label.RelabelNeeded(m.Mode) {
+			return
+		}
+
+		err = label.Relabel(m.Source, mountLabel, label.IsShared(m.Mode))
+		if err == syscall.ENOTSUP {
+			err = nil
+		}
+		if err != nil {
+			path = ""
+			err = errors.Wrapf(err, "error setting label on mount source '%s'", m.Source)
 		}
-		return
 	}()
 
 	if m.Volume != nil {