Implement incremental file sync using client session

Also exposes shared cache and garbage collection/prune
for the source data.

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
Upstream-commit: 5c3d2d552b0430672d5f481ab2d37036f6e92166
Component: engine

components/engine/integration-cli/docker_api_build_test.go

@@ -12,6 +12,8 @@ import (
 	"strings"
 
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/client/session"
+	"github.com/docker/docker/client/session/filesync"
 	"github.com/docker/docker/integration-cli/checker"
 	"github.com/docker/docker/integration-cli/cli/build/fakecontext"
 	"github.com/docker/docker/integration-cli/cli/build/fakegit"
@@ -22,6 +24,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/net/context"
+	"golang.org/x/sync/errgroup"
 )
 
 func (s *DockerSuite) TestBuildAPIDockerFileRemote(c *check.C) {
@@ -363,6 +366,108 @@ func (s *DockerRegistrySuite) TestBuildCopyFromForcePull(c *check.C) {
 	assert.Contains(c, string(out), "Successfully built")
 }
 
+func (s *DockerSuite) TestBuildWithSession(c *check.C) {
+	testRequires(c, ExperimentalDaemon)
+
+	dockerfile := `
+		FROM busybox
+		COPY file /
+		RUN cat /file
+	`
+
+	fctx := fakecontext.New(c, "",
+		fakecontext.WithFile("file", "some content"),
+	)
+	defer fctx.Close()
+
+	out := testBuildWithSession(c, fctx.Dir, dockerfile)
+	assert.Contains(c, out, "some content")
+
+	fctx.Add("second", "contentcontent")
+
+	dockerfile += `
+	COPY second /
+	RUN cat /second
+	`
+
+	out = testBuildWithSession(c, fctx.Dir, dockerfile)
+	assert.Equal(c, strings.Count(out, "Using cache"), 2)
+	assert.Contains(c, out, "contentcontent")
+
+	client, err := request.NewClient()
+	require.NoError(c, err)
+
+	du, err := client.DiskUsage(context.TODO())
+	assert.Nil(c, err)
+	assert.True(c, du.BuilderSize > 10)
+
+	out = testBuildWithSession(c, fctx.Dir, dockerfile)
+	assert.Equal(c, strings.Count(out, "Using cache"), 4)
+
+	du2, err := client.DiskUsage(context.TODO())
+	assert.Nil(c, err)
+	assert.Equal(c, du.BuilderSize, du2.BuilderSize)
+
+	// rebuild with regular tar, confirm cache still applies
+	fctx.Add("Dockerfile", dockerfile)
+	res, body, err := request.Post(
+		"/build",
+		request.RawContent(fctx.AsTarReader(c)),
+		request.ContentType("application/x-tar"))
+	require.NoError(c, err)
+	assert.Equal(c, http.StatusOK, res.StatusCode)
+
+	outBytes, err := testutil.ReadBody(body)
+	require.NoError(c, err)
+	assert.Contains(c, string(outBytes), "Successfully built")
+	assert.Equal(c, strings.Count(string(outBytes), "Using cache"), 4)
+
+	_, err = client.BuildCachePrune(context.TODO())
+	assert.Nil(c, err)
+
+	du, err = client.DiskUsage(context.TODO())
+	assert.Nil(c, err)
+	assert.Equal(c, du.BuilderSize, int64(0))
+}
+
+func testBuildWithSession(c *check.C, dir, dockerfile string) (outStr string) {
+	client, err := request.NewClient()
+	require.NoError(c, err)
+
+	sess, err := session.NewSession("foo1", "foo")
+	assert.Nil(c, err)
+
+	fsProvider := filesync.NewFSSyncProvider(dir, nil)
+	sess.Allow(fsProvider)
+
+	g, ctx := errgroup.WithContext(context.Background())
+
+	g.Go(func() error {
+		return sess.Run(ctx, client.DialSession)
+	})
+
+	g.Go(func() error {
+		res, body, err := request.Post("/build?remote=client-session&session="+sess.UUID(), func(req *http.Request) error {
+			req.Body = ioutil.NopCloser(strings.NewReader(dockerfile))
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+		assert.Equal(c, res.StatusCode, http.StatusOK)
+		out, err := testutil.ReadBody(body)
+		require.NoError(c, err)
+		assert.Contains(c, string(out), "Successfully built")
+		sess.Close()
+		outStr = string(out)
+		return nil
+	})
+
+	err = g.Wait()
+	assert.Nil(c, err)
+	return
+}
+
 type buildLine struct {
 	Stream string
 	Aux    struct {

components/engine/integration-cli/docker_cli_daemon_test.go

@@ -1789,7 +1789,7 @@ func (s *DockerDaemonSuite) TestDaemonNoSpaceLeftOnDeviceError(c *check.C) {
 
 	// create a 2MiB image and mount it as graph root
 	// Why in a container? Because `mount` sometimes behaves weirdly and often fails outright on this test in debian:jessie (which is what the test suite runs under if run from the Makefile)
-	dockerCmd(c, "run", "--rm", "-v", testDir+":/test", "busybox", "sh", "-c", "dd of=/test/testfs.img bs=1M seek=2 count=0")
+	dockerCmd(c, "run", "--rm", "-v", testDir+":/test", "busybox", "sh", "-c", "dd of=/test/testfs.img bs=1M seek=3 count=0")
 	icmd.RunCommand("mkfs.ext4", "-F", filepath.Join(testDir, "testfs.img")).Assert(c, icmd.Success)
 
 	result := icmd.RunCommand("losetup", "-f", "--show", filepath.Join(testDir, "testfs.img"))

components/engine/integration-cli/request/request.go

@@ -17,6 +17,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/docker/docker/api"
 	dclient "github.com/docker/docker/client"
 	"github.com/docker/docker/opts"
 	"github.com/docker/docker/pkg/ioutils"
@@ -170,7 +171,7 @@ func NewClient() (dclient.APIClient, error) {
 	if err != nil {
 		return nil, err
 	}
-	return dclient.NewClient(host, "", httpClient, nil)
+	return dclient.NewClient(host, api.DefaultVersion, httpClient, nil)
 }
 
 // FIXME(vdemeester) httputil.ClientConn is deprecated, use http.Client instead (closer to actual client)