Implement incremental file sync using client session

Also exposes shared cache and garbage collection/prune
for the source data.

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
Upstream-commit: 5c3d2d552b0430672d5f481ab2d37036f6e92166
Component: engine

components/engine/api/server/backend/build/backend.go

@@ -4,11 +4,11 @@ import (
 	"fmt"
 
 	"github.com/docker/distribution/reference"
+	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/backend"
 	"github.com/docker/docker/builder"
-	"github.com/docker/docker/builder/dockerfile"
+	"github.com/docker/docker/builder/fscache"
 	"github.com/docker/docker/image"
-	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/stringid"
 	"github.com/pkg/errors"
 	"golang.org/x/net/context"
@@ -20,16 +20,21 @@ type ImageComponent interface {
 	TagImageWithReference(image.ID, string, reference.Named) error
 }
 
+// Builder defines interface for running a build
+type Builder interface {
+	Build(context.Context, backend.BuildConfig) (*builder.Result, error)
+}
+
 // Backend provides build functionality to the API router
 type Backend struct {
-	manager        *dockerfile.BuildManager
+	builder        Builder
+	fsCache        *fscache.FSCache
 	imageComponent ImageComponent
 }
 
 // NewBackend creates a new build backend from components
-func NewBackend(components ImageComponent, builderBackend builder.Backend, sg dockerfile.SessionGetter, idMappings *idtools.IDMappings) (*Backend, error) {
-	manager := dockerfile.NewBuildManager(builderBackend, sg, idMappings)
-	return &Backend{imageComponent: components, manager: manager}, nil
+func NewBackend(components ImageComponent, builder Builder, fsCache *fscache.FSCache) (*Backend, error) {
+	return &Backend{imageComponent: components, builder: builder, fsCache: fsCache}, nil
 }
 
 // Build builds an image from a Source
@@ -40,7 +45,7 @@ func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string
 		return "", err
 	}
 
-	build, err := b.manager.Build(ctx, config)
+	build, err := b.builder.Build(ctx, config)
 	if err != nil {
 		return "", err
 	}
@@ -58,6 +63,15 @@ func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string
 	return imageID, err
 }
 
+// PruneCache removes all cached build sources
+func (b *Backend) PruneCache(ctx context.Context) (*types.BuildCachePruneReport, error) {
+	size, err := b.fsCache.Prune()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to prune build cache")
+	}
+	return &types.BuildCachePruneReport{SpaceReclaimed: size}, nil
+}
+
 func squashBuild(build *builder.Result, imageComponent ImageComponent) (string, error) {
 	var fromID string
 	if build.FromImage != nil {

components/engine/api/server/router/build/backend.go

@@ -1,6 +1,7 @@
 package build
 
 import (
+	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/backend"
 	"golang.org/x/net/context"
 )
@@ -10,6 +11,9 @@ type Backend interface {
 	// Build a Docker image returning the id of the image
 	// TODO: make this return a reference instead of string
 	Build(context.Context, backend.BuildConfig) (string, error)
+
+	// Prune build cache
+	PruneCache(context.Context) (*types.BuildCachePruneReport, error)
 }
 
 type experimentalProvider interface {

components/engine/api/server/router/build/build.go

@@ -24,5 +24,6 @@ func (r *buildRouter) Routes() []router.Route {
 func (r *buildRouter) initRoutes() {
 	r.routes = []router.Route{
 		router.NewPostRoute("/build", r.postBuild, router.WithCancel),
+		router.NewPostRoute("/build/prune", r.postPrune, router.WithCancel),
 	}
 }

components/engine/api/server/router/build/build_routes.go

@@ -132,6 +132,14 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui
 	return options, nil
 }
 
+func (br *buildRouter) postPrune(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
+	report, err := br.backend.PruneCache(ctx)
+	if err != nil {
+		return err
+	}
+	return httputils.WriteJSON(w, http.StatusOK, report)
+}
+
 func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
 	var (
 		notVerboseBuffer = bytes.NewBuffer(nil)

components/engine/api/server/router/system/system.go

@@ -2,6 +2,7 @@ package system
 
 import (
 	"github.com/docker/docker/api/server/router"
+	"github.com/docker/docker/builder/fscache"
 	"github.com/docker/docker/daemon/cluster"
 )
 
@@ -11,13 +12,15 @@ type systemRouter struct {
 	backend Backend
 	cluster *cluster.Cluster
 	routes  []router.Route
+	builder *fscache.FSCache
 }
 
 // NewRouter initializes a new system router
-func NewRouter(b Backend, c *cluster.Cluster) router.Router {
+func NewRouter(b Backend, c *cluster.Cluster, fscache *fscache.FSCache) router.Router {
 	r := &systemRouter{
 		backend: b,
 		cluster: c,
+		builder: fscache,
 	}
 
 	r.routes = []router.Route{

components/engine/api/server/router/system/system_routes.go

@@ -17,6 +17,7 @@ import (
 	timetypes "github.com/docker/docker/api/types/time"
 	"github.com/docker/docker/api/types/versions"
 	"github.com/docker/docker/pkg/ioutils"
+	pkgerrors "github.com/pkg/errors"
 	"golang.org/x/net/context"
 )
 
@@ -75,6 +76,11 @@ func (s *systemRouter) getDiskUsage(ctx context.Context, w http.ResponseWriter,
 	if err != nil {
 		return err
 	}
+	builderSize, err := s.builder.DiskUsage()
+	if err != nil {
+		return pkgerrors.Wrap(err, "error getting build cache usage")
+	}
+	du.BuilderSize = builderSize
 
 	return httputils.WriteJSON(w, http.StatusOK, du)
 }

components/engine/api/swagger.yaml

@@ -4745,6 +4745,27 @@ paths:
           schema:
             $ref: "#/definitions/ErrorResponse"
       tags: ["Image"]
+  /build/prune:
+    post:
+      summary: "Delete builder cache"
+      produces:
+        - "application/json"
+      operationId: "BuildPrune"
+      responses:
+        200:
+          description: "No error"
+          schema:
+            type: "object"
+            properties:
+              SpaceReclaimed:
+                description: "Disk space reclaimed in bytes"
+                type: "integer"
+                format: "int64"
+        500:
+          description: "Server error"
+          schema:
+            $ref: "#/definitions/ErrorResponse"
+      tags: ["Image"]
   /images/create:
     post:
       summary: "Create an image"

components/engine/api/types/types.go

@@ -489,10 +489,11 @@ type Runtime struct {
 // DiskUsage contains response of Engine API:
 // GET "/system/df"
 type DiskUsage struct {
-	LayersSize int64
-	Images     []*ImageSummary
-	Containers []*Container
-	Volumes    []*Volume
+	LayersSize  int64
+	Images      []*ImageSummary
+	Containers  []*Container
+	Volumes     []*Volume
+	BuilderSize int64
 }
 
 // ContainersPruneReport contains the response for Engine API:
@@ -516,6 +517,12 @@ type ImagesPruneReport struct {
 	SpaceReclaimed uint64
 }
 
+// BuildCachePruneReport contains the response for Engine API:
+// POST "/build/prune"
+type BuildCachePruneReport struct {
+	SpaceReclaimed uint64
+}
+
 // NetworksPruneReport contains the response for Engine API:
 // POST "/networks/prune"
 type NetworksPruneReport struct {