toolshed/docker-cli
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Extract squash and tagging from the Dockerfile builder.

Browse Source 
Remove pathCache and replace it with syncmap
Cleanup NewBuilder
Create an api/server/backend/build
Extract BuildTagger

Signed-off-by: Daniel Nephin <dnephin@docker.com>
Upstream-commit: 0296797f0f39477d675128c93c1646b3186937ee
Component: engine
This commit is contained in:
Daniel Nephin
2017-05-01 12:07:32 -04:00
parent dff17659b7
commit 3d193f829d
15 changed files with 337 additions and 259 deletions
Download Patch File Download Diff File Expand all files Collapse all files
components/engine/api/server/backend/build/backend.go
+70
View File
@@ -0,0 +1,70 @@
package build
import (
"fmt"
"github.com/docker/distribution/reference"
"github.com/docker/docker/api/types/backend"
"github.com/docker/docker/builder"
"github.com/docker/docker/builder/dockerfile"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/stringid"
"github.com/pkg/errors"
"golang.org/x/net/context"
)
// ImageComponent provides an interface for working with images
type ImageComponent interface {
SquashImage(from string, to string) (string, error)
TagImageWithReference(image.ID, reference.Named) error
}
// Backend provides build functionality to the API router
type Backend struct {
manager *dockerfile.BuildManager
imageComponent ImageComponent
}
// NewBackend creates a new build backend from components
func NewBackend(components ImageComponent, builderBackend builder.Backend) *Backend {
manager := dockerfile.NewBuildManager(builderBackend)
return &Backend{imageComponent: components, manager: manager}
}
// Build builds an image from a Source
func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string, error) {
options := config.Options
tagger, err := NewTagger(b.imageComponent, config.ProgressWriter.StdoutFormatter, options.Tags)
if err != nil {
return "", err
}
build, err := b.manager.Build(ctx, config)
if err != nil {
return "", err
}
var imageID = build.ImageID
if options.Squash {
if imageID, err = squashBuild(build, b.imageComponent); err != nil {
return "", err
}
}
stdout := config.ProgressWriter.StdoutFormatter
fmt.Fprintf(stdout, "Successfully built %s\n", stringid.TruncateID(imageID))
err = tagger.TagImages(image.ID(imageID))
return imageID, err
}
func squashBuild(build *builder.Result, imageComponent ImageComponent) (string, error) {
var fromID string
if build.FromImage != nil {
fromID = build.FromImage.ImageID()
}
imageID, err := imageComponent.SquashImage(build.ImageID, fromID)
if err != nil {
return "", errors.Wrap(err, "error squashing image")
}
return imageID, nil
}
components/engine/api/server/backend/build/tag.go
+77
View File
@@ -0,0 +1,77 @@
package build
import (
"fmt"
"io"
"github.com/docker/distribution/reference"
"github.com/docker/docker/image"
"github.com/pkg/errors"
)
// Tagger is responsible for tagging an image created by a builder
type Tagger struct {
imageComponent ImageComponent
stdout io.Writer
repoAndTags []reference.Named
}
// NewTagger returns a new Tagger for tagging the images of a build.
// If any of the names are invalid tags an error is returned.
func NewTagger(backend ImageComponent, stdout io.Writer, names []string) (*Tagger, error) {
reposAndTags, err := sanitizeRepoAndTags(names)
if err != nil {
return nil, err
}
return &Tagger{
imageComponent: backend,
stdout: stdout,
repoAndTags: reposAndTags,
}, nil
}
// TagImages creates image tags for the imageID
func (bt *Tagger) TagImages(imageID image.ID) error {
for _, rt := range bt.repoAndTags {
if err := bt.imageComponent.TagImageWithReference(imageID, rt); err != nil {
return err
}
fmt.Fprintf(bt.stdout, "Successfully tagged %s\n", reference.FamiliarString(rt))
}
return nil
}
// sanitizeRepoAndTags parses the raw "t" parameter received from the client
// to a slice of repoAndTag.
// It also validates each repoName and tag.
func sanitizeRepoAndTags(names []string) ([]reference.Named, error) {
var (
repoAndTags []reference.Named
// This map is used for deduplicating the "-t" parameter.
uniqNames = make(map[string]struct{})
)
for _, repo := range names {
if repo == "" {
continue
}
ref, err := reference.ParseNormalizedNamed(repo)
if err != nil {
return nil, err
}
if _, isCanonical := ref.(reference.Canonical); isCanonical {
return nil, errors.New("build tag cannot contain a digest")
}
ref = reference.TagNameOnly(ref)
nameWithTag := ref.String()
if _, exists := uniqNames[nameWithTag]; !exists {
uniqNames[nameWithTag] = struct{}{}
repoAndTags = append(repoAndTags, ref)
}
}
return repoAndTags, nil
}
components/engine/api/server/router/build/backend.go
+6 -9
View File
@@ -1,20 +1,17 @@
package build
import (
"io"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/backend"
"golang.org/x/net/context"
)
// Backend abstracts an image builder whose only purpose is to build an image referenced by an imageID.
type Backend interface {
// BuildFromContext builds a Docker image referenced by an imageID string.
//
// Note: Tagging an image should not be done by a Builder, it should instead be done
// by the caller.
//
// Build a Docker image returning the id of the image
// TODO: make this return a reference instead of string
BuildFromContext(ctx context.Context, src io.ReadCloser, buildOptions *types.ImageBuildOptions, pg backend.ProgressWriter) (string, error)
Build(context.Context, backend.BuildConfig) (string, error)
}
type experimentalProvider interface {
HasExperimental() bool
}
components/engine/api/server/router/build/build.go
+3 -4
View File
@@ -5,14 +5,13 @@ import "github.com/docker/docker/api/server/router"
// buildRouter is a router to talk with the build controller
type buildRouter struct {
backend Backend
daemon experimentalProvider
routes []router.Route
}
// NewRouter initializes a new build router
func NewRouter(b Backend) router.Router {
r := &buildRouter{
backend: b,
}
func NewRouter(b Backend, d experimentalProvider) router.Router {
r := &buildRouter{backend: b, daemon: d}
r.initRoutes()
return r
}
components/engine/api/server/router/build/build_routes.go
+59 -45
View File
@@ -13,6 +13,7 @@ import (
"sync"
"github.com/Sirupsen/logrus"
apierrors "github.com/docker/docker/api/errors"
"github.com/docker/docker/api/server/httputils"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/backend"
@@ -22,6 +23,7 @@ import (
"github.com/docker/docker/pkg/progress"
"github.com/docker/docker/pkg/streamformatter"
units "github.com/docker/go-units"
"github.com/pkg/errors"
"golang.org/x/net/context"
)
@@ -87,9 +89,6 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui
options.Ulimits = buildUlimits
}
var buildArgs = map[string]*string{}
buildArgsJSON := r.FormValue("buildargs")
// Note that there are two ways a --build-arg might appear in the
// json of the query param:
// "foo":"bar"
@@ -102,25 +101,27 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui
// the fact they mentioned it, we need to pass that along to the builder
// so that it can print a warning about "foo" being unused if there is
// no "ARG foo" in the Dockerfile.
buildArgsJSON := r.FormValue("buildargs")
if buildArgsJSON != "" {
var buildArgs = map[string]*string{}
if err := json.Unmarshal([]byte(buildArgsJSON), &buildArgs); err != nil {
return nil, err
}
options.BuildArgs = buildArgs
}
var labels = map[string]string{}
labelsJSON := r.FormValue("labels")
if labelsJSON != "" {
var labels = map[string]string{}
if err := json.Unmarshal([]byte(labelsJSON), &labels); err != nil {
return nil, err
}
options.Labels = labels
}
var cacheFrom = []string{}
cacheFromJSON := r.FormValue("cachefrom")
if cacheFromJSON != "" {
var cacheFrom = []string{}
if err := json.Unmarshal([]byte(cacheFromJSON), &cacheFrom); err != nil {
return nil, err
}
@@ -130,33 +131,8 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui
return options, nil
}
type syncWriter struct {
w io.Writer
mu sync.Mutex
}
func (s *syncWriter) Write(b []byte) (count int, err error) {
s.mu.Lock()
count, err = s.w.Write(b)
s.mu.Unlock()
return
}
func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
var (
authConfigs = map[string]types.AuthConfig{}
authConfigsEncoded = r.Header.Get("X-Registry-Config")
notVerboseBuffer = bytes.NewBuffer(nil)
)
if authConfigsEncoded != "" {
authConfigsJSON := base64.NewDecoder(base64.URLEncoding, strings.NewReader(authConfigsEncoded))
if err := json.NewDecoder(authConfigsJSON).Decode(&authConfigs); err != nil {
// for a pull it is not an error if no auth was given
// to increase compatibility with the existing api it is defaulting
// to be empty.
}
}
var notVerboseBuffer = bytes.NewBuffer(nil)
w.Header().Set("Content-Type", "application/json")
@@ -183,7 +159,12 @@ func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *
if err != nil {
return errf(err)
}
buildOptions.AuthConfigs = authConfigs
buildOptions.AuthConfigs = getAuthConfigs(r.Header)
if buildOptions.Squash && !br.daemon.HasExperimental() {
return apierrors.NewBadRequestError(
errors.New("squash is only supported with experimental mode"))
}
// Currently, only used if context is from a remote url.
// Look at code in DetectContextFromRemoteURL for more information.
@@ -199,18 +180,12 @@ func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *
if buildOptions.SuppressOutput {
out = notVerboseBuffer
}
out = &syncWriter{w: out}
stdout := &streamformatter.StdoutFormatter{Writer: out, StreamFormatter: sf}
stderr := &streamformatter.StderrFormatter{Writer: out, StreamFormatter: sf}
pg := backend.ProgressWriter{
Output: out,
StdoutFormatter: stdout,
StderrFormatter: stderr,
ProgressReaderFunc: createProgressReader,
}
imgID, err := br.backend.BuildFromContext(ctx, r.Body, buildOptions, pg)
imgID, err := br.backend.Build(ctx, backend.BuildConfig{
Source: r.Body,
Options: buildOptions,
ProgressWriter: buildProgressWriter(out, sf, createProgressReader),
})
if err != nil {
return errf(err)
}
@@ -219,8 +194,47 @@ func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *
// should be just the image ID and we'll print that to stdout.
if buildOptions.SuppressOutput {
stdout := &streamformatter.StdoutFormatter{Writer: output, StreamFormatter: sf}
fmt.Fprintf(stdout, "%s\n", string(imgID))
fmt.Fprintln(stdout, imgID)
}
return nil
}
func getAuthConfigs(header http.Header) map[string]types.AuthConfig {
authConfigs := map[string]types.AuthConfig{}
authConfigsEncoded := header.Get("X-Registry-Config")
if authConfigsEncoded == "" {
return authConfigs
}
authConfigsJSON := base64.NewDecoder(base64.URLEncoding, strings.NewReader(authConfigsEncoded))
// Pulling an image does not error when no auth is provided so to remain
// consistent with the existing api decode errors are ignored
json.NewDecoder(authConfigsJSON).Decode(&authConfigs)
return authConfigs
}
type syncWriter struct {
w io.Writer
mu sync.Mutex
}
func (s *syncWriter) Write(b []byte) (count int, err error) {
s.mu.Lock()
count, err = s.w.Write(b)
s.mu.Unlock()
return
}
func buildProgressWriter(out io.Writer, sf *streamformatter.StreamFormatter, createProgressReader func(io.ReadCloser) io.ReadCloser) backend.ProgressWriter {
out = &syncWriter{w: out}
stdout := &streamformatter.StdoutFormatter{Writer: out, StreamFormatter: sf}
stderr := &streamformatter.StderrFormatter{Writer: out, StreamFormatter: sf}
return backend.ProgressWriter{
Output: out,
StdoutFormatter: stdout,
StderrFormatter: stderr,
ProgressReaderFunc: createProgressReader,
}
}
components/engine/api/types/backend/backend.go
-9
View File
@@ -6,7 +6,6 @@ import (
"time"
"github.com/docker/docker/api/types"
"github.com/docker/docker/pkg/streamformatter"
)
// ContainerAttachConfig holds the streams to use when connecting to a container to view logs.
@@ -99,11 +98,3 @@ type ContainerCommitConfig struct {
types.ContainerCommitConfig
Changes []string
}
// ProgressWriter is a data object to transport progress streams to the client
type ProgressWriter struct {
Output io.Writer
StdoutFormatter *streamformatter.StdoutFormatter
StderrFormatter *streamformatter.StderrFormatter
ProgressReaderFunc func(io.ReadCloser) io.ReadCloser
}
components/engine/api/types/backend/build.go
+23
View File
@@ -0,0 +1,23 @@
package backend
import (
"io"
"github.com/docker/docker/api/types"
"github.com/docker/docker/pkg/streamformatter"
)
// ProgressWriter is a data object to transport progress streams to the client
type ProgressWriter struct {
Output io.Writer
StdoutFormatter *streamformatter.StdoutFormatter
StderrFormatter *streamformatter.StderrFormatter
ProgressReaderFunc func(io.ReadCloser) io.ReadCloser
}
// BuildConfig is the configuration used by a BuildManager to start a build
type BuildConfig struct {
Source io.ReadCloser
ProgressWriter ProgressWriter
Options *types.ImageBuildOptions
}