Add docker build --iidfile=FILE

This is synonymous with `docker run --cidfile=FILE` and writes the digest of
the newly built image to the named file. This is intended to be used by build
systems which want to avoid tagging (perhaps because they are in CI or
otherwise want to avoid fixed names which can clash) by enabling e.g. Makefile
constructs like:

    image.id: Dockerfile
    	docker build --iidfile=image.id .

    do-some-more-stuff: image.id
    	do-stuff-with <image.id

Currently the only way to achieve this is to use `docker build -q` and capture
the stdout, but at the expense of losing the build output.

In non-silent mode (without `-q`) with API >= v1.29 the caller will now see a
`JSONMessage` with the `Aux` field containing a `types.BuildResult` in the
output stream for each image/layer produced during the build, with the final
one being the end product.  Having all of the intermediate images might be
interesting in some cases.

In silent mode (with `-q`) there is no change, on success the only output will
be the resulting image digest as it was previosuly.

There was no wrapper to just output an Aux section without enclosing it in a
Progress, so add one here.

Added some tests to integration cli tests.

Signed-off-by: Ian Campbell <ian.campbell@docker.com>
Upstream-commit: 5894bc1abf8186802d360d20739b57bfffed51df
Component: engine

components/engine/api/server/router/build/build_routes.go

@@ -132,7 +132,10 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui
 }
 
 func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
-	var notVerboseBuffer = bytes.NewBuffer(nil)
+	var (
+		notVerboseBuffer = bytes.NewBuffer(nil)
+		version          = httputils.VersionFromContext(ctx)
+	)
 
 	w.Header().Set("Content-Type", "application/json")
 
@@ -177,10 +180,12 @@ func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *
 		return progress.NewProgressReader(in, progressOutput, r.ContentLength, "Downloading context", buildOptions.RemoteContext)
 	}
 
+	wantAux := versions.GreaterThanOrEqualTo(version, "1.30")
+
 	imgID, err := br.backend.Build(ctx, backend.BuildConfig{
 		Source:         r.Body,
 		Options:        buildOptions,
-		ProgressWriter: buildProgressWriter(out, createProgressReader),
+		ProgressWriter: buildProgressWriter(out, wantAux, createProgressReader),
 	})
 	if err != nil {
 		return errf(err)
@@ -221,13 +226,19 @@ func (s *syncWriter) Write(b []byte) (count int, err error) {
 	return
 }
 
-func buildProgressWriter(out io.Writer, createProgressReader func(io.ReadCloser) io.ReadCloser) backend.ProgressWriter {
+func buildProgressWriter(out io.Writer, wantAux bool, createProgressReader func(io.ReadCloser) io.ReadCloser) backend.ProgressWriter {
 	out = &syncWriter{w: out}
 
+	var aux *streamformatter.AuxFormatter
+	if wantAux {
+		aux = &streamformatter.AuxFormatter{Writer: out}
+	}
+
 	return backend.ProgressWriter{
 		Output:             out,
 		StdoutFormatter:    streamformatter.NewStdoutWriter(out),
 		StderrFormatter:    streamformatter.NewStderrWriter(out),
+		AuxFormatter:       aux,
 		ProgressReaderFunc: createProgressReader,
 	}
 }

components/engine/api/types/backend/build.go

@@ -4,6 +4,7 @@ import (
 	"io"
 
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/pkg/streamformatter"
 )
 
 // ProgressWriter is a data object to transport progress streams to the client
@@ -11,6 +12,7 @@ type ProgressWriter struct {
 	Output             io.Writer
 	StdoutFormatter    io.Writer
 	StderrFormatter    io.Writer
+	AuxFormatter       *streamformatter.AuxFormatter
 	ProgressReaderFunc func(io.ReadCloser) io.ReadCloser
 }
 

components/engine/api/types/types.go

@@ -530,3 +530,8 @@ type PushResult struct {
 	Digest string
 	Size   int
 }
+
+// BuildResult contains the image id of a successful build
+type BuildResult struct {
+	ID string
+}

components/engine/builder/dockerfile/builder.go

@@ -15,6 +15,7 @@ import (
 	"github.com/docker/docker/builder/dockerfile/command"
 	"github.com/docker/docker/builder/dockerfile/parser"
 	"github.com/docker/docker/builder/remotecontext"
+	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/docker/docker/pkg/stringid"
 	"github.com/pkg/errors"
 	"golang.org/x/net/context"
@@ -89,6 +90,7 @@ type Builder struct {
 
 	Stdout io.Writer
 	Stderr io.Writer
+	Aux    *streamformatter.AuxFormatter
 	Output io.Writer
 
 	docker    builder.Backend
@@ -114,6 +116,7 @@ func newBuilder(clientCtx context.Context, options builderOptions) *Builder {
 		options:       config,
 		Stdout:        options.ProgressWriter.StdoutFormatter,
 		Stderr:        options.ProgressWriter.StderrFormatter,
+		Aux:           options.ProgressWriter.AuxFormatter,
 		Output:        options.ProgressWriter.Output,
 		docker:        options.Backend,
 		tmpContainers: map[string]struct{}{},
@@ -161,6 +164,13 @@ func (b *Builder) build(source builder.Source, dockerfile *parser.Result) (*buil
 	return &builder.Result{ImageID: dispatchState.imageID, FromImage: dispatchState.baseImage}, nil
 }
 
+func emitImageID(aux *streamformatter.AuxFormatter, state *dispatchState) error {
+	if aux == nil || state.imageID == "" {
+		return nil
+	}
+	return aux.Emit(types.BuildResult{ID: state.imageID})
+}
+
 func (b *Builder) dispatchDockerfileWithCancellation(dockerfile *parser.Result) (*dispatchState, error) {
 	shlex := NewShellLex(dockerfile.EscapeToken)
 	state := newDispatchState()
@@ -176,6 +186,15 @@ func (b *Builder) dispatchDockerfileWithCancellation(dockerfile *parser.Result)
 			// Not cancelled yet, keep going...
 		}
 
+		// If this is a FROM and we have a previous image then
+		// emit an aux message for that image since it is the
+		// end of the previous stage
+		if n.Value == command.From {
+			if err := emitImageID(b.Aux, state); err != nil {
+				return nil, err
+			}
+		}
+
 		if n.Value == command.From && state.isCurrentStage(b.options.Target) {
 			break
 		}
@@ -198,6 +217,12 @@ func (b *Builder) dispatchDockerfileWithCancellation(dockerfile *parser.Result)
 			b.clearTmp()
 		}
 	}
+
+	// Emit a final aux message for the final image
+	if err := emitImageID(b.Aux, state); err != nil {
+		return nil, err
+	}
+
 	return state, nil
 }
 

components/engine/cli/command/image/build.go

@@ -4,6 +4,7 @@ import (
 	"archive/tar"
 	"bufio"
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -65,6 +66,7 @@ type buildOptions struct {
 	networkMode    string
 	squash         bool
 	target         string
+	imageIDFile    string
 }
 
 // NewBuildCommand creates a new `docker build` command
@@ -117,6 +119,7 @@ func NewBuildCommand(dockerCli *command.DockerCli) *cobra.Command {
 	flags.SetAnnotation("network", "version", []string{"1.25"})
 	flags.Var(&options.extraHosts, "add-host", "Add a custom host-to-IP mapping (host:ip)")
 	flags.StringVar(&options.target, "target", "", "Set the target build stage to build.")
+	flags.StringVar(&options.imageIDFile, "iidfile", "", "Write the image ID to the file")
 
 	command.AddTrustVerificationFlags(flags)
 
@@ -162,6 +165,12 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
 		progBuff = bytes.NewBuffer(nil)
 		buildBuff = bytes.NewBuffer(nil)
 	}
+	if options.imageIDFile != "" {
+		// Avoid leaving a stale file if we eventually fail
+		if err := os.Remove(options.imageIDFile); err != nil && !os.IsNotExist(err) {
+			return errors.Wrap(err, "Removing image ID file")
+		}
+	}
 
 	if options.dockerfileName == "-" {
 		if specifiedContext == "-" {
@@ -316,7 +325,17 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
 	}
 	defer response.Body.Close()
 
-	err = jsonmessage.DisplayJSONMessagesStream(response.Body, buildBuff, dockerCli.Out().FD(), dockerCli.Out().IsTerminal(), nil)
+	imageID := ""
+	aux := func(auxJSON *json.RawMessage) {
+		var result types.BuildResult
+		if err := json.Unmarshal(*auxJSON, &result); err != nil {
+			fmt.Fprintf(dockerCli.Err(), "Failed to parse aux message: %s", err)
+		} else {
+			imageID = result.ID
+		}
+	}
+
+	err = jsonmessage.DisplayJSONMessagesStream(response.Body, buildBuff, dockerCli.Out().FD(), dockerCli.Out().IsTerminal(), aux)
 	if err != nil {
 		if jerr, ok := err.(*jsonmessage.JSONError); ok {
 			// If no error code is set, default to 1
@@ -344,9 +363,18 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
 	// Everything worked so if -q was provided the output from the daemon
 	// should be just the image ID and we'll print that to stdout.
 	if options.quiet {
-		fmt.Fprintf(dockerCli.Out(), "%s", buildBuff)
+		imageID = fmt.Sprintf("%s", buildBuff)
+		fmt.Fprintf(dockerCli.Out(), imageID)
 	}
 
+	if options.imageIDFile != "" {
+		if imageID == "" {
+			return errors.Errorf("Server did not provide an image ID. Cannot write %s", options.imageIDFile)
+		}
+		if err := ioutil.WriteFile(options.imageIDFile, []byte(imageID), 0666); err != nil {
+			return err
+		}
+	}
 	if command.IsTrusted() {
 		// Since the build was successful, now we must tag any of the resolved
 		// images from the above Dockerfile rewrite.

components/engine/docs/api/version-history.md

@@ -18,6 +18,7 @@ keywords: "API, Docker, rcli, REST, documentation"
 [Docker Engine API v1.30](https://docs.docker.com/engine/api/v1.30/) documentation
 
 * `GET /info` now returns the list of supported logging drivers, including plugins.
+* `POST /build/` now (when not silent) produces an `Aux` message in the JSON output stream with payload `types.BuildResult` for each image produced. The final such message will reference the image resulting from the build.
 
 ## v1.29 API changes
 

components/engine/docs/reference/commandline/build.md

@@ -35,6 +35,7 @@ Options:
   -f, --file string             Name of the Dockerfile (Default is 'PATH/Dockerfile')
       --force-rm                Always remove intermediate containers
       --help                    Print usage
+      --iidfile string          Write the image ID to the file
       --isolation string        Container isolation technology
       --label value             Set metadata for an image (default [])
   -m, --memory string           Memory limit

components/engine/integration-cli/docker_cli_build_test.go

@@ -28,6 +28,7 @@ import (
 	"github.com/docker/docker/pkg/testutil"
 	icmd "github.com/docker/docker/pkg/testutil/cmd"
 	"github.com/go-check/check"
+	"github.com/opencontainers/go-digest"
 )
 
 func (s *DockerSuite) TestBuildJSONEmptyRun(c *check.C) {
@@ -6398,3 +6399,49 @@ CMD echo foo
 	out, _ := dockerCmd(c, "inspect", "--format", "{{ json .Config.Cmd }}", "build2")
 	c.Assert(strings.TrimSpace(out), checker.Equals, `["/bin/sh","-c","echo foo"]`)
 }
+
+func (s *DockerSuite) TestBuildIidFile(c *check.C) {
+	tmpDir, err := ioutil.TempDir("", "TestBuildIidFile")
+	if err != nil {
+		c.Fatal(err)
+	}
+	defer os.RemoveAll(tmpDir)
+	tmpIidFile := filepath.Join(tmpDir, "iid")
+
+	name := "testbuildiidfile"
+	// Use a Dockerfile with multiple stages to ensure we get the last one
+	cli.BuildCmd(c, name,
+		build.WithDockerfile(`FROM `+minimalBaseImage()+` AS stage1
+ENV FOO FOO
+FROM `+minimalBaseImage()+`
+ENV BAR BAZ`),
+		cli.WithFlags("--iidfile", tmpIidFile))
+
+	id, err := ioutil.ReadFile(tmpIidFile)
+	c.Assert(err, check.IsNil)
+	d, err := digest.Parse(string(id))
+	c.Assert(err, check.IsNil)
+	c.Assert(d.String(), checker.Equals, getIDByName(c, name))
+}
+
+func (s *DockerSuite) TestBuildIidFileCleanupOnFail(c *check.C) {
+	tmpDir, err := ioutil.TempDir("", "TestBuildIidFileCleanupOnFail")
+	if err != nil {
+		c.Fatal(err)
+	}
+	defer os.RemoveAll(tmpDir)
+	tmpIidFile := filepath.Join(tmpDir, "iid")
+
+	err = ioutil.WriteFile(tmpIidFile, []byte("Dummy"), 0666)
+	c.Assert(err, check.IsNil)
+
+	cli.Docker(cli.Build("testbuildiidfilecleanuponfail"),
+		build.WithDockerfile(`FROM `+minimalBaseImage()+`
+	RUN /non/existing/command`),
+		cli.WithFlags("--iidfile", tmpIidFile)).Assert(c, icmd.Expected{
+		ExitCode: 1,
+	})
+	_, err = os.Stat(tmpIidFile)
+	c.Assert(err, check.NotNil)
+	c.Assert(os.IsNotExist(err), check.Equals, true)
+}

components/engine/man/docker-build.1.md

@@ -11,6 +11,7 @@ docker-build - Build an image from a Dockerfile
 [**--cpu-shares**[=*0*]]
 [**--cgroup-parent**[=*CGROUP-PARENT*]]
 [**--help**]
+[**--iidfile**[=*CIDFILE*]]
 [**-f**|**--file**[=*PATH/Dockerfile*]]
 [**-squash**] *Experimental*
 [**--force-rm**]
@@ -104,6 +105,9 @@ option can be set multiple times.
 **--no-cache**=*true*|*false*
    Do not use cache when building the image. The default is *false*.
 
+**--iidfile**=""
+   Write the image ID to the file
+
 **--help**
   Print usage statement
 

components/engine/pkg/jsonmessage/jsonmessage.go

@@ -109,7 +109,7 @@ type JSONMessage struct {
 	TimeNano        int64         `json:"timeNano,omitempty"`
 	Error           *JSONError    `json:"errorDetail,omitempty"`
 	ErrorMessage    string        `json:"error,omitempty"` //deprecated
-	// Aux contains out-of-band data, such as digests for push signing.
+	// Aux contains out-of-band data, such as digests for push signing and image id after building.
 	Aux *json.RawMessage `json:"aux,omitempty"`
 }
 

components/engine/pkg/streamformatter/streamformatter.go

@@ -132,3 +132,28 @@ func (out *progressOutput) WriteProgress(prog progress.Progress) error {
 
 	return nil
 }
+
+// AuxFormatter is a streamFormatter that writes aux progress messages
+type AuxFormatter struct {
+	io.Writer
+}
+
+// Emit emits the given interface as an aux progress message
+func (sf *AuxFormatter) Emit(aux interface{}) error {
+	auxJSONBytes, err := json.Marshal(aux)
+	if err != nil {
+		return err
+	}
+	auxJSON := new(json.RawMessage)
+	*auxJSON = auxJSONBytes
+	msgJSON, err := json.Marshal(&jsonmessage.JSONMessage{Aux: auxJSON})
+	if err != nil {
+		return err
+	}
+	msgJSON = appendNewline(msgJSON)
+	n, err := sf.Writer.Write(msgJSON)
+	if n != len(msgJSON) {
+		return io.ErrShortWrite
+	}
+	return err
+}