From 66621995e009ec0cef4a99cb17bcf9c8f96a2fad Mon Sep 17 00:00:00 2001 From: John Howard Date: Tue, 3 Oct 2017 10:20:41 -0700 Subject: [PATCH 1/9] Builder - parser - remove OS Signed-off-by: John Howard Upstream-commit: 98dd1fdca1f5b82cbc7066c4a48f9ddd8f135095 Component: engine --- components/engine/builder/dockerfile/parser/parser.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/components/engine/builder/dockerfile/parser/parser.go b/components/engine/builder/dockerfile/parser/parser.go index 277176ee1c..1ff6fba560 100644 --- a/components/engine/builder/dockerfile/parser/parser.go +++ b/components/engine/builder/dockerfile/parser/parser.go @@ -237,10 +237,7 @@ func newNodeFromLine(line string, directive *Directive) (*Node, error) { type Result struct { AST *Node EscapeToken rune - // TODO @jhowardmsft - see https://github.com/moby/moby/issues/34617 - // This next field will be removed in a future update for LCOW support. - OS string - Warnings []string + Warnings []string } // PrintWarnings to the writer @@ -320,7 +317,6 @@ func Parse(rwc io.Reader) (*Result, error) { AST: root, Warnings: warnings, EscapeToken: d.escapeToken, - OS: d.platformToken, }, handleScannerError(scanner.Err()) } From bf670c643552df2209d75d0aaaf5d26a7b8d1cca Mon Sep 17 00:00:00 2001 From: John Howard Date: Tue, 3 Oct 2017 11:32:54 -0700 Subject: [PATCH 2/9] Builder - dockerfile - just use API for now, and unit test fix Signed-off-by: John Howard Upstream-commit: 735e5d22b7ca208acc9ad7373bb8f93167ee3f85 Component: engine --- components/engine/builder/dockerfile/builder.go | 12 +++--------- .../engine/builder/dockerfile/dispatchers_test.go | 1 + 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/components/engine/builder/dockerfile/builder.go b/components/engine/builder/dockerfile/builder.go index d20bc0403a..2a41c1f0c3 100644 --- a/components/engine/builder/dockerfile/builder.go +++ b/components/engine/builder/dockerfile/builder.go @@ -105,17 +105,11 @@ func (bm *BuildManager) Build(ctx context.Context, config backend.BuildConfig) ( } os := runtime.GOOS - optionsPlatform := system.ParsePlatform(config.Options.Platform) - if dockerfile.OS != "" { - if optionsPlatform.OS != "" && optionsPlatform.OS != dockerfile.OS { - return nil, fmt.Errorf("invalid platform") - } - os = dockerfile.OS - } else if optionsPlatform.OS != "" { - os = optionsPlatform.OS + apiPlatform := system.ParsePlatform(config.Options.Platform) + if apiPlatform.OS != "" { + os = apiPlatform.OS } config.Options.Platform = os - dockerfile.OS = os builderOptions := builderOptions{ Options: config.Options, diff --git a/components/engine/builder/dockerfile/dispatchers_test.go b/components/engine/builder/dockerfile/dispatchers_test.go index 6d52e7e619..7ad0d8d3c2 100644 --- a/components/engine/builder/dockerfile/dispatchers_test.go +++ b/components/engine/builder/dockerfile/dispatchers_test.go @@ -208,6 +208,7 @@ func TestOnbuild(t *testing.T) { func TestWorkdir(t *testing.T) { b := newBuilderWithMockBackend() sb := newDispatchRequest(b, '`', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults()) + sb.state.baseImage = &mockImage{} workingDir := "/app" if runtime.GOOS == "windows" { workingDir = "C:\\app" From 96ca7d0f5983317d1ffa3fb0fdbc0d5edcec9289 Mon Sep 17 00:00:00 2001 From: John Howard Date: Tue, 3 Oct 2017 11:38:37 -0700 Subject: [PATCH 3/9] Builder - Parser. Remove platform parser directive Signed-off-by: John Howard Upstream-commit: 9cae03900fc27ff39e913978ca8f084691954881 Component: engine --- .../builder/dockerfile/parser/parser.go | 51 +++---------------- 1 file changed, 7 insertions(+), 44 deletions(-) diff --git a/components/engine/builder/dockerfile/parser/parser.go b/components/engine/builder/dockerfile/parser/parser.go index 1ff6fba560..b065b8a4ea 100644 --- a/components/engine/builder/dockerfile/parser/parser.go +++ b/components/engine/builder/dockerfile/parser/parser.go @@ -7,13 +7,11 @@ import ( "fmt" "io" "regexp" - "runtime" "strconv" "strings" "unicode" "github.com/docker/docker/builder/dockerfile/command" - "github.com/docker/docker/pkg/system" "github.com/pkg/errors" ) @@ -81,11 +79,10 @@ func (node *Node) AddChild(child *Node, startLine, endLine int) { } var ( - dispatch map[string]func(string, *Directive) (*Node, map[string]bool, error) - tokenWhitespace = regexp.MustCompile(`[\t\v\f\r ]+`) - tokenEscapeCommand = regexp.MustCompile(`^#[ \t]*escape[ \t]*=[ \t]*(?P.).*$`) - tokenPlatformCommand = regexp.MustCompile(`^#[ \t]*platform[ \t]*=[ \t]*(?P.*)$`) - tokenComment = regexp.MustCompile(`^#.*$`) + dispatch map[string]func(string, *Directive) (*Node, map[string]bool, error) + tokenWhitespace = regexp.MustCompile(`[\t\v\f\r ]+`) + tokenEscapeCommand = regexp.MustCompile(`^#[ \t]*escape[ \t]*=[ \t]*(?P.).*$`) + tokenComment = regexp.MustCompile(`^#.*$`) ) // DefaultEscapeToken is the default escape token @@ -95,11 +92,9 @@ const DefaultEscapeToken = '\\' // parsing directives. type Directive struct { escapeToken rune // Current escape token - platformToken string // Current platform token lineContinuationRegex *regexp.Regexp // Current line continuation regex processingComplete bool // Whether we are done looking for directives escapeSeen bool // Whether the escape directive has been seen - platformSeen bool // Whether the platform directive has been seen } // setEscapeToken sets the default token for escaping characters in a Dockerfile. @@ -112,25 +107,9 @@ func (d *Directive) setEscapeToken(s string) error { return nil } -// setPlatformToken sets the default platform for pulling images in a Dockerfile. -func (d *Directive) setPlatformToken(s string) error { - s = strings.ToLower(s) - valid := []string{runtime.GOOS} - if system.LCOWSupported() { - valid = append(valid, "linux") - } - for _, item := range valid { - if s == item { - d.platformToken = s - return nil - } - } - return fmt.Errorf("invalid PLATFORM '%s'. Must be one of %v", s, valid) -} - -// possibleParserDirective looks for one or more parser directives '# escapeToken=' and -// '# platform='. Parser directives must precede any builder instruction -// or other comments, and cannot be repeated. +// possibleParserDirective looks for parser directives, eg '# escapeToken='. +// Parser directives must precede any builder instruction or other comments, +// and cannot be repeated. func (d *Directive) possibleParserDirective(line string) error { if d.processingComplete { return nil @@ -149,22 +128,6 @@ func (d *Directive) possibleParserDirective(line string) error { } } - // Only recognise a platform token if LCOW is supported - if system.LCOWSupported() { - tpcMatch := tokenPlatformCommand.FindStringSubmatch(strings.ToLower(line)) - if len(tpcMatch) != 0 { - for i, n := range tokenPlatformCommand.SubexpNames() { - if n == "platform" { - if d.platformSeen { - return errors.New("only one platform parser directive can be used") - } - d.platformSeen = true - return d.setPlatformToken(tpcMatch[i]) - } - } - } - } - d.processingComplete = true return nil } From fe74b4e69a70ef2c454d317fb838d2e3d528ba41 Mon Sep 17 00:00:00 2001 From: John Howard Date: Tue, 3 Oct 2017 15:34:33 -0700 Subject: [PATCH 4/9] Builder - add --platform to FROM statement Signed-off-by: John Howard Upstream-commit: 7f0c2d23e11485c7f026dd8c111c60c2e1e03375 Component: engine --- .../engine/builder/dockerfile/dispatchers.go | 2 +- .../dockerfile/instructions/commands.go | 9 ++++--- .../builder/dockerfile/instructions/parse.go | 27 +++++++++++++++---- .../dockerfile/instructions/parse_test.go | 13 ++++++++- 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/components/engine/builder/dockerfile/dispatchers.go b/components/engine/builder/dockerfile/dispatchers.go index 99d4aa627e..38e6f768c4 100644 --- a/components/engine/builder/dockerfile/dispatchers.go +++ b/components/engine/builder/dockerfile/dispatchers.go @@ -148,7 +148,7 @@ func (d *dispatchRequest) getImageMount(imageRefOrID string) (*imageMount, error return d.builder.imageSources.Get(imageRefOrID, localOnly) } -// FROM imagename[:tag | @digest] [AS build-stage-name] +// FROM [--platform=platform] imagename[:tag | @digest] [AS build-stage-name] // func initializeStage(d dispatchRequest, cmd *instructions.Stage) error { d.builder.imageProber.Reset() diff --git a/components/engine/builder/dockerfile/instructions/commands.go b/components/engine/builder/dockerfile/instructions/commands.go index d4f55ceb43..44382dc61e 100644 --- a/components/engine/builder/dockerfile/instructions/commands.go +++ b/components/engine/builder/dockerfile/instructions/commands.go @@ -357,10 +357,11 @@ type ShellCommand struct { // Stage represents a single stage in a multi-stage build type Stage struct { - Name string - Commands []Command - BaseName string - SourceCode string + Name string + Commands []Command + BaseName string + SourceCode string + OperatingSystem string } // AddCommand to the stage diff --git a/components/engine/builder/dockerfile/instructions/parse.go b/components/engine/builder/dockerfile/instructions/parse.go index 9226f4d46e..9b94ee3660 100644 --- a/components/engine/builder/dockerfile/instructions/parse.go +++ b/components/engine/builder/dockerfile/instructions/parse.go @@ -3,6 +3,7 @@ package instructions // import "github.com/docker/docker/builder/dockerfile/inst import ( "fmt" "regexp" + "runtime" "sort" "strconv" "strings" @@ -12,6 +13,7 @@ import ( "github.com/docker/docker/api/types/strslice" "github.com/docker/docker/builder/dockerfile/command" "github.com/docker/docker/builder/dockerfile/parser" + "github.com/docker/docker/pkg/system" "github.com/pkg/errors" ) @@ -271,16 +273,31 @@ func parseFrom(req parseRequest) (*Stage, error) { return nil, err } + flPlatform := req.flags.AddString("platform", "") if err := req.flags.Parse(); err != nil { return nil, err } + specPlatform := system.ParsePlatform(flPlatform.Value) + if specPlatform.OS == "" { + specPlatform.OS = runtime.GOOS + } + if err := system.ValidatePlatform(specPlatform); err != nil { + return nil, fmt.Errorf("invalid platform %q on FROM", flPlatform.Value) + } + if !system.IsOSSupported(specPlatform.OS) { + return nil, fmt.Errorf("unsupported platform %q on FROM", flPlatform.Value) + } + if err != nil { + return nil, err + } code := strings.TrimSpace(req.original) - + fmt.Println("JJH", specPlatform.OS) return &Stage{ - BaseName: req.args[0], - Name: stageName, - SourceCode: code, - Commands: []Command{}, + BaseName: req.args[0], + Name: stageName, + SourceCode: code, + Commands: []Command{}, + OperatingSystem: specPlatform.OS, }, nil } diff --git a/components/engine/builder/dockerfile/instructions/parse_test.go b/components/engine/builder/dockerfile/instructions/parse_test.go index ffd6d4f45c..0c27f203d0 100644 --- a/components/engine/builder/dockerfile/instructions/parse_test.go +++ b/components/engine/builder/dockerfile/instructions/parse_test.go @@ -1,6 +1,8 @@ package instructions // import "github.com/docker/docker/builder/dockerfile/instructions" import ( + "fmt" + "runtime" "strings" "testing" @@ -184,6 +186,16 @@ func TestErrorCases(t *testing.T) { dockerfile: `foo bar`, expectedError: "unknown instruction: FOO", }, + { + name: "Invalid platform", + dockerfile: `FROM --platform=invalid busybox`, + expectedError: `invalid platform "invalid"`, + }, + { + name: "Only OS", + dockerfile: fmt.Sprintf(`FROM --platform=%s/%s busybox`, runtime.GOOS, runtime.GOARCH), + expectedError: `invalid platform`, + }, } for _, c := range cases { r := strings.NewReader(c.dockerfile) @@ -196,5 +208,4 @@ func TestErrorCases(t *testing.T) { _, err = ParseInstruction(n) testutil.ErrorContains(t, err, c.expectedError) } - } From d7c0222450e3687d1c427e0eb7bd57c753dc3793 Mon Sep 17 00:00:00 2001 From: John Howard Date: Wed, 4 Oct 2017 14:26:56 -0700 Subject: [PATCH 5/9] Builder: Plumbing through platform in `FROM` statement Signed-off-by: John Howard Upstream-commit: 69fa84bc3d57dafd19800642c5ba196bc6d45f90 Component: engine --- .../engine/builder/dockerfile/builder.go | 3 +- .../engine/builder/dockerfile/dispatchers.go | 55 +++++++++++++------ .../builder/dockerfile/dispatchers_test.go | 3 + .../engine/builder/dockerfile/evaluator.go | 3 +- .../engine/builder/dockerfile/imagecontext.go | 12 ++-- .../builder/dockerfile/instructions/parse.go | 7 +-- .../engine/builder/dockerfile/internals.go | 10 ++-- 7 files changed, 53 insertions(+), 40 deletions(-) diff --git a/components/engine/builder/dockerfile/builder.go b/components/engine/builder/dockerfile/builder.go index 2a41c1f0c3..d328235a14 100644 --- a/components/engine/builder/dockerfile/builder.go +++ b/components/engine/builder/dockerfile/builder.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "io/ioutil" - "runtime" "strings" "time" @@ -104,7 +103,7 @@ func (bm *BuildManager) Build(ctx context.Context, config backend.BuildConfig) ( source = src } - os := runtime.GOOS + os := "" apiPlatform := system.ParsePlatform(config.Options.Platform) if apiPlatform.OS != "" { os = apiPlatform.OS diff --git a/components/engine/builder/dockerfile/dispatchers.go b/components/engine/builder/dockerfile/dispatchers.go index 38e6f768c4..33e6bd4c23 100644 --- a/components/engine/builder/dockerfile/dispatchers.go +++ b/components/engine/builder/dockerfile/dispatchers.go @@ -145,14 +145,14 @@ func (d *dispatchRequest) getImageMount(imageRefOrID string) (*imageMount, error imageRefOrID = stage.Image localOnly = true } - return d.builder.imageSources.Get(imageRefOrID, localOnly) + return d.builder.imageSources.Get(imageRefOrID, localOnly, d.state.baseImage.OperatingSystem()) } // FROM [--platform=platform] imagename[:tag | @digest] [AS build-stage-name] // func initializeStage(d dispatchRequest, cmd *instructions.Stage) error { d.builder.imageProber.Reset() - image, err := d.getFromImage(d.shlex, cmd.BaseName) + image, err := d.getFromImage(d.shlex, cmd.BaseName, cmd.OperatingSystem) if err != nil { return err } @@ -210,20 +210,44 @@ func (d *dispatchRequest) getExpandedImageName(shlex *shell.Lex, name string) (s } return name, nil } -func (d *dispatchRequest) getImageOrStage(name string) (builder.Image, error) { + +// getOsFromFlagsAndStage calculates the operating system if we need to pull an image. +// stagePlatform contains the value supplied by optional `--platform=` on +// a current FROM statement. b.builder.options.Platform contains the operating +// system part of the optional flag passed in the API call (or CLI flag +// through `docker build --platform=...`). +func (d *dispatchRequest) getOsFromFlagsAndStage(stagePlatform string) string { + osForPull := "" + // First, take the API platform if nothing provided on FROM + if stagePlatform == "" && d.builder.options.Platform != "" { + osForPull = d.builder.options.Platform + } + // Next, use the FROM flag if that was provided + if osForPull == "" && stagePlatform != "" { + osForPull = stagePlatform + } + // Finally, assume the host OS + if osForPull == "" { + osForPull = runtime.GOOS + } + return osForPull +} + +func (d *dispatchRequest) getImageOrStage(name string, stagePlatform string) (builder.Image, error) { var localOnly bool if im, ok := d.stages.getByName(name); ok { name = im.Image localOnly = true } + os := d.getOsFromFlagsAndStage(stagePlatform) + // Windows cannot support a container with no base image unless it is LCOW. if name == api.NoBaseImageSpecifier { imageImage := &image.Image{} imageImage.OS = runtime.GOOS if runtime.GOOS == "windows" { - optionsOS := system.ParsePlatform(d.builder.options.Platform).OS - switch optionsOS { + switch os { case "windows", "": return nil, errors.New("Windows does not support FROM scratch") case "linux": @@ -232,23 +256,23 @@ func (d *dispatchRequest) getImageOrStage(name string) (builder.Image, error) { } imageImage.OS = "linux" default: - return nil, errors.Errorf("operating system %q is not supported", optionsOS) + return nil, errors.Errorf("operating system %q is not supported", os) } } return builder.Image(imageImage), nil } - imageMount, err := d.builder.imageSources.Get(name, localOnly) + imageMount, err := d.builder.imageSources.Get(name, localOnly, os) if err != nil { return nil, err } return imageMount.Image(), nil } -func (d *dispatchRequest) getFromImage(shlex *shell.Lex, name string) (builder.Image, error) { +func (d *dispatchRequest) getFromImage(shlex *shell.Lex, name string, stagePlatform string) (builder.Image, error) { name, err := d.getExpandedImageName(shlex, name) if err != nil { return nil, err } - return d.getImageOrStage(name) + return d.getImageOrStage(name, stagePlatform) } func dispatchOnbuild(d dispatchRequest, c *instructions.OnbuildCommand) error { @@ -264,8 +288,7 @@ func dispatchOnbuild(d dispatchRequest, c *instructions.OnbuildCommand) error { func dispatchWorkdir(d dispatchRequest, c *instructions.WorkdirCommand) error { runConfig := d.state.runConfig var err error - baseImageOS := system.ParsePlatform(d.state.operatingSystem).OS - runConfig.WorkingDir, err = normalizeWorkdir(baseImageOS, runConfig.WorkingDir, c.Path) + runConfig.WorkingDir, err = normalizeWorkdir(d.state.baseImage.OperatingSystem(), runConfig.WorkingDir, c.Path) if err != nil { return err } @@ -281,7 +304,7 @@ func dispatchWorkdir(d dispatchRequest, c *instructions.WorkdirCommand) error { } comment := "WORKDIR " + runConfig.WorkingDir - runConfigWithCommentCmd := copyRunConfig(runConfig, withCmdCommentString(comment, baseImageOS)) + runConfigWithCommentCmd := copyRunConfig(runConfig, withCmdCommentString(comment, d.state.baseImage.OperatingSystem())) containerID, err := d.builder.probeAndCreate(d.state, runConfigWithCommentCmd) if err != nil || containerID == "" { return err @@ -316,7 +339,7 @@ func dispatchRun(d dispatchRequest, c *instructions.RunCommand) error { return system.ErrNotSupportedOperatingSystem } stateRunConfig := d.state.runConfig - cmdFromArgs := resolveCmdLine(c.ShellDependantCmdLine, stateRunConfig, d.state.operatingSystem) + cmdFromArgs := resolveCmdLine(c.ShellDependantCmdLine, stateRunConfig, d.state.baseImage.OperatingSystem()) buildArgs := d.state.buildArgs.FilterAllowed(stateRunConfig.Env) saveCmd := cmdFromArgs @@ -397,8 +420,7 @@ func prependEnvOnCmd(buildArgs *buildArgs, buildArgVars []string, cmd strslice.S // func dispatchCmd(d dispatchRequest, c *instructions.CmdCommand) error { runConfig := d.state.runConfig - optionsOS := system.ParsePlatform(d.builder.options.Platform).OS - cmd := resolveCmdLine(c.ShellDependantCmdLine, runConfig, optionsOS) + cmd := resolveCmdLine(c.ShellDependantCmdLine, runConfig, d.state.baseImage.OperatingSystem()) runConfig.Cmd = cmd // set config as already being escaped, this prevents double escaping on windows runConfig.ArgsEscaped = true @@ -441,8 +463,7 @@ func dispatchHealthcheck(d dispatchRequest, c *instructions.HealthCheckCommand) // func dispatchEntrypoint(d dispatchRequest, c *instructions.EntrypointCommand) error { runConfig := d.state.runConfig - optionsOS := system.ParsePlatform(d.builder.options.Platform).OS - cmd := resolveCmdLine(c.ShellDependantCmdLine, runConfig, optionsOS) + cmd := resolveCmdLine(c.ShellDependantCmdLine, runConfig, d.state.baseImage.OperatingSystem()) runConfig.Entrypoint = cmd if !d.state.cmdSet { runConfig.Cmd = nil diff --git a/components/engine/builder/dockerfile/dispatchers_test.go b/components/engine/builder/dockerfile/dispatchers_test.go index 7ad0d8d3c2..d65caf8745 100644 --- a/components/engine/builder/dockerfile/dispatchers_test.go +++ b/components/engine/builder/dockerfile/dispatchers_test.go @@ -225,6 +225,7 @@ func TestWorkdir(t *testing.T) { func TestCmd(t *testing.T) { b := newBuilderWithMockBackend() sb := newDispatchRequest(b, '`', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults()) + sb.state.baseImage = &mockImage{} command := "./executable" cmd := &instructions.CmdCommand{ @@ -282,6 +283,7 @@ func TestHealthcheckCmd(t *testing.T) { func TestEntrypoint(t *testing.T) { b := newBuilderWithMockBackend() sb := newDispatchRequest(b, '`', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults()) + sb.state.baseImage = &mockImage{} entrypointCmd := "/usr/sbin/nginx" cmd := &instructions.EntrypointCommand{ @@ -357,6 +359,7 @@ func TestStopSignal(t *testing.T) { } b := newBuilderWithMockBackend() sb := newDispatchRequest(b, '`', nil, newBuildArgs(make(map[string]*string)), newStagesBuildResults()) + sb.state.baseImage = &mockImage{} signal := "SIGKILL" cmd := &instructions.StopSignalCommand{ diff --git a/components/engine/builder/dockerfile/evaluator.go b/components/engine/builder/dockerfile/evaluator.go index 74264faf2e..9f747001c7 100644 --- a/components/engine/builder/dockerfile/evaluator.go +++ b/components/engine/builder/dockerfile/evaluator.go @@ -37,8 +37,7 @@ import ( func dispatch(d dispatchRequest, cmd instructions.Command) (err error) { if c, ok := cmd.(instructions.PlatformSpecific); ok { - optionsOS := system.ParsePlatform(d.builder.options.Platform).OS - err := c.CheckPlatform(optionsOS) + err := c.CheckPlatform(d.state.baseImage.OperatingSystem()) if err != nil { return errdefs.InvalidParameter(err) } diff --git a/components/engine/builder/dockerfile/imagecontext.go b/components/engine/builder/dockerfile/imagecontext.go index 0d4af384af..fd2b942393 100644 --- a/components/engine/builder/dockerfile/imagecontext.go +++ b/components/engine/builder/dockerfile/imagecontext.go @@ -6,13 +6,12 @@ import ( "github.com/docker/docker/api/types/backend" "github.com/docker/docker/builder" dockerimage "github.com/docker/docker/image" - "github.com/docker/docker/pkg/system" "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/net/context" ) -type getAndMountFunc func(string, bool) (builder.Image, builder.ROLayer, error) +type getAndMountFunc func(string, bool, string) (builder.Image, builder.ROLayer, error) // imageSources mounts images and provides a cache for mounted images. It tracks // all images so they can be unmounted at the end of the build. @@ -23,7 +22,7 @@ type imageSources struct { } func newImageSources(ctx context.Context, options builderOptions) *imageSources { - getAndMount := func(idOrRef string, localOnly bool) (builder.Image, builder.ROLayer, error) { + getAndMount := func(idOrRef string, localOnly bool, osForPull string) (builder.Image, builder.ROLayer, error) { pullOption := backend.PullOptionNoPull if !localOnly { if options.Options.PullParent { @@ -32,12 +31,11 @@ func newImageSources(ctx context.Context, options builderOptions) *imageSources pullOption = backend.PullOptionPreferLocal } } - optionsPlatform := system.ParsePlatform(options.Options.Platform) return options.Backend.GetImageAndReleasableLayer(ctx, idOrRef, backend.GetImageAndLayerOptions{ PullOption: pullOption, AuthConfig: options.Options.AuthConfigs, Output: options.ProgressWriter.Output, - OS: optionsPlatform.OS, + OS: osForPull, }) } @@ -47,12 +45,12 @@ func newImageSources(ctx context.Context, options builderOptions) *imageSources } } -func (m *imageSources) Get(idOrRef string, localOnly bool) (*imageMount, error) { +func (m *imageSources) Get(idOrRef string, localOnly bool, osForPull string) (*imageMount, error) { if im, ok := m.byImageID[idOrRef]; ok { return im, nil } - image, layer, err := m.getImage(idOrRef, localOnly) + image, layer, err := m.getImage(idOrRef, localOnly, osForPull) if err != nil { return nil, err } diff --git a/components/engine/builder/dockerfile/instructions/parse.go b/components/engine/builder/dockerfile/instructions/parse.go index 9b94ee3660..6d310eac08 100644 --- a/components/engine/builder/dockerfile/instructions/parse.go +++ b/components/engine/builder/dockerfile/instructions/parse.go @@ -3,7 +3,6 @@ package instructions // import "github.com/docker/docker/builder/dockerfile/inst import ( "fmt" "regexp" - "runtime" "sort" "strconv" "strings" @@ -278,20 +277,16 @@ func parseFrom(req parseRequest) (*Stage, error) { return nil, err } specPlatform := system.ParsePlatform(flPlatform.Value) - if specPlatform.OS == "" { - specPlatform.OS = runtime.GOOS - } if err := system.ValidatePlatform(specPlatform); err != nil { return nil, fmt.Errorf("invalid platform %q on FROM", flPlatform.Value) } - if !system.IsOSSupported(specPlatform.OS) { + if specPlatform.OS != "" && !system.IsOSSupported(specPlatform.OS) { return nil, fmt.Errorf("unsupported platform %q on FROM", flPlatform.Value) } if err != nil { return nil, err } code := strings.TrimSpace(req.original) - fmt.Println("JJH", specPlatform.OS) return &Stage{ BaseName: req.args[0], Name: stageName, diff --git a/components/engine/builder/dockerfile/internals.go b/components/engine/builder/dockerfile/internals.go index c8b34f8f65..e141a78465 100644 --- a/components/engine/builder/dockerfile/internals.go +++ b/components/engine/builder/dockerfile/internals.go @@ -83,8 +83,7 @@ func (b *Builder) commit(dispatchState *dispatchState, comment string) error { return errors.New("Please provide a source image with `from` prior to commit") } - optionsPlatform := system.ParsePlatform(b.options.Platform) - runConfigWithCommentCmd := copyRunConfig(dispatchState.runConfig, withCmdComment(comment, optionsPlatform.OS)) + runConfigWithCommentCmd := copyRunConfig(dispatchState.runConfig, withCmdComment(comment, dispatchState.baseImage.OperatingSystem())) hit, err := b.probeCache(dispatchState, runConfigWithCommentCmd) if err != nil || hit { return err @@ -164,16 +163,15 @@ func (b *Builder) performCopy(state *dispatchState, inst copyInstruction) error commentStr := fmt.Sprintf("%s %s%s in %s ", inst.cmdName, chownComment, srcHash, inst.dest) // TODO: should this have been using origPaths instead of srcHash in the comment? - optionsPlatform := system.ParsePlatform(b.options.Platform) runConfigWithCommentCmd := copyRunConfig( state.runConfig, - withCmdCommentString(commentStr, optionsPlatform.OS)) + withCmdCommentString(commentStr, state.baseImage.OperatingSystem())) hit, err := b.probeCache(state, runConfigWithCommentCmd) if err != nil || hit { return err } - imageMount, err := b.imageSources.Get(state.imageID, true) + imageMount, err := b.imageSources.Get(state.imageID, true, state.baseImage.OperatingSystem()) if err != nil { return errors.Wrapf(err, "failed to get destination image %q", state.imageID) } @@ -184,7 +182,7 @@ func (b *Builder) performCopy(state *dispatchState, inst copyInstruction) error } defer rwLayer.Release() - destInfo, err := createDestInfo(state.runConfig.WorkingDir, inst, rwLayer, b.options.Platform) + destInfo, err := createDestInfo(state.runConfig.WorkingDir, inst, rwLayer, state.baseImage.OperatingSystem()) if err != nil { return err } From 60e6ee9365d0eb1235191d206b6559e467566e29 Mon Sep 17 00:00:00 2001 From: John Howard Date: Mon, 5 Feb 2018 14:41:45 -0800 Subject: [PATCH 6/9] Builder: Fix CI issues Signed-off-by: John Howard Upstream-commit: 317513d6984c0ba5df41dc578b22eb32fec55b55 Component: engine --- components/engine/builder/dockerfile/dispatchers.go | 12 ++++++------ components/engine/builder/dockerfile/evaluator.go | 2 +- components/engine/builder/dockerfile/internals.go | 8 ++++---- components/engine/daemon/images/image_history.go | 5 ++++- components/engine/daemon/images/images.go | 4 +++- 5 files changed, 18 insertions(+), 13 deletions(-) diff --git a/components/engine/builder/dockerfile/dispatchers.go b/components/engine/builder/dockerfile/dispatchers.go index 33e6bd4c23..be1183684c 100644 --- a/components/engine/builder/dockerfile/dispatchers.go +++ b/components/engine/builder/dockerfile/dispatchers.go @@ -145,7 +145,7 @@ func (d *dispatchRequest) getImageMount(imageRefOrID string) (*imageMount, error imageRefOrID = stage.Image localOnly = true } - return d.builder.imageSources.Get(imageRefOrID, localOnly, d.state.baseImage.OperatingSystem()) + return d.builder.imageSources.Get(imageRefOrID, localOnly, d.state.operatingSystem) } // FROM [--platform=platform] imagename[:tag | @digest] [AS build-stage-name] @@ -288,7 +288,7 @@ func dispatchOnbuild(d dispatchRequest, c *instructions.OnbuildCommand) error { func dispatchWorkdir(d dispatchRequest, c *instructions.WorkdirCommand) error { runConfig := d.state.runConfig var err error - runConfig.WorkingDir, err = normalizeWorkdir(d.state.baseImage.OperatingSystem(), runConfig.WorkingDir, c.Path) + runConfig.WorkingDir, err = normalizeWorkdir(d.state.operatingSystem, runConfig.WorkingDir, c.Path) if err != nil { return err } @@ -304,7 +304,7 @@ func dispatchWorkdir(d dispatchRequest, c *instructions.WorkdirCommand) error { } comment := "WORKDIR " + runConfig.WorkingDir - runConfigWithCommentCmd := copyRunConfig(runConfig, withCmdCommentString(comment, d.state.baseImage.OperatingSystem())) + runConfigWithCommentCmd := copyRunConfig(runConfig, withCmdCommentString(comment, d.state.operatingSystem)) containerID, err := d.builder.probeAndCreate(d.state, runConfigWithCommentCmd) if err != nil || containerID == "" { return err @@ -339,7 +339,7 @@ func dispatchRun(d dispatchRequest, c *instructions.RunCommand) error { return system.ErrNotSupportedOperatingSystem } stateRunConfig := d.state.runConfig - cmdFromArgs := resolveCmdLine(c.ShellDependantCmdLine, stateRunConfig, d.state.baseImage.OperatingSystem()) + cmdFromArgs := resolveCmdLine(c.ShellDependantCmdLine, stateRunConfig, d.state.operatingSystem) buildArgs := d.state.buildArgs.FilterAllowed(stateRunConfig.Env) saveCmd := cmdFromArgs @@ -420,7 +420,7 @@ func prependEnvOnCmd(buildArgs *buildArgs, buildArgVars []string, cmd strslice.S // func dispatchCmd(d dispatchRequest, c *instructions.CmdCommand) error { runConfig := d.state.runConfig - cmd := resolveCmdLine(c.ShellDependantCmdLine, runConfig, d.state.baseImage.OperatingSystem()) + cmd := resolveCmdLine(c.ShellDependantCmdLine, runConfig, d.state.operatingSystem) runConfig.Cmd = cmd // set config as already being escaped, this prevents double escaping on windows runConfig.ArgsEscaped = true @@ -463,7 +463,7 @@ func dispatchHealthcheck(d dispatchRequest, c *instructions.HealthCheckCommand) // func dispatchEntrypoint(d dispatchRequest, c *instructions.EntrypointCommand) error { runConfig := d.state.runConfig - cmd := resolveCmdLine(c.ShellDependantCmdLine, runConfig, d.state.baseImage.OperatingSystem()) + cmd := resolveCmdLine(c.ShellDependantCmdLine, runConfig, d.state.operatingSystem) runConfig.Entrypoint = cmd if !d.state.cmdSet { runConfig.Cmd = nil diff --git a/components/engine/builder/dockerfile/evaluator.go b/components/engine/builder/dockerfile/evaluator.go index 9f747001c7..0f76845086 100644 --- a/components/engine/builder/dockerfile/evaluator.go +++ b/components/engine/builder/dockerfile/evaluator.go @@ -37,7 +37,7 @@ import ( func dispatch(d dispatchRequest, cmd instructions.Command) (err error) { if c, ok := cmd.(instructions.PlatformSpecific); ok { - err := c.CheckPlatform(d.state.baseImage.OperatingSystem()) + err := c.CheckPlatform(d.state.operatingSystem) if err != nil { return errdefs.InvalidParameter(err) } diff --git a/components/engine/builder/dockerfile/internals.go b/components/engine/builder/dockerfile/internals.go index e141a78465..53748f0619 100644 --- a/components/engine/builder/dockerfile/internals.go +++ b/components/engine/builder/dockerfile/internals.go @@ -83,7 +83,7 @@ func (b *Builder) commit(dispatchState *dispatchState, comment string) error { return errors.New("Please provide a source image with `from` prior to commit") } - runConfigWithCommentCmd := copyRunConfig(dispatchState.runConfig, withCmdComment(comment, dispatchState.baseImage.OperatingSystem())) + runConfigWithCommentCmd := copyRunConfig(dispatchState.runConfig, withCmdComment(comment, dispatchState.operatingSystem)) hit, err := b.probeCache(dispatchState, runConfigWithCommentCmd) if err != nil || hit { return err @@ -165,13 +165,13 @@ func (b *Builder) performCopy(state *dispatchState, inst copyInstruction) error // TODO: should this have been using origPaths instead of srcHash in the comment? runConfigWithCommentCmd := copyRunConfig( state.runConfig, - withCmdCommentString(commentStr, state.baseImage.OperatingSystem())) + withCmdCommentString(commentStr, state.operatingSystem)) hit, err := b.probeCache(state, runConfigWithCommentCmd) if err != nil || hit { return err } - imageMount, err := b.imageSources.Get(state.imageID, true, state.baseImage.OperatingSystem()) + imageMount, err := b.imageSources.Get(state.imageID, true, state.operatingSystem) if err != nil { return errors.Wrapf(err, "failed to get destination image %q", state.imageID) } @@ -182,7 +182,7 @@ func (b *Builder) performCopy(state *dispatchState, inst copyInstruction) error } defer rwLayer.Release() - destInfo, err := createDestInfo(state.runConfig.WorkingDir, inst, rwLayer, state.baseImage.OperatingSystem()) + destInfo, err := createDestInfo(state.runConfig.WorkingDir, inst, rwLayer, state.operatingSystem) if err != nil { return err } diff --git a/components/engine/daemon/images/image_history.go b/components/engine/daemon/images/image_history.go index 2b92292631..b4ca25b1b6 100644 --- a/components/engine/daemon/images/image_history.go +++ b/components/engine/daemon/images/image_history.go @@ -7,6 +7,7 @@ import ( "github.com/docker/distribution/reference" "github.com/docker/docker/api/types/image" "github.com/docker/docker/layer" + "github.com/docker/docker/pkg/system" ) // ImageHistory returns a slice of ImageHistory structures for the specified image @@ -31,7 +32,9 @@ func (i *ImageService) ImageHistory(name string) ([]*image.HistoryResponseItem, if len(img.RootFS.DiffIDs) <= layerCounter { return nil, fmt.Errorf("too many non-empty layers in History section") } - + if !system.IsOSSupported(img.OperatingSystem()) { + return nil, system.ErrNotSupportedOperatingSystem + } rootFS.Append(img.RootFS.DiffIDs[layerCounter]) l, err := i.layerStores[img.OperatingSystem()].Get(rootFS.ChainID()) if err != nil { diff --git a/components/engine/daemon/images/images.go b/components/engine/daemon/images/images.go index 46056f15b5..49212341c5 100644 --- a/components/engine/daemon/images/images.go +++ b/components/engine/daemon/images/images.go @@ -271,7 +271,9 @@ func (i *ImageService) SquashImage(id, parent string) (string, error) { rootFS := image.NewRootFS() parentImg = &image.Image{RootFS: rootFS} } - + if !system.IsOSSupported(img.OperatingSystem()) { + return "", errors.Wrap(err, system.ErrNotSupportedOperatingSystem.Error()) + } l, err := i.layerStores[img.OperatingSystem()].Get(img.RootFS.ChainID()) if err != nil { return "", errors.Wrap(err, "error getting image layer") From d649560b32c4528d56714ea6149e631903051f26 Mon Sep 17 00:00:00 2001 From: John Howard Date: Fri, 23 Feb 2018 13:03:49 -0800 Subject: [PATCH 7/9] Builder: Review feedback Signed-off-by: John Howard Upstream-commit: 14429056d3745ca052fba448d879788d16bbb01b Component: engine --- .../engine/builder/dockerfile/dispatchers.go | 40 +++++++++---------- .../dockerfile/instructions/commands.go | 11 ++--- .../builder/dockerfile/instructions/parse.go | 20 +++------- .../dockerfile/instructions/parse_test.go | 12 ------ 4 files changed, 31 insertions(+), 52 deletions(-) diff --git a/components/engine/builder/dockerfile/dispatchers.go b/components/engine/builder/dockerfile/dispatchers.go index be1183684c..991c433b2b 100644 --- a/components/engine/builder/dockerfile/dispatchers.go +++ b/components/engine/builder/dockerfile/dispatchers.go @@ -152,7 +152,10 @@ func (d *dispatchRequest) getImageMount(imageRefOrID string) (*imageMount, error // func initializeStage(d dispatchRequest, cmd *instructions.Stage) error { d.builder.imageProber.Reset() - image, err := d.getFromImage(d.shlex, cmd.BaseName, cmd.OperatingSystem) + if err := system.ValidatePlatform(&cmd.Platform); err != nil { + return err + } + image, err := d.getFromImage(d.shlex, cmd.BaseName, cmd.Platform.OS) if err != nil { return err } @@ -215,32 +218,29 @@ func (d *dispatchRequest) getExpandedImageName(shlex *shell.Lex, name string) (s // stagePlatform contains the value supplied by optional `--platform=` on // a current FROM statement. b.builder.options.Platform contains the operating // system part of the optional flag passed in the API call (or CLI flag -// through `docker build --platform=...`). -func (d *dispatchRequest) getOsFromFlagsAndStage(stagePlatform string) string { - osForPull := "" - // First, take the API platform if nothing provided on FROM - if stagePlatform == "" && d.builder.options.Platform != "" { - osForPull = d.builder.options.Platform +// through `docker build --platform=...`). Precedence is for an explicit +// platform indication in the FROM statement. +func (d *dispatchRequest) getOsFromFlagsAndStage(stageOS string) string { + switch { + case stageOS != "": + return stageOS + case d.builder.options.Platform != "": + // Note this is API "platform", but by this point, as the daemon is not + // multi-arch aware yet, it is guaranteed to only hold the OS part here. + return d.builder.options.Platform + default: + return runtime.GOOS } - // Next, use the FROM flag if that was provided - if osForPull == "" && stagePlatform != "" { - osForPull = stagePlatform - } - // Finally, assume the host OS - if osForPull == "" { - osForPull = runtime.GOOS - } - return osForPull } -func (d *dispatchRequest) getImageOrStage(name string, stagePlatform string) (builder.Image, error) { +func (d *dispatchRequest) getImageOrStage(name string, stageOS string) (builder.Image, error) { var localOnly bool if im, ok := d.stages.getByName(name); ok { name = im.Image localOnly = true } - os := d.getOsFromFlagsAndStage(stagePlatform) + os := d.getOsFromFlagsAndStage(stageOS) // Windows cannot support a container with no base image unless it is LCOW. if name == api.NoBaseImageSpecifier { @@ -267,12 +267,12 @@ func (d *dispatchRequest) getImageOrStage(name string, stagePlatform string) (bu } return imageMount.Image(), nil } -func (d *dispatchRequest) getFromImage(shlex *shell.Lex, name string, stagePlatform string) (builder.Image, error) { +func (d *dispatchRequest) getFromImage(shlex *shell.Lex, name string, stageOS string) (builder.Image, error) { name, err := d.getExpandedImageName(shlex, name) if err != nil { return nil, err } - return d.getImageOrStage(name, stagePlatform) + return d.getImageOrStage(name, stageOS) } func dispatchOnbuild(d dispatchRequest, c *instructions.OnbuildCommand) error { diff --git a/components/engine/builder/dockerfile/instructions/commands.go b/components/engine/builder/dockerfile/instructions/commands.go index 44382dc61e..a10140cf04 100644 --- a/components/engine/builder/dockerfile/instructions/commands.go +++ b/components/engine/builder/dockerfile/instructions/commands.go @@ -7,6 +7,7 @@ import ( "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/strslice" + specs "github.com/opencontainers/image-spec/specs-go/v1" ) // KeyValuePair represent an arbitrary named value (useful in slice instead of map[string] string to preserve ordering) @@ -357,11 +358,11 @@ type ShellCommand struct { // Stage represents a single stage in a multi-stage build type Stage struct { - Name string - Commands []Command - BaseName string - SourceCode string - OperatingSystem string + Name string + Commands []Command + BaseName string + SourceCode string + Platform specs.Platform } // AddCommand to the stage diff --git a/components/engine/builder/dockerfile/instructions/parse.go b/components/engine/builder/dockerfile/instructions/parse.go index 6d310eac08..e2d69a4887 100644 --- a/components/engine/builder/dockerfile/instructions/parse.go +++ b/components/engine/builder/dockerfile/instructions/parse.go @@ -276,23 +276,13 @@ func parseFrom(req parseRequest) (*Stage, error) { if err := req.flags.Parse(); err != nil { return nil, err } - specPlatform := system.ParsePlatform(flPlatform.Value) - if err := system.ValidatePlatform(specPlatform); err != nil { - return nil, fmt.Errorf("invalid platform %q on FROM", flPlatform.Value) - } - if specPlatform.OS != "" && !system.IsOSSupported(specPlatform.OS) { - return nil, fmt.Errorf("unsupported platform %q on FROM", flPlatform.Value) - } - if err != nil { - return nil, err - } code := strings.TrimSpace(req.original) return &Stage{ - BaseName: req.args[0], - Name: stageName, - SourceCode: code, - Commands: []Command{}, - OperatingSystem: specPlatform.OS, + BaseName: req.args[0], + Name: stageName, + SourceCode: code, + Commands: []Command{}, + Platform: *system.ParsePlatform(flPlatform.Value), }, nil } diff --git a/components/engine/builder/dockerfile/instructions/parse_test.go b/components/engine/builder/dockerfile/instructions/parse_test.go index 0c27f203d0..b78114d306 100644 --- a/components/engine/builder/dockerfile/instructions/parse_test.go +++ b/components/engine/builder/dockerfile/instructions/parse_test.go @@ -1,8 +1,6 @@ package instructions // import "github.com/docker/docker/builder/dockerfile/instructions" import ( - "fmt" - "runtime" "strings" "testing" @@ -186,16 +184,6 @@ func TestErrorCases(t *testing.T) { dockerfile: `foo bar`, expectedError: "unknown instruction: FOO", }, - { - name: "Invalid platform", - dockerfile: `FROM --platform=invalid busybox`, - expectedError: `invalid platform "invalid"`, - }, - { - name: "Only OS", - dockerfile: fmt.Sprintf(`FROM --platform=%s/%s busybox`, runtime.GOOS, runtime.GOARCH), - expectedError: `invalid platform`, - }, } for _, c := range cases { r := strings.NewReader(c.dockerfile) From 80e0c0b83d834f3b389650d1cf20a595238817af Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 26 Mar 2018 13:45:05 -0400 Subject: [PATCH 8/9] Remove myself from CODEOWNERS Signed-off-by: Daniel Nephin Upstream-commit: 92dfe76911c8ce9beea367e7e906ea138b467135 Component: engine --- components/engine/.github/CODEOWNERS | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/components/engine/.github/CODEOWNERS b/components/engine/.github/CODEOWNERS index ea7ae05184..9081854965 100644 --- a/components/engine/.github/CODEOWNERS +++ b/components/engine/.github/CODEOWNERS @@ -3,7 +3,7 @@ # # KEEP THIS FILE SORTED. Order is important. Last match takes precedence. -builder/** @dnephin @tonistiigi +builder/** @tonistiigi client/** @dnephin contrib/mkimage/** @tianon daemon/graphdriver/devmapper/** @rhvgoyal @@ -12,10 +12,9 @@ daemon/graphdriver/overlay/** @dmcgowan daemon/graphdriver/overlay2/** @dmcgowan daemon/graphdriver/windows/** @johnstep @jhowardmsft daemon/logger/awslogs/** @samuelkarp -hack/** @dnephin @tianon +hack/** @tianon hack/integration-cli-on-swarm/** @AkihiroSuda -integration-cli/** @dnephin @vdemeester -integration/** @dnephin @vdemeester -pkg/testutil/** @dnephin +integration-cli/** @vdemeester +integration/** @vdemeester plugin/** @cpuguy83 project/** @thaJeztah From 7f54d333796648f0016f8faf1da0b1b333360ee7 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 26 Mar 2018 22:16:38 +0200 Subject: [PATCH 9/9] Bump go-digest to v1.0.0-rc.1 Changes included: - digest: allow separators in algorithm field - disallow upper characters (/A-F/) in hex-encoded portion Signed-off-by: Sebastiaan van Stijn Upstream-commit: 82c44711cc59c150d09c87506a9bd648599ea0d6 Component: engine --- components/engine/vendor.conf | 2 +- .../opencontainers/go-digest/README.md | 24 ++++---- .../opencontainers/go-digest/algorithm.go | 54 +++++++++++++++++- .../opencontainers/go-digest/digest.go | 56 ++++++++++++------- .../opencontainers/go-digest/digester.go | 14 +++++ .../opencontainers/go-digest/doc.go | 14 +++++ .../opencontainers/go-digest/verifiers.go | 14 +++++ 7 files changed, 142 insertions(+), 36 deletions(-) diff --git a/components/engine/vendor.conf b/components/engine/vendor.conf index faf4f9c043..34ec02356f 100644 --- a/components/engine/vendor.conf +++ b/components/engine/vendor.conf @@ -61,7 +61,7 @@ github.com/ishidawataru/sctp 07191f837fedd2f13d1ec7b5f885f0f3ec54b1cb # get graph and distribution packages github.com/docker/distribution edc3ab29cdff8694dd6feb85cfeb4b5f1b38ed9c github.com/vbatts/tar-split v0.10.2 -github.com/opencontainers/go-digest a6d0ee40d4207ea02364bd3b9e8e77b9159ba1eb +github.com/opencontainers/go-digest v1.0.0-rc1 # get go-zfs packages github.com/mistifyio/go-zfs 22c9b32c84eb0d0c6f4043b6e90fc94073de92fa diff --git a/components/engine/vendor/github.com/opencontainers/go-digest/README.md b/components/engine/vendor/github.com/opencontainers/go-digest/README.md index 9d6174cfdc..0f5a04092c 100644 --- a/components/engine/vendor/github.com/opencontainers/go-digest/README.md +++ b/components/engine/vendor/github.com/opencontainers/go-digest/README.md @@ -1,10 +1,10 @@ # go-digest -[![GoDoc](https://godoc.org/github.com/docker/go-digest?status.svg)](https://godoc.org/github.com/docker/go-digest) [![Go Report Card](https://goreportcard.com/badge/github.com/docker/go-digest)](https://goreportcard.com/report/github.com/docker/go-digest) [![Build Status](https://travis-ci.org/docker/go-digest.svg?branch=master)](https://travis-ci.org/docker/go-digest) +[![GoDoc](https://godoc.org/github.com/opencontainers/go-digest?status.svg)](https://godoc.org/github.com/opencontainers/go-digest) [![Go Report Card](https://goreportcard.com/badge/github.com/opencontainers/go-digest)](https://goreportcard.com/report/github.com/opencontainers/go-digest) [![Build Status](https://travis-ci.org/opencontainers/go-digest.svg?branch=master)](https://travis-ci.org/opencontainers/go-digest) Common digest package used across the container ecosystem. -Please see the [godoc](https://godoc.org/github.com/docker/go-digest) for more information. +Please see the [godoc](https://godoc.org/github.com/opencontainers/go-digest) for more information. # What is a digest? @@ -49,7 +49,7 @@ can power a rich, safe, content distribution system. # Usage -While the [godoc](https://godoc.org/github.com/docker/go-digest) is +While the [godoc](https://godoc.org/github.com/opencontainers/go-digest) is considered the best resource, a few important items need to be called out when using this package. @@ -76,7 +76,7 @@ out when using this package. The Go API, at this stage, is considered stable, unless otherwise noted. -As always, before using a package export, read the [godoc](https://godoc.org/github.com/docker/go-digest). +As always, before using a package export, read the [godoc](https://godoc.org/github.com/opencontainers/go-digest). # Contributing @@ -88,16 +88,16 @@ the alternatives you tried before submitting a PR. # Reporting security issues -The maintainers take security seriously. If you discover a security -issue, please bring it to their attention right away! +Please DO NOT file a public issue, instead send your report privately to +security@opencontainers.org. -Please DO NOT file a public issue, instead send your report privately -to security@docker.com. +The maintainers take security seriously. If you discover a security issue, +please bring it to their attention right away! -Security reports are greatly appreciated and we will publicly thank you -for it. We also like to send gifts—if you're into Docker schwag, make -sure to let us know. We currently do not offer a paid security bounty -program, but are not ruling it out in the future. +If you are reporting a security issue, do not create an issue or file a pull +request on GitHub. Instead, disclose the issue responsibly by sending an email +to security@opencontainers.org (which is inhabited only by the maintainers of +the various OCI projects). # Copyright and license diff --git a/components/engine/vendor/github.com/opencontainers/go-digest/algorithm.go b/components/engine/vendor/github.com/opencontainers/go-digest/algorithm.go index a3c44801d5..8813bd26f1 100644 --- a/components/engine/vendor/github.com/opencontainers/go-digest/algorithm.go +++ b/components/engine/vendor/github.com/opencontainers/go-digest/algorithm.go @@ -1,3 +1,17 @@ +// Copyright 2017 Docker, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package digest import ( @@ -5,6 +19,7 @@ import ( "fmt" "hash" "io" + "regexp" ) // Algorithm identifies and implementation of a digester by an identifier. @@ -14,9 +29,9 @@ type Algorithm string // supported digest types const ( - SHA256 Algorithm = "sha256" // sha256 with hex encoding - SHA384 Algorithm = "sha384" // sha384 with hex encoding - SHA512 Algorithm = "sha512" // sha512 with hex encoding + SHA256 Algorithm = "sha256" // sha256 with hex encoding (lower case only) + SHA384 Algorithm = "sha384" // sha384 with hex encoding (lower case only) + SHA512 Algorithm = "sha512" // sha512 with hex encoding (lower case only) // Canonical is the primary digest algorithm used with the distribution // project. Other digests may be used but this one is the primary storage @@ -36,6 +51,14 @@ var ( SHA384: crypto.SHA384, SHA512: crypto.SHA512, } + + // anchoredEncodedRegexps contains anchored regular expressions for hex-encoded digests. + // Note that /A-F/ disallowed. + anchoredEncodedRegexps = map[Algorithm]*regexp.Regexp{ + SHA256: regexp.MustCompile(`^[a-f0-9]{64}$`), + SHA384: regexp.MustCompile(`^[a-f0-9]{96}$`), + SHA512: regexp.MustCompile(`^[a-f0-9]{128}$`), + } ) // Available returns true if the digest type is available for use. If this @@ -111,6 +134,14 @@ func (a Algorithm) Hash() hash.Hash { return algorithms[a].New() } +// Encode encodes the raw bytes of a digest, typically from a hash.Hash, into +// the encoded portion of the digest. +func (a Algorithm) Encode(d []byte) string { + // TODO(stevvooe): Currently, all algorithms use a hex encoding. When we + // add support for back registration, we can modify this accordingly. + return fmt.Sprintf("%x", d) +} + // FromReader returns the digest of the reader using the algorithm. func (a Algorithm) FromReader(rd io.Reader) (Digest, error) { digester := a.Digester() @@ -142,3 +173,20 @@ func (a Algorithm) FromBytes(p []byte) Digest { func (a Algorithm) FromString(s string) Digest { return a.FromBytes([]byte(s)) } + +// Validate validates the encoded portion string +func (a Algorithm) Validate(encoded string) error { + r, ok := anchoredEncodedRegexps[a] + if !ok { + return ErrDigestUnsupported + } + // Digests much always be hex-encoded, ensuring that their hex portion will + // always be size*2 + if a.Size()*2 != len(encoded) { + return ErrDigestInvalidLength + } + if r.MatchString(encoded) { + return nil + } + return ErrDigestInvalidFormat +} diff --git a/components/engine/vendor/github.com/opencontainers/go-digest/digest.go b/components/engine/vendor/github.com/opencontainers/go-digest/digest.go index 7c66c30c01..ad398cba2f 100644 --- a/components/engine/vendor/github.com/opencontainers/go-digest/digest.go +++ b/components/engine/vendor/github.com/opencontainers/go-digest/digest.go @@ -1,3 +1,17 @@ +// Copyright 2017 Docker, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package digest import ( @@ -31,16 +45,21 @@ func NewDigest(alg Algorithm, h hash.Hash) Digest { // functions. This is also useful for rebuilding digests from binary // serializations. func NewDigestFromBytes(alg Algorithm, p []byte) Digest { - return Digest(fmt.Sprintf("%s:%x", alg, p)) + return NewDigestFromEncoded(alg, alg.Encode(p)) } -// NewDigestFromHex returns a Digest from alg and a the hex encoded digest. +// NewDigestFromHex is deprecated. Please use NewDigestFromEncoded. func NewDigestFromHex(alg, hex string) Digest { - return Digest(fmt.Sprintf("%s:%s", alg, hex)) + return NewDigestFromEncoded(Algorithm(alg), hex) +} + +// NewDigestFromEncoded returns a Digest from alg and the encoded digest. +func NewDigestFromEncoded(alg Algorithm, encoded string) Digest { + return Digest(fmt.Sprintf("%s:%s", alg, encoded)) } // DigestRegexp matches valid digest types. -var DigestRegexp = regexp.MustCompile(`[a-zA-Z0-9-_+.]+:[a-fA-F0-9]+`) +var DigestRegexp = regexp.MustCompile(`[a-z0-9]+(?:[.+_-][a-z0-9]+)*:[a-zA-Z0-9=_-]+`) // DigestRegexpAnchored matches valid digest types, anchored to the start and end of the match. var DigestRegexpAnchored = regexp.MustCompile(`^` + DigestRegexp.String() + `$`) @@ -82,26 +101,18 @@ func FromString(s string) Digest { // error if not. func (d Digest) Validate() error { s := string(d) - i := strings.Index(s, ":") - - // validate i then run through regexp - if i < 0 || i+1 == len(s) || !DigestRegexpAnchored.MatchString(s) { + if i <= 0 || i+1 == len(s) { return ErrDigestInvalidFormat } - - algorithm := Algorithm(s[:i]) + algorithm, encoded := Algorithm(s[:i]), s[i+1:] if !algorithm.Available() { + if !DigestRegexpAnchored.MatchString(s) { + return ErrDigestInvalidFormat + } return ErrDigestUnsupported } - - // Digests much always be hex-encoded, ensuring that their hex portion will - // always be size*2 - if algorithm.Size()*2 != len(s[i+1:]) { - return ErrDigestInvalidLength - } - - return nil + return algorithm.Validate(encoded) } // Algorithm returns the algorithm portion of the digest. This will panic if @@ -119,12 +130,17 @@ func (d Digest) Verifier() Verifier { } } -// Hex returns the hex digest portion of the digest. This will panic if the +// Encoded returns the encoded portion of the digest. This will panic if the // underlying digest is not in a valid format. -func (d Digest) Hex() string { +func (d Digest) Encoded() string { return string(d[d.sepIndex()+1:]) } +// Hex is deprecated. Please use Digest.Encoded. +func (d Digest) Hex() string { + return d.Encoded() +} + func (d Digest) String() string { return string(d) } diff --git a/components/engine/vendor/github.com/opencontainers/go-digest/digester.go b/components/engine/vendor/github.com/opencontainers/go-digest/digester.go index 918a3f9191..36fa2728ef 100644 --- a/components/engine/vendor/github.com/opencontainers/go-digest/digester.go +++ b/components/engine/vendor/github.com/opencontainers/go-digest/digester.go @@ -1,3 +1,17 @@ +// Copyright 2017 Docker, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package digest import "hash" diff --git a/components/engine/vendor/github.com/opencontainers/go-digest/doc.go b/components/engine/vendor/github.com/opencontainers/go-digest/doc.go index f64b0db32b..491ea1ef1f 100644 --- a/components/engine/vendor/github.com/opencontainers/go-digest/doc.go +++ b/components/engine/vendor/github.com/opencontainers/go-digest/doc.go @@ -1,3 +1,17 @@ +// Copyright 2017 Docker, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Package digest provides a generalized type to opaquely represent message // digests and their operations within the registry. The Digest type is // designed to serve as a flexible identifier in a content-addressable system. diff --git a/components/engine/vendor/github.com/opencontainers/go-digest/verifiers.go b/components/engine/vendor/github.com/opencontainers/go-digest/verifiers.go index f1db6cda84..32125e9187 100644 --- a/components/engine/vendor/github.com/opencontainers/go-digest/verifiers.go +++ b/components/engine/vendor/github.com/opencontainers/go-digest/verifiers.go @@ -1,3 +1,17 @@ +// Copyright 2017 Docker, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package digest import (