Merge pull request #3107 from thaJeztah/20.10_backport_sigurg_handling

[20.10 backport] Ignore SIGURG on Unix (including Darwin)

.circleci/config.yml

@@ -42,6 +42,7 @@ jobs:
     docker: [{image: 'docker:19.03-git'}]
     environment:
       DOCKER_BUILDKIT: 1
+      BUILDX_VERSION: "v0.5.1"
     parallelism: 3
     steps:
       - checkout
@@ -55,21 +56,14 @@ jobs:
       - run:
           name: "Docker info"
           command: docker info
-      - run:
-          name: "Cross - build image"
-          command: |
-            docker build --progress=plain -f dockerfiles/Dockerfile.cross --tag cli-builder:$CIRCLE_BUILD_NUM .
-      - run:
-          name: "Cross"
-          command: |
-            name=cross-$CIRCLE_BUILD_NUM-$CIRCLE_NODE_INDEX
-            docker run \
-                -e CROSS_GROUP=$CIRCLE_NODE_INDEX \
-                --name $name cli-builder:$CIRCLE_BUILD_NUM \
-                make cross
-            docker cp \
-                $name:/go/src/github.com/docker/cli/build \
-                /work/build
+      - run: apk add make curl
+      - run: mkdir -vp ~/.docker/cli-plugins/
+      - run: curl -fsSL --output ~/.docker/cli-plugins/docker-buildx https://github.com/docker/buildx/releases/download/${BUILDX_VERSION}/buildx-${BUILDX_VERSION}.linux-amd64
+      - run: chmod a+x ~/.docker/cli-plugins/docker-buildx
+      - run: docker buildx version
+      - run: docker context create buildctx
+      - run: docker buildx create --use buildctx && docker buildx inspect --bootstrap
+      - run: GROUP_INDEX=$CIRCLE_NODE_INDEX GROUP_TOTAL=$CIRCLE_NODE_TOTAL docker buildx bake cross --progress=plain
       - store_artifacts:
           path: /work/build
 

.dockerignore

@@ -1,6 +1,5 @@
 .circleci
 .dockerignore
-.git
 .github
 .gitignore
 appveyor.yml

.gitignore

@@ -8,8 +8,7 @@
 Thumbs.db
 .editorconfig
 /build/
-cli/winresources/rsrc_386.syso
-cli/winresources/rsrc_amd64.syso
+cli/winresources/rsrc_*.syso
 /man/man1/
 /man/man5/
 /man/man8/

Dockerfile

@@ -0,0 +1,61 @@
+#syntax=docker/dockerfile:1.2
+
+ARG BASE_VARIANT=alpine
+ARG GO_VERSION=1.13.15
+
+FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-${BASE_VARIANT} AS gostable
+FROM --platform=$BUILDPLATFORM golang:1.16-${BASE_VARIANT} AS golatest	
+
+FROM gostable AS go-linux	
+FROM golatest AS go-darwin
+FROM golatest AS go-windows-amd64
+FROM golatest AS go-windows-386
+FROM golatest AS go-windows-arm
+FROM --platform=$BUILDPLATFORM tonistiigi/golang:497feff1-${BASE_VARIANT} AS go-windows-arm64
+FROM go-windows-${TARGETARCH} AS go-windows
+
+FROM --platform=$BUILDPLATFORM tonistiigi/xx@sha256:620d36a9d7f1e3b102a5c7e8eff12081ac363828b3a44390f24fa8da2d49383d AS xx
+
+FROM go-${TARGETOS} AS build-base-alpine
+COPY --from=xx / /
+RUN apk add --no-cache clang lld llvm file git
+WORKDIR /go/src/github.com/docker/cli
+
+FROM build-base-alpine AS build-alpine
+ARG TARGETPLATFORM
+# gcc is installed for libgcc only
+RUN xx-apk add --no-cache musl-dev gcc
+
+FROM go-${TARGETOS} AS build-base-buster
+COPY --from=xx / /
+RUN apt-get update && apt-get install --no-install-recommends -y clang lld file
+WORKDIR /go/src/github.com/docker/cli
+
+FROM build-base-buster AS build-buster
+ARG TARGETPLATFORM
+RUN xx-apt install --no-install-recommends -y libc6-dev libgcc-8-dev
+
+FROM build-${BASE_VARIANT} AS build
+# GO_LINKMODE defines if static or dynamic binary should be produced
+ARG GO_LINKMODE=static
+# GO_BUILDTAGS defines additional build tags
+ARG GO_BUILDTAGS
+# GO_STRIP strips debugging symbols if set
+ARG GO_STRIP
+# CGO_ENABLED manually sets if cgo is used
+ARG CGO_ENABLED
+# VERSION sets the version for the produced binary
+ARG VERSION
+RUN --mount=ro --mount=type=cache,target=/root/.cache \
+    --mount=from=dockercore/golang-cross:xx-sdk-extras,target=/xx-sdk,src=/xx-sdk \
+    --mount=type=tmpfs,target=cli/winresources \
+    xx-go --wrap && \
+    # export GOCACHE=$(go env GOCACHE)/$(xx-info)$([ -f /etc/alpine-release ] && echo "alpine") && \
+    TARGET=/out ./scripts/build/binary && \
+    xx-verify $([ "$GO_LINKMODE" = "static" ] && echo "--static") /out/docker
+
+FROM build-base-${BASE_VARIANT} AS dev 
+COPY . .
+
+FROM scratch AS binary
+COPY --from=build /out .

Makefile

@@ -30,8 +30,7 @@ lint: ## run all the lint tools
 	gometalinter --config gometalinter.json ./...
 
 .PHONY: binary
-binary: ## build executable for Linux
-	@echo "WARNING: binary creates a Linux executable. Use cross for macOS or Windows."
+binary:
 	./scripts/build/binary
 
 .PHONY: plugins
@@ -39,28 +38,20 @@ plugins: ## build example CLI plugins
 	./scripts/build/plugins
 
 .PHONY: cross
-cross: ## build executable for macOS and Windows
-	./scripts/build/cross
-
-.PHONY: binary-windows
-binary-windows: ## build executable for Windows
-	./scripts/build/windows
+cross:
+	./scripts/build/binary
 
 .PHONY: plugins-windows
 plugins-windows: ## build example CLI plugins for Windows
 	./scripts/build/plugins-windows
 
-.PHONY: binary-osx
-binary-osx: ## build executable for macOS
-	./scripts/build/osx
-
 .PHONY: plugins-osx
 plugins-osx: ## build example CLI plugins for macOS
 	./scripts/build/plugins-osx
 
 .PHONY: dynbinary
 dynbinary: ## build dynamically linked binary
-	./scripts/build/dynbinary
+	GO_LINKMODE=dynamic ./scripts/build/binary
 
 vendor: vendor.conf ## check that vendor matches vendor.conf
 	rm -rf vendor

README.md

@@ -12,18 +12,31 @@ Development
 
 `docker/cli` is developed using Docker.
 
-Build a linux binary:
+Build CLI from source:
 
 ```
-$ make -f docker.Makefile binary
+$ docker buildx bake
 ```
 
 Build binaries for all supported platforms:
 
 ```
-$ make -f docker.Makefile cross
+$ docker buildx bake cross
 ```
 
+Build for a specific platform:
+
+```
+$ docker buildx bake --set binary.platform=linux/arm64 
+```
+
+Build dynamic binary for glibc or musl:
+
+```
+$ USE_GLIBC=1 docker buildx bake dynbinary 
+```
+
+
 Run all linting:
 
 ```
@@ -44,12 +57,6 @@ Start an interactive development environment:
 $ make -f docker.Makefile shell
 ```
 
-In the development environment you can run many tasks, including build binaries:
-
-```
-$ make binary
-```
-
 Legal
 =====
 *Brought to you courtesy of our legal counsel. For more context,

cli/command/container/attach.go

@@ -97,7 +97,8 @@ func runAttach(dockerCli command.Cli, opts *attachOptions) error {
 	}
 
 	if opts.proxy && !c.Config.Tty {
-		sigc := ForwardAllSignals(ctx, dockerCli, opts.container)
+		sigc := notfiyAllSignals()
+		go ForwardAllSignals(ctx, dockerCli, opts.container, sigc)
 		defer signal.StopCatch(sigc)
 	}
 

cli/command/container/client_test.go

@@ -32,6 +32,7 @@ type fakeClient struct {
 	containerExportFunc     func(string) (io.ReadCloser, error)
 	containerExecResizeFunc func(id string, options types.ResizeOptions) error
 	containerRemoveFunc     func(ctx context.Context, container string, options types.ContainerRemoveOptions) error
+	containerKillFunc       func(ctx context.Context, container, signal string) error
 	Version                 string
 }
 
@@ -154,3 +155,10 @@ func (f *fakeClient) ContainerExecResize(_ context.Context, id string, options t
 	}
 	return nil
 }
+
+func (f *fakeClient) ContainerKill(ctx context.Context, container, signal string) error {
+	if f.containerKillFunc != nil {
+		return f.containerKillFunc(ctx, container, signal)
+	}
+	return nil
+}

cli/command/container/run.go

@@ -131,7 +131,8 @@ func runContainer(dockerCli command.Cli, opts *runOptions, copts *containerOptio
 		return runStartContainerErr(err)
 	}
 	if opts.sigProxy {
-		sigc := ForwardAllSignals(ctx, dockerCli, createResponse.ID)
+		sigc := notfiyAllSignals()
+		go ForwardAllSignals(ctx, dockerCli, createResponse.ID, sigc)
 		defer signal.StopCatch(sigc)
 	}
 

cli/command/container/signals.go

@@ -0,0 +1,61 @@
+package container
+
+import (
+	"context"
+	"os"
+	gosignal "os/signal"
+
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/docker/pkg/signal"
+	"github.com/sirupsen/logrus"
+)
+
+// ForwardAllSignals forwards signals to the container
+//
+// The channel you pass in must already be setup to receive any signals you want to forward.
+func ForwardAllSignals(ctx context.Context, cli command.Cli, cid string, sigc <-chan os.Signal) {
+	var (
+		s  os.Signal
+		ok bool
+	)
+	for {
+		select {
+		case s, ok = <-sigc:
+			if !ok {
+				return
+			}
+		case <-ctx.Done():
+			return
+		}
+
+		if s == signal.SIGCHLD || s == signal.SIGPIPE {
+			continue
+		}
+
+		// In go1.14+, the go runtime issues SIGURG as an interrupt to support pre-emptable system calls on Linux.
+		// Since we can't forward that along we'll check that here.
+		if isRuntimeSig(s) {
+			continue
+		}
+		var sig string
+		for sigStr, sigN := range signal.SignalMap {
+			if sigN == s {
+				sig = sigStr
+				break
+			}
+		}
+		if sig == "" {
+			continue
+		}
+
+		if err := cli.Client().ContainerKill(ctx, cid, sig); err != nil {
+			logrus.Debugf("Error sending signal: %s", err)
+		}
+	}
+}
+
+func notfiyAllSignals() chan os.Signal {
+	sigc := make(chan os.Signal, 128)
+	gosignal.Notify(sigc)
+	return sigc
+}

cli/command/container/signals_test.go

@@ -0,0 +1,48 @@
+package container
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/docker/cli/internal/test"
+	"github.com/docker/docker/pkg/signal"
+)
+
+func TestForwardSignals(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	called := make(chan struct{})
+	client := &fakeClient{containerKillFunc: func(ctx context.Context, container, signal string) error {
+		close(called)
+		return nil
+	}}
+
+	cli := test.NewFakeCli(client)
+	sigc := make(chan os.Signal)
+	defer close(sigc)
+
+	go ForwardAllSignals(ctx, cli, t.Name(), sigc)
+
+	timer := time.NewTimer(30 * time.Second)
+	defer timer.Stop()
+
+	select {
+	case <-timer.C:
+		t.Fatal("timeout waiting to send signal")
+	case sigc <- signal.SignalMap["TERM"]:
+	}
+	if !timer.Stop() {
+		<-timer.C
+	}
+	timer.Reset(30 * time.Second)
+
+	select {
+	case <-called:
+	case <-timer.C:
+		t.Fatal("timeout waiting for signal to be processed")
+	}
+
+}

cli/command/container/signals_unix.go

@@ -0,0 +1,13 @@
+// +build !windows
+
+package container
+
+import (
+	"os"
+
+	"golang.org/x/sys/unix"
+)
+
+func isRuntimeSig(s os.Signal) bool {
+	return s == unix.SIGURG
+}

cli/command/container/signals_unix_test.go

@@ -0,0 +1,59 @@
+// +build !windows
+
+package container
+
+import (
+	"context"
+	"os"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/docker/cli/internal/test"
+	"golang.org/x/sys/unix"
+	"gotest.tools/v3/assert"
+)
+
+func TestIgnoredSignals(t *testing.T) {
+	ignoredSignals := []syscall.Signal{unix.SIGPIPE, unix.SIGCHLD, unix.SIGURG}
+
+	for _, s := range ignoredSignals {
+		t.Run(unix.SignalName(s), func(t *testing.T) {
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+
+			var called bool
+			client := &fakeClient{containerKillFunc: func(ctx context.Context, container, signal string) error {
+				called = true
+				return nil
+			}}
+
+			cli := test.NewFakeCli(client)
+			sigc := make(chan os.Signal)
+			defer close(sigc)
+
+			done := make(chan struct{})
+			go func() {
+				ForwardAllSignals(ctx, cli, t.Name(), sigc)
+				close(done)
+			}()
+
+			timer := time.NewTimer(30 * time.Second)
+			defer timer.Stop()
+
+			select {
+			case <-timer.C:
+				t.Fatal("timeout waiting to send signal")
+			case sigc <- s:
+			case <-done:
+			}
+
+			// cancel the context so ForwardAllSignals will exit after it has processed the signal we sent.
+			// This is how we know the signal was actually processed and are not introducing a flakey test.
+			cancel()
+			<-done
+
+			assert.Assert(t, !called, "kill was called")
+		})
+	}
+}

cli/command/container/signals_windows.go

@@ -0,0 +1,7 @@
+package container
+
+import "os"
+
+func isRuntimeSig(_ os.Signal) bool {
+	return false
+}

cli/command/container/start.go

@@ -74,7 +74,8 @@ func runStart(dockerCli command.Cli, opts *startOptions) error {
 
 		// We always use c.ID instead of container to maintain consistency during `docker start`
 		if !c.Config.Tty {
-			sigc := ForwardAllSignals(ctx, dockerCli, c.ID)
+			sigc := notfiyAllSignals()
+			go ForwardAllSignals(ctx, dockerCli, c.ID, sigc)
 			defer signal.StopCatch(sigc)
 		}
 

cli/command/container/tty.go

@@ -95,32 +95,3 @@ func MonitorTtySize(ctx context.Context, cli command.Cli, id string, isExec bool
 	}
 	return nil
 }
-
-// ForwardAllSignals forwards signals to the container
-func ForwardAllSignals(ctx context.Context, cli command.Cli, cid string) chan os.Signal {
-	sigc := make(chan os.Signal, 128)
-	signal.CatchAll(sigc)
-	go func() {
-		for s := range sigc {
-			if s == signal.SIGCHLD || s == signal.SIGPIPE {
-				continue
-			}
-			var sig string
-			for sigStr, sigN := range signal.SignalMap {
-				if sigN == s {
-					sig = sigStr
-					break
-				}
-			}
-			if sig == "" {
-				fmt.Fprintf(cli.Err(), "Unsupported signal: %v. Discarding.\n", s)
-				continue
-			}
-
-			if err := cli.Client().ContainerKill(ctx, cid, sig); err != nil {
-				logrus.Debugf("Error sending signal: %s", err)
-			}
-		}
-	}()
-	return sigc
-}

cli/command/system/info.go

@@ -402,10 +402,7 @@ func printServerWarningsLegacy(dockerCli command.Cli, info types.Info) {
 	if !info.SwapLimit {
 		fmt.Fprintln(dockerCli.Err(), "WARNING: No swap limit support")
 	}
-	if !info.KernelMemory {
-		fmt.Fprintln(dockerCli.Err(), "WARNING: No kernel memory limit support")
-	}
-	if !info.OomKillDisable {
+	if !info.OomKillDisable && info.CgroupVersion != "2" {
 		fmt.Fprintln(dockerCli.Err(), "WARNING: No oom kill disable support")
 	}
 	if !info.CPUCfsQuota {

cli/command/system/info_test.go

@@ -248,7 +248,6 @@ func TestPrettyPrintInfo(t *testing.T) {
 	sampleInfoDaemonWarnings.Warnings = []string{
 		"WARNING: No memory limit support",
 		"WARNING: No swap limit support",
-		"WARNING: No kernel memory limit support",
 		"WARNING: No oom kill disable support",
 		"WARNING: No cpu cfs quota support",
 		"WARNING: No cpu cfs period support",

cli/command/system/testdata/docker-info-daemon-warnings.json.golden

@@ -1 +1 @@
-{"ID":"EKHL:QDUU:QZ7U:MKGD:VDXK:S27Q:GIPU:24B7:R7VT:DGN6:QCSF:2UBX","Containers":0,"ContainersRunning":0,"ContainersPaused":0,"ContainersStopped":0,"Images":0,"Driver":"aufs","DriverStatus":[["Root Dir","/var/lib/docker/aufs"],["Backing Filesystem","extfs"],["Dirs","0"],["Dirperm1 Supported","true"]],"Plugins":{"Volume":["local"],"Network":["bridge","host","macvlan","null","overlay"],"Authorization":null,"Log":["awslogs","fluentd","gcplogs","gelf","journald","json-file","logentries","splunk","syslog"]},"MemoryLimit":true,"SwapLimit":true,"KernelMemory":true,"KernelMemoryTCP":false,"CpuCfsPeriod":true,"CpuCfsQuota":true,"CPUShares":true,"CPUSet":true,"PidsLimit":false,"IPv4Forwarding":true,"BridgeNfIptables":true,"BridgeNfIp6tables":true,"Debug":true,"NFd":33,"OomKillDisable":true,"NGoroutines":135,"SystemTime":"2017-08-24T17:44:34.077811894Z","LoggingDriver":"json-file","CgroupDriver":"cgroupfs","NEventsListener":0,"KernelVersion":"4.4.0-87-generic","OperatingSystem":"Ubuntu 16.04.3 LTS","OSVersion":"","OSType":"linux","Architecture":"x86_64","IndexServerAddress":"https://index.docker.io/v1/","RegistryConfig":{"AllowNondistributableArtifactsCIDRs":null,"AllowNondistributableArtifactsHostnames":null,"InsecureRegistryCIDRs":["127.0.0.0/8"],"IndexConfigs":{"docker.io":{"Name":"docker.io","Mirrors":null,"Secure":true,"Official":true}},"Mirrors":null},"NCPU":2,"MemTotal":2097356800,"GenericResources":null,"DockerRootDir":"/var/lib/docker","HttpProxy":"","HttpsProxy":"","NoProxy":"","Name":"system-sample","Labels":["provider=digitalocean"],"ExperimentalBuild":false,"ServerVersion":"17.06.1-ce","Runtimes":{"runc":{"path":"docker-runc"}},"DefaultRuntime":"runc","Swarm":{"NodeID":"","NodeAddr":"","LocalNodeState":"inactive","ControlAvailable":false,"Error":"","RemoteManagers":null},"LiveRestoreEnabled":false,"Isolation":"","InitBinary":"docker-init","ContainerdCommit":{"ID":"6e23458c129b551d5c9871e5174f6b1b7f6d1170","Expected":"6e23458c129b551d5c9871e5174f6b1b7f6d1170"},"RuncCommit":{"ID":"810190ceaa507aa2727d7ae6f4790c76ec150bd2","Expected":"810190ceaa507aa2727d7ae6f4790c76ec150bd2"},"InitCommit":{"ID":"949e6fa","Expected":"949e6fa"},"SecurityOptions":["name=apparmor","name=seccomp,profile=default"],"DefaultAddressPools":[{"Base":"10.123.0.0/16","Size":24}],"Warnings":["WARNING: No memory limit support","WARNING: No swap limit support","WARNING: No kernel memory limit support","WARNING: No oom kill disable support","WARNING: No cpu cfs quota support","WARNING: No cpu cfs period support","WARNING: No cpu shares support","WARNING: No cpuset support","WARNING: IPv4 forwarding is disabled","WARNING: bridge-nf-call-iptables is disabled","WARNING: bridge-nf-call-ip6tables is disabled"],"ClientInfo":{"Debug":true,"Context":"default","Plugins":[],"Warnings":null}}
+{"ID":"EKHL:QDUU:QZ7U:MKGD:VDXK:S27Q:GIPU:24B7:R7VT:DGN6:QCSF:2UBX","Containers":0,"ContainersRunning":0,"ContainersPaused":0,"ContainersStopped":0,"Images":0,"Driver":"aufs","DriverStatus":[["Root Dir","/var/lib/docker/aufs"],["Backing Filesystem","extfs"],["Dirs","0"],["Dirperm1 Supported","true"]],"Plugins":{"Volume":["local"],"Network":["bridge","host","macvlan","null","overlay"],"Authorization":null,"Log":["awslogs","fluentd","gcplogs","gelf","journald","json-file","logentries","splunk","syslog"]},"MemoryLimit":true,"SwapLimit":true,"KernelMemory":true,"KernelMemoryTCP":false,"CpuCfsPeriod":true,"CpuCfsQuota":true,"CPUShares":true,"CPUSet":true,"PidsLimit":false,"IPv4Forwarding":true,"BridgeNfIptables":true,"BridgeNfIp6tables":true,"Debug":true,"NFd":33,"OomKillDisable":true,"NGoroutines":135,"SystemTime":"2017-08-24T17:44:34.077811894Z","LoggingDriver":"json-file","CgroupDriver":"cgroupfs","NEventsListener":0,"KernelVersion":"4.4.0-87-generic","OperatingSystem":"Ubuntu 16.04.3 LTS","OSVersion":"","OSType":"linux","Architecture":"x86_64","IndexServerAddress":"https://index.docker.io/v1/","RegistryConfig":{"AllowNondistributableArtifactsCIDRs":null,"AllowNondistributableArtifactsHostnames":null,"InsecureRegistryCIDRs":["127.0.0.0/8"],"IndexConfigs":{"docker.io":{"Name":"docker.io","Mirrors":null,"Secure":true,"Official":true}},"Mirrors":null},"NCPU":2,"MemTotal":2097356800,"GenericResources":null,"DockerRootDir":"/var/lib/docker","HttpProxy":"","HttpsProxy":"","NoProxy":"","Name":"system-sample","Labels":["provider=digitalocean"],"ExperimentalBuild":false,"ServerVersion":"17.06.1-ce","Runtimes":{"runc":{"path":"docker-runc"}},"DefaultRuntime":"runc","Swarm":{"NodeID":"","NodeAddr":"","LocalNodeState":"inactive","ControlAvailable":false,"Error":"","RemoteManagers":null},"LiveRestoreEnabled":false,"Isolation":"","InitBinary":"docker-init","ContainerdCommit":{"ID":"6e23458c129b551d5c9871e5174f6b1b7f6d1170","Expected":"6e23458c129b551d5c9871e5174f6b1b7f6d1170"},"RuncCommit":{"ID":"810190ceaa507aa2727d7ae6f4790c76ec150bd2","Expected":"810190ceaa507aa2727d7ae6f4790c76ec150bd2"},"InitCommit":{"ID":"949e6fa","Expected":"949e6fa"},"SecurityOptions":["name=apparmor","name=seccomp,profile=default"],"DefaultAddressPools":[{"Base":"10.123.0.0/16","Size":24}],"Warnings":["WARNING: No memory limit support","WARNING: No swap limit support","WARNING: No oom kill disable support","WARNING: No cpu cfs quota support","WARNING: No cpu cfs period support","WARNING: No cpu shares support","WARNING: No cpuset support","WARNING: IPv4 forwarding is disabled","WARNING: bridge-nf-call-iptables is disabled","WARNING: bridge-nf-call-ip6tables is disabled"],"ClientInfo":{"Debug":true,"Context":"default","Plugins":[],"Warnings":null}}

cli/command/system/testdata/docker-info-warnings.golden

@@ -1,6 +1,5 @@
 WARNING: No memory limit support
 WARNING: No swap limit support
-WARNING: No kernel memory limit support
 WARNING: No oom kill disable support
 WARNING: No cpu cfs quota support
 WARNING: No cpu cfs period support

cli/config/config.go

@@ -24,12 +24,12 @@ const (
 )
 
 var (
-	initConfigDir sync.Once
+	initConfigDir = new(sync.Once)
 	configDir     string
 	homeDir       string
 )
 
-// resetHomeDir is used in testing to resets the "homeDir" package variable to
+// resetHomeDir is used in testing to reset the "homeDir" package variable to
 // force re-lookup of the home directory between tests.
 func resetHomeDir() {
 	homeDir = ""
@@ -42,6 +42,13 @@ func getHomeDir() string {
 	return homeDir
 }
 
+// resetConfigDir is used in testing to reset the "configDir" package variable
+// and its sync.Once to force re-lookup between tests.
+func resetConfigDir() {
+	configDir = ""
+	initConfigDir = new(sync.Once)
+}
+
 func setConfigDir() {
 	if configDir != "" {
 		return
@@ -97,10 +104,15 @@ func LoadFromReader(configData io.Reader) (*configfile.ConfigFile, error) {
 	return &configFile, err
 }
 
+// TODO remove this temporary hack, which is used to warn about the deprecated ~/.dockercfg file
+var printLegacyFileWarning bool
+
 // Load reads the configuration files in the given directory, and sets up
 // the auth config information and returns values.
 // FIXME: use the internal golang config parser
 func Load(configDir string) (*configfile.ConfigFile, error) {
+	printLegacyFileWarning = false
+
 	if configDir == "" {
 		configDir = Dir()
 	}
@@ -125,6 +137,7 @@ func Load(configDir string) (*configfile.ConfigFile, error) {
 	// Can't find latest config file so check for the old one
 	filename = filepath.Join(getHomeDir(), oldConfigfile)
 	if file, err := os.Open(filename); err == nil {
+		printLegacyFileWarning = true
 		defer file.Close()
 		if err := configFile.LegacyLoadFromReader(file); err != nil {
 			return configFile, errors.Wrap(err, filename)
@@ -140,6 +153,9 @@ func LoadDefaultConfigFile(stderr io.Writer) *configfile.ConfigFile {
 	if err != nil {
 		fmt.Fprintf(stderr, "WARNING: Error loading config file: %v\n", err)
 	}
+	if printLegacyFileWarning {
+		_, _ = fmt.Fprintln(stderr, "WARNING: Support for the legacy ~/.dockercfg configuration file and file-format is deprecated and will be removed in an upcoming release")
+	}
 	if !configFile.ContainsAuth() {
 		configFile.CredentialsStore = credentials.DetectDefaultStore(configFile.CredentialsStore)
 	}

cli/config/config_test.go

@@ -12,9 +12,11 @@ import (
 
 	"github.com/docker/cli/cli/config/configfile"
 	"github.com/docker/cli/cli/config/credentials"
+	"github.com/docker/cli/cli/config/types"
 	"gotest.tools/v3/assert"
 	is "gotest.tools/v3/assert/cmp"
 	"gotest.tools/v3/env"
+	"gotest.tools/v3/fs"
 )
 
 var homeKey = "HOME"
@@ -223,6 +225,31 @@ func TestOldJSON(t *testing.T) {
 	}
 }
 
+func TestOldJSONFallbackDeprecationWarning(t *testing.T) {
+	js := `{"https://index.docker.io/v1/":{"auth":"am9lam9lOmhlbGxv","email":"user@example.com"}}`
+	tmpHome := fs.NewDir(t, t.Name(), fs.WithFile(oldConfigfile, js))
+	defer tmpHome.Remove()
+	defer env.PatchAll(t, map[string]string{homeKey: tmpHome.Path(), "DOCKER_CONFIG": ""})()
+
+	// reset the homeDir, configDir, and its sync.Once, to force them being resolved again
+	resetHomeDir()
+	resetConfigDir()
+
+	buffer := new(bytes.Buffer)
+	configFile := LoadDefaultConfigFile(buffer)
+	expected := configfile.New(tmpHome.Join(configFileDir, ConfigFileName))
+	expected.AuthConfigs = map[string]types.AuthConfig{
+		"https://index.docker.io/v1/": {
+			Username:      "joejoe",
+			Password:      "hello",
+			Email:         "user@example.com",
+			ServerAddress: "https://index.docker.io/v1/",
+		},
+	}
+	assert.Assert(t, strings.Contains(buffer.String(), "WARNING: Support for the legacy ~/.dockercfg configuration file and file-format is deprecated and will be removed in an upcoming release"))
+	assert.Check(t, is.DeepEqual(expected, configFile))
+}
+
 func TestNewJSON(t *testing.T) {
 	tmpHome, err := ioutil.TempDir("", "config-test")
 	assert.NilError(t, err)

cli/winresources/res_windows.go

@@ -5,7 +5,7 @@ These resources are used to provide
     * An icon
     * A Windows manifest declaring Windows version support
 
-The resource object files are generated with go generate.
+The resource object files are generated when building with scripts/build/binary .
 The resource source files are located in scripts/winresources.
 This occurs automatically when you run scripts/build/windows.
 
@@ -14,5 +14,3 @@ is included.
 
 */
 package winresources
-
-//go:generate ../../scripts/gen/windows-resources

cmd/docker/docker_test.go

@@ -17,7 +17,9 @@ import (
 func TestClientDebugEnabled(t *testing.T) {
 	defer debug.Disable()
 
-	tcmd := newDockerCommand(&command.DockerCli{})
+	cli, err := command.NewDockerCli()
+	assert.NilError(t, err)
+	tcmd := newDockerCommand(cli)
 	tcmd.SetFlag("debug", "true")
 	cmd, _, err := tcmd.HandleGlobalFlags()
 	assert.NilError(t, err)

docker-bake.hcl

@@ -0,0 +1,63 @@
+variable "VERSION" {
+    default = ""
+}
+
+variable "USE_GLIBC" {
+    default = ""
+}
+
+variable "STRIP_TARGET" {
+    default = ""
+}
+
+group "default" {
+    targets = ["binary"]
+}
+
+target "binary" {
+    target = "binary"
+    platforms = ["local"]
+    output = ["build"]
+    args = {
+        BASE_VARIANT = USE_GLIBC != "" ? "buster" : "alpine"
+        VERSION = VERSION
+        GO_STRIP = STRIP_TARGET
+    }
+}
+
+target "dynbinary" {
+    inherits = ["binary"]
+    args = {
+        GO_LINKMODE = "dynamic"
+    }
+}
+
+variable "GROUP_TOTAL" {
+    default = "1"
+}
+
+variable "GROUP_INDEX" {
+    default = "0"
+}
+
+function "platforms" {
+    params = []
+    result = ["linux/amd64", "linux/386", "linux/arm64", "linux/arm", "linux/ppc64le", "linux/s390x", "darwin/amd64", "darwin/arm64", "windows/amd64"]
+}
+
+function "glen" {
+    params = [platforms, GROUP_TOTAL]
+    result = ceil(length(platforms)/GROUP_TOTAL)
+}
+
+target "_all_platforms" {
+    platforms = slice(platforms(), GROUP_INDEX*glen(platforms(), GROUP_TOTAL),min(length(platforms()), (GROUP_INDEX+1)*glen(platforms(), GROUP_TOTAL)))
+}
+
+target "cross" {
+    inherits = ["binary", "_all_platforms"]
+}
+
+target "dynbinary-cross" {
+    inherits = ["dynbinary", "_all_platforms"]
+}

docker.Makefile

@@ -38,11 +38,6 @@ build_linter_image:
 	# build dockerfile from stdin so that we don't send the build-context; source is bind-mounted in the development environment
 	cat ./dockerfiles/Dockerfile.lint | docker build ${DOCKER_BUILD_ARGS} --build-arg=GO_VERSION -t $(LINTER_IMAGE_NAME) -
 
-.PHONY: build_cross_image
-build_cross_image:
-	# build dockerfile from stdin so that we don't send the build-context; source is bind-mounted in the development environment
-	cat ./dockerfiles/Dockerfile.cross | docker build ${DOCKER_BUILD_ARGS} --build-arg=GO_VERSION -t $(CROSS_IMAGE_NAME) -
-
 .PHONY: build_shell_validate_image
 build_shell_validate_image:
 	# build dockerfile from stdin so that we don't send the build-context; source is bind-mounted in the development environment
@@ -80,22 +75,10 @@ test-unit: build_docker_image ## run unit tests (using go test)
 .PHONY: test ## run unit and e2e tests
 test: test-unit test-e2e
 
-.PHONY: cross
-cross: build_cross_image ## build the CLI for macOS and Windows
-	$(DOCKER_RUN) $(CROSS_IMAGE_NAME) make cross
-
-.PHONY: binary-windows
-binary-windows: build_cross_image ## build the CLI for Windows
-	$(DOCKER_RUN) $(CROSS_IMAGE_NAME) make $@
-
 .PHONY: plugins-windows
 plugins-windows: build_cross_image ## build the example CLI plugins for Windows
 	$(DOCKER_RUN) $(CROSS_IMAGE_NAME) make $@
 
-.PHONY: binary-osx
-binary-osx: build_cross_image ## build the CLI for macOS
-	$(DOCKER_RUN) $(CROSS_IMAGE_NAME) make $@
-
 .PHONY: plugins-osx
 plugins-osx: build_cross_image ## build the example CLI plugins for macOS
 	$(DOCKER_RUN) $(CROSS_IMAGE_NAME) make $@
@@ -120,9 +103,6 @@ fmt: ## run gofmt
 vendor: build_docker_image vendor.conf ## download dependencies (vendor/) listed in vendor.conf
 	$(DOCKER_RUN) -it $(DEV_DOCKER_IMAGE_NAME) make vendor
 
-dynbinary: build_cross_image ## build the CLI dynamically linked
-	$(DOCKER_RUN) -it $(CROSS_IMAGE_NAME) make dynbinary
-
 .PHONY: authors
 authors: ## generate AUTHORS file from git history
 	$(DOCKER_RUN) -it $(DEV_DOCKER_IMAGE_NAME) make authors

docs/extend/index.md

@@ -122,7 +122,7 @@ enabled, and use it to create a volume.
 To disable a plugin, use the `docker plugin disable` command. To completely
 remove it, use the `docker plugin remove` command. For other available
 commands and options, see the
-[command line reference](../reference/commandline/index.md).
+[command line reference](https://docs.docker.com/engine/reference/commandline/cli/).
 
 
 ## Developing a plugin

docs/reference/builder.md

@@ -33,11 +33,11 @@ a `Dockerfile` and a *context*. The build's context is the set of files at a
 specified location `PATH` or `URL`. The `PATH` is a directory on your local
 filesystem. The `URL` is a Git repository location.
 
-A context is processed recursively. So, a `PATH` includes any subdirectories and
-the `URL` includes the repository and its submodules. This example shows a
-build command that uses the current directory as context:
+The build context is processed recursively. So, a `PATH` includes any subdirectories
+and the `URL` includes the repository and its submodules. This example shows a
+build command that uses the current directory (`.`) as build context:
 
-```bash
+```console
 $ docker build .
 
 Sending build context to Docker daemon  6.51 MB
@@ -52,8 +52,9 @@ Dockerfile.
 
 > **Warning**
 >
-> Do not use your root directory, `/`, as the `PATH` as it causes the build to
-> transfer the entire contents of your hard drive to the Docker daemon.
+> Do not use your root directory, `/`, as the `PATH` for  your build context, as
+> it causes the build to transfer the entire contents of your hard drive to the
+> Docker daemon.
 {:.warning}
 
 To use a file in the build context, the `Dockerfile` refers to the file specified
@@ -66,32 +67,37 @@ Traditionally, the `Dockerfile` is called `Dockerfile` and located in the root
 of the context. You use the `-f` flag with `docker build` to point to a Dockerfile
 anywhere in your file system.
 
-```bash
+```console
 $ docker build -f /path/to/a/Dockerfile .
 ```
 
 You can specify a repository and tag at which to save the new image if
 the build succeeds:
 
-```bash
+```console
 $ docker build -t shykes/myapp .
 ```
 
 To tag the image into multiple repositories after the build,
 add multiple `-t` parameters when you run the `build` command:
 
-```bash
+```console
 $ docker build -t shykes/myapp:1.0.2 -t shykes/myapp:latest .
 ```
 
 Before the Docker daemon runs the instructions in the `Dockerfile`, it performs
 a preliminary validation of the `Dockerfile` and returns an error if the syntax is incorrect:
 
-```bash
+```console
 $ docker build -t test/myapp .
 
-Sending build context to Docker daemon 2.048 kB
-Error response from daemon: Unknown instruction: RUNCMD
+[+] Building 0.3s (2/2) FINISHED
+ => [internal] load build definition from Dockerfile                       0.1s
+ => => transferring dockerfile: 60B                                        0.0s
+ => [internal] load .dockerignore                                          0.1s
+ => => transferring context: 2B                                            0.0s
+error: failed to solve: rpc error: code = Unknown desc = failed to solve with frontend dockerfile.v0: failed to create LLB definition:
+dockerfile parse error line 2: unknown instruction: RUNCMD
 ```
 
 The Docker daemon runs the instructions in the `Dockerfile` one-by-one,
@@ -104,38 +110,35 @@ Note that each instruction is run independently, and causes a new image
 to be created - so `RUN cd /tmp` will not have any effect on the next
 instructions.
 
-Whenever possible, Docker will re-use the intermediate images (cache),
-to accelerate the `docker build` process significantly. This is indicated by
-the `Using cache` message in the console output.
-(For more information, see the [`Dockerfile` best practices guide](https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/):
+Whenever possible, Docker uses a build-cache to accelerate the `docker build`
+process significantly. This is indicated by the `CACHED` message in the console
+output. (For more information, see the [`Dockerfile` best practices guide](https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/):
 
-```bash
+```console
 $ docker build -t svendowideit/ambassador .
 
-Sending build context to Docker daemon 15.36 kB
-Step 1/4 : FROM alpine:3.2
- ---> 31f630c65071
-Step 2/4 : MAINTAINER SvenDowideit@home.org.au
- ---> Using cache
- ---> 2a1c91448f5f
-Step 3/4 : RUN apk update &&      apk add socat &&        rm -r /var/cache/
- ---> Using cache
- ---> 21ed6e7fbb73
-Step 4/4 : CMD env | grep _TCP= | (sed 's/.*_PORT_\([0-9]*\)_TCP=tcp:\/\/\(.*\):\(.*\)/socat -t 100000000 TCP4-LISTEN:\1,fork,reuseaddr TCP4:\2:\3 \&/' && echo wait) | sh
- ---> Using cache
- ---> 7ea8aef582cc
-Successfully built 7ea8aef582cc
+[+] Building 0.7s (6/6) FINISHED
+ => [internal] load build definition from Dockerfile                       0.1s
+ => => transferring dockerfile: 286B                                       0.0s
+ => [internal] load .dockerignore                                          0.1s
+ => => transferring context: 2B                                            0.0s
+ => [internal] load metadata for docker.io/library/alpine:3.2              0.4s
+ => CACHED [1/2] FROM docker.io/library/alpine:3.2@sha256:e9a2035f9d0d7ce  0.0s
+ => CACHED [2/2] RUN apk add --no-cache socat                              0.0s
+ => exporting to image                                                     0.0s
+ => => exporting layers                                                    0.0s
+ => => writing image sha256:1affb80ca37018ac12067fa2af38cc5bcc2a8f09963de  0.0s
+ => => naming to docker.io/svendowideit/ambassador                         0.0s
 ```
 
-Build cache is only used from images that have a local parent chain. This means
-that these images were created by previous builds or the whole chain of images
-was loaded with `docker load`. If you wish to use build cache of a specific
-image you can specify it with `--cache-from` option. Images specified with
-`--cache-from` do not need to have a parent chain and may be pulled from other
-registries.
+By default, the build cache is based on results from previous builds on the machine
+on which you are building. The `--cache-from` option also allows you to use a
+build-cache that's distributed through an image registry refer to the
+[specifying external cache sources](commandline/build.md#specifying-external-cache-sources)
+section in the `docker build` command reference.
 
-When you're done with your build, you're ready to look into [*Pushing a
-repository to its registry*](https://docs.docker.com/engine/tutorials/dockerrepos/#/contributing-to-docker-hub).
+When you're done with your build, you're ready to look into [scanning your image with `docker scan`](https://docs.docker.com/engine/scan/),
+and [pushing your image to Docker Hub](https://docs.docker.com/docker-hub/repos/).
 
 
 ## BuildKit
@@ -315,6 +318,8 @@ The following parser directives are supported:
 
 ## syntax
 
+<a name="external-implementation-features"><!-- included for deep-links to old section --></a>
+
 ```dockerfile
 # syntax=[remote image reference]
 ```
@@ -322,55 +327,73 @@ The following parser directives are supported:
 For example:
 
 ```dockerfile
-# syntax=docker/dockerfile
-# syntax=docker/dockerfile:1.0
+# syntax=docker/dockerfile:1
 # syntax=docker.io/docker/dockerfile:1
-# syntax=docker/dockerfile:1.0.0-experimental
 # syntax=example.com/user/repo:tag@sha256:abcdef...
 ```
 
-This feature is only enabled if the [BuildKit](#buildkit) backend is used.
+This feature is only available when using the [BuildKit](#buildkit) backend, and
+is ignored when using the classic builder backend.
 
-The syntax directive defines the location of the Dockerfile builder that is used for
-building the current Dockerfile. The BuildKit backend allows to seamlessly use
-external implementations of builders that are distributed as Docker images and
-execute inside a container sandbox environment.
+The syntax directive defines the location of the Dockerfile syntax that is used
+to build the Dockerfile. The BuildKit backend allows to seamlessly use external
+implementations that are distributed as Docker images and execute inside a
+container sandbox environment.
 
-Custom Dockerfile implementation allows you to:
+Custom Dockerfile implementations allows you to:
 
-  - Automatically get bugfixes without updating the daemon
+  - Automatically get bugfixes without updating the Docker daemon
   - Make sure all users are using the same implementation to build your Dockerfile
-  - Use the latest features without updating the daemon
-  - Try out new experimental or third-party features
+  - Use the latest features without updating the Docker daemon
+  - Try out new features or third-party features before they are integrated in the Docker daemon
+  - Use [alternative build definitions, or create your own](https://github.com/moby/buildkit#exploring-llb)
 
 ### Official releases
 
 Docker distributes official versions of the images that can be used for building
 Dockerfiles under `docker/dockerfile` repository on Docker Hub. There are two
-channels where new images are released: stable and experimental.
+channels where new images are released: `stable` and `labs`.
 
-Stable channel follows semantic versioning. For example:
+Stable channel follows [semantic versioning](https://semver.org). For example:
 
-  - `docker/dockerfile:1.0.0` - only allow immutable version `1.0.0`
-  - `docker/dockerfile:1.0` - allow versions `1.0.*`
-  - `docker/dockerfile:1` - allow versions `1.*.*`
-  - `docker/dockerfile:latest` - latest release on stable channel
+  - `docker/dockerfile:1` - kept updated with the latest `1.x.x` minor _and_ patch release
+  - `docker/dockerfile:1.2` -  kept updated with the latest `1.2.x` patch release,
+    and stops receiving updates once version `1.3.0` is released.
+  - `docker/dockerfile:1.2.1` - immutable: never updated
 
-The experimental channel uses incremental versioning with the major and minor
-component from the stable channel on the time of the release. For example:
+We recommend using `docker/dockerfile:1`, which always points to the latest stable
+release of the version 1 syntax, and receives both "minor" and "patch" updates
+for the version 1 release cycle. BuildKit automatically checks for updates of the
+syntax when performing a build, making sure you are using the most current version.
 
-  - `docker/dockerfile:1.0.1-experimental` - only allow immutable version `1.0.1-experimental`
-  - `docker/dockerfile:1.0-experimental` - latest experimental releases after `1.0`
-  - `docker/dockerfile:experimental` - latest release on experimental channel
+If a specific version is used, such as `1.2` or `1.2.1`, the Dockerfile needs to
+be updated manually to continue receiving bugfixes and new features. Old versions
+of the Dockerfile remain compatible with the new versions of the builder.
 
-You should choose a channel that best fits your needs. If you only want
-bugfixes, you should use `docker/dockerfile:1.0`. If you want to benefit from
-experimental features, you should use the experimental channel. If you are using
-the experimental channel, newer releases may not be backwards compatible, so it
+**labs channel**
+
+The "labs" channel provides early access to Dockerfile features that are not yet
+available in the stable channel. Labs channel images are released in conjunction
+with the stable releases, and follow the same versioning with the `-labs` suffix,
+for example:
+
+  - `docker/dockerfile:labs` - latest release on labs channel
+  - `docker/dockerfile:1-labs` - same as `dockerfile:1` in the stable channel, with labs features enabled
+  - `docker/dockerfile:1.2-labs` -  same as `dockerfile:1.2` in the stable channel, with labs features enabled
+  - `docker/dockerfile:1.2.1-labs` - immutable: never updated. Same as `dockerfile:1.2.1` in the stable channel, with labs features enabled
+
+Choose a channel that best fits your needs; if you want to benefit from
+new features, use the labs channel. Images in the labs channel provide a superset
+of the features in the stable channel; note that `stable` features in the labs
+channel images follow [semantic versioning](https://semver.org), but "labs"
+features do not, and newer releases may not be backwards compatible, so it
 is recommended to use an immutable full version variant.
 
-For master builds and nightly feature releases refer to the description in
-[the source repository](https://github.com/moby/buildkit/blob/master/README.md).
+For documentation on "labs" features, master builds, and nightly feature releases,
+refer to the description in [the BuildKit source repository on GitHub](https://github.com/moby/buildkit/blob/master/README.md).
+For a full list of available images, visit the [image repository on Docker Hub](https://hub.docker.com/r/docker/dockerfile),
+and the [docker/dockerfile-upstream image repository](https://hub.docker.com/r/docker/dockerfile-upstream)
+for development builds.
 
 ## escape
 
@@ -413,14 +436,15 @@ RUN dir c:\
 
 Results in:
 
-```powershell
-PS C:\John> docker build -t cmd .
+```console
+PS E:\myproject> docker build -t cmd .
+
 Sending build context to Docker daemon 3.072 kB
 Step 1/2 : FROM microsoft/nanoserver
  ---> 22738ff49c6d
 Step 2/2 : COPY testfile.txt c:\RUN dir c:
 GetFileAttributesEx c:RUN: The system cannot find the file specified.
-PS C:\John>
+PS E:\myproject>
 ```
 
 One solution to the above would be to use `/` as the target of both the `COPY`
@@ -441,8 +465,9 @@ RUN dir c:\
 
 Results in:
 
-```powershell
-PS C:\John> docker build -t succeeds --no-cache=true .
+```console
+PS E:\myproject> docker build -t succeeds --no-cache=true .
+
 Sending build context to Docker daemon 3.072 kB
 Step 1/3 : FROM microsoft/nanoserver
  ---> 22738ff49c6d
@@ -467,7 +492,7 @@ Step 3/3 : RUN dir c:\
  ---> 01c7f3bef04f
 Removing intermediate container a2c157f842f5
 Successfully built 01c7f3bef04f
-PS C:\John>
+PS E:\myproject>
 ```
 
 ## Environment replacement
@@ -910,8 +935,8 @@ the most-recently-applied value overrides any previously-set value.
 To view an image's labels, use the `docker image inspect` command. You can use
 the `--format` option to show just the labels;
  
-```bash
-docker image inspect --format='{{json .Config.Labels}}' myimage
+```console
+$ docker image inspect --format='{{json .Config.Labels}}' myimage
 ```
 ```json
 {
@@ -938,7 +963,7 @@ easily, for example with `docker inspect`. To set a label corresponding to the
 `MAINTAINER` field you could use:
 
 ```dockerfile
-LABEL maintainer="SvenDowideit@home.org.au"
+LABEL org.opencontainers.image.authors="SvenDowideit@home.org.au"
 ```
 
 This will then be visible from `docker inspect` with the other labels.
@@ -980,8 +1005,8 @@ port on the host, so the port will not be the same for TCP and UDP.
 Regardless of the `EXPOSE` settings, you can override them at runtime by using
 the `-p` flag. For example
 
-```bash
-docker run -p 80:80/tcp -p 80:80/udp ...
+```console
+$ docker run -p 80:80/tcp -p 80:80/udp ...
 ```
 
 To set up port redirection on the host system, see [using the -P flag](run.md#expose-incoming-ports).
@@ -1397,7 +1422,7 @@ An `ENTRYPOINT` allows you to configure a container that will run as an executab
 For example, the following starts nginx with its default content, listening
 on port 80:
 
-```bash
+```console
 $ docker run -i -t --rm -p 80:80 nginx
 ```
 
@@ -1432,7 +1457,7 @@ CMD ["-c"]
 
 When you run the container, you can see that `top` is the only process:
 
-```bash
+```console
 $ docker run -it --rm --name test  top -H
 
 top - 08:25:00 up  7:27,  0 users,  load average: 0.00, 0.01, 0.05
@@ -1447,7 +1472,7 @@ KiB Swap:  1441840 total,        0 used,  1441840 free.  1324440 cached Mem
 
 To examine the result further, you can use `docker exec`:
 
-```bash
+```console
 $ docker exec -it test ps aux
 
 USER       PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
@@ -1519,7 +1544,7 @@ If you run this image with `docker run -it --rm -p 80:80 --name test apache`,
 you can then examine the container's processes with `docker exec`, or `docker top`,
 and then ask the script to stop Apache:
 
-```bash
+```console
 $ docker exec -it test ps aux
 
 USER       PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
@@ -1579,7 +1604,7 @@ ENTRYPOINT exec top -b
 
 When you run this image, you'll see the single `PID 1` process:
 
-```bash
+```console
 $ docker run -it --rm --name test top
 
 Mem: 1704520K used, 352148K free, 0K shrd, 0K buff, 140368121167873K cached
@@ -1591,7 +1616,7 @@ Load average: 0.08 0.03 0.05 2/98 6
 
 Which exits cleanly on `docker stop`:
 
-```bash
+```console
 $ /usr/bin/time docker stop test
 
 test
@@ -1610,7 +1635,7 @@ CMD --ignored-param1
 
 You can then run it (giving it a name for the next step):
 
-```bash
+```console
 $ docker run -it --name test top --ignored-param2
 
 Mem: 1704184K used, 352484K free, 0K shrd, 0K buff, 140621524238337K cached
@@ -1626,7 +1651,7 @@ You can see from the output of `top` that the specified `ENTRYPOINT` is not `PID
 If you then run `docker stop test`, the container will not exit cleanly - the
 `stop` command will be forced to send a `SIGKILL` after the timeout:
 
-```bash
+```console
 $ docker exec -it test ps aux
 
 PID   USER     COMMAND
@@ -1859,9 +1884,10 @@ ARG user
 USER $user
 # ...
 ```
+
 A user builds this file by calling:
 
-```bash
+```console
 $ docker build --build-arg user=what_user .
 ```
 
@@ -1900,7 +1926,7 @@ RUN echo $CONT_IMG_VER
 
 Then, assume this image is built with this command:
 
-```bash
+```console
 $ docker build --build-arg CONT_IMG_VER=v2.0.1 .
 ```
 
@@ -1922,7 +1948,7 @@ RUN echo $CONT_IMG_VER
 Unlike an `ARG` instruction, `ENV` values are always persisted in the built
 image. Consider a docker build without the `--build-arg` flag:
 
-```bash
+```console
 $ docker build .
 ```
 
@@ -1948,10 +1974,11 @@ corresponding `ARG` instruction in the Dockerfile.
 - `NO_PROXY`
 - `no_proxy`
 
-To use these, simply pass them on the command line using the flag:
+To use these, pass them on the command line using the `--build-arg` flag, for
+example:
 
-```bash
---build-arg <varname>=<value>
+```console
+$ docker build --build-arg HTTPS_PROXY=https://my-proxy.example.com .
 ```
 
 By default, these pre-defined variables are excluded from the output of
@@ -2298,8 +2325,9 @@ RUN c:\example\Execute-MyCmdlet -sample 'hello world'
 
 Resulting in:
 
-```powershell
-PS E:\docker\build\shell> docker build -t shell .
+```console
+PS E:\myproject> docker build -t shell .
+
 Sending build context to Docker daemon 4.096 kB
 Step 1/5 : FROM microsoft/nanoserver
  ---> 22738ff49c6d
@@ -2314,9 +2342,9 @@ Step 3/5 : RUN New-Item -ItemType Directory C:\Example
     Directory: C:\
 
 
-Mode                LastWriteTime         Length Name
-----                -------------         ------ ----
-d-----       10/28/2016  11:26 AM                Example
+Mode         LastWriteTime              Length Name
+----         -------------              ------ ----
+d-----       10/28/2016  11:26 AM              Example
 
 
  ---> 3f2fbf1395d9
@@ -2330,7 +2358,7 @@ hello world
  ---> 8e559e9bf424
 Removing intermediate container be6d8e63fe75
 Successfully built 8e559e9bf424
-PS E:\docker\build\shell>
+PS E:\myproject>
 ```
 
 The `SHELL` instruction could also be used to modify the way in which
@@ -2340,61 +2368,10 @@ environment variable expansion semantics could be modified.
 The `SHELL` instruction can also be used on Linux should an alternate shell be
 required such as `zsh`, `csh`, `tcsh` and others.
 
-## External implementation features
-
-This feature is only available when using the  [BuildKit](#buildkit) backend.
-
-Docker build supports experimental features like cache mounts, build secrets and
-ssh forwarding that are enabled by using an external implementation of the
-builder with a syntax directive. To learn about these features,
-[refer to the documentation in BuildKit repository](https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/experimental.md).
-
 ## Dockerfile examples
 
-Below you can see some examples of Dockerfile syntax.
+For examples of Dockerfiles, refer to:
 
-```dockerfile
-# Nginx
-#
-# VERSION               0.0.1
-
-FROM      ubuntu
-LABEL Description="This image is used to start the foobar executable" Vendor="ACME Products" Version="1.0"
-RUN apt-get update && apt-get install -y inotify-tools nginx apache2 openssh-server
-```
-
-```dockerfile
-# Firefox over VNC
-#
-# VERSION               0.3
-
-FROM ubuntu
-
-# Install vnc, xvfb in order to create a 'fake' display and firefox
-RUN apt-get update && apt-get install -y x11vnc xvfb firefox
-RUN mkdir ~/.vnc
-# Setup a password
-RUN x11vnc -storepasswd 1234 ~/.vnc/passwd
-# Autostart firefox (might not be the best way, but it does the trick)
-RUN bash -c 'echo "firefox" >> /.bashrc'
-
-EXPOSE 5900
-CMD    ["x11vnc", "-forever", "-usepw", "-create"]
-```
-
-```dockerfile
-# Multiple images example
-#
-# VERSION               0.1
-
-FROM ubuntu
-RUN echo foo > bar
-# Will output something like ===> 907ad6c2736f
-
-FROM ubuntu
-RUN echo moo > oink
-# Will output something like ===> 695d7793cbe4
-
-# You'll now have two images, 907ad6c2736f with /bar, and 695d7793cbe4 with
-# /oink.
-```
+- The ["build images" section](https://docs.docker.com/develop/develop-images/dockerfile_best-practices/)
+- The ["get started](https://docs.docker.com/get-started/)
+- The [language-specific getting started guides](https://docs.docker.com/language/)

docs/reference/commandline/cli.md

@@ -1,7 +1,7 @@
 ---
 title: "Use the Docker command line"
 description: "Docker's CLI command description and usage"
-keywords: "Docker, Docker documentation, CLI, command line"
+keywords: "Docker, Docker documentation, CLI, command line, config.json, CLI configuration file"
 redirect_from:
   - /go/experimental/
   - /engine/reference/commandline/engine/
@@ -62,30 +62,22 @@ the [installation](https://docs.docker.com/install/) instructions for your opera
 
 ## Environment variables
 
-For easy reference, the following list of environment variables are supported
-by the `docker` command line:
+The following list of environment variables are supported by the `docker` command
+line:
 
-* `DOCKER_API_VERSION` The API version to use (e.g. `1.19`)
-* `DOCKER_CONFIG` The location of your client configuration files.
-* `DOCKER_HOST` Daemon socket to connect to.
-* `DOCKER_STACK_ORCHESTRATOR` Configure the default orchestrator to use when using `docker stack` management commands.
-* `DOCKER_CONTENT_TRUST` When set Docker uses notary to sign and verify images.
-  Equates to `--disable-content-trust=false` for build, create, pull, push, run.
-* `DOCKER_CONTENT_TRUST_SERVER` The URL of the Notary server to use. This defaults
-  to the same URL as the registry.
-* `DOCKER_HIDE_LEGACY_COMMANDS` When set, Docker hides "legacy" top-level commands (such as `docker rm`, and
-  `docker pull`) in `docker help` output, and only `Management commands` per object-type (e.g., `docker container`) are
-  printed. This may become the default in a future release, at which point this environment-variable is removed.
-* `DOCKER_CONTEXT` Specify the context to use (overrides DOCKER_HOST env var and default context set with "docker context use")
-* `DOCKER_DEFAULT_PLATFORM` Specify the default platform for the commands that take the `--platform` flag.
-
-#### Shared Environment variables
-
-These environment variables can be used both with the `docker` command line and
-`dockerd` command line:
-
-* `DOCKER_CERT_PATH` The location of your authentication keys.
-* `DOCKER_TLS_VERIFY` When set Docker uses TLS and verifies the remote.
+| Variable                      | Description                                                                                                                           |
+|:------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------|
+| `DOCKER_API_VERSION`          | Override the negotiated API version to use for debugging (e.g. `1.19`)                                                                |
+| `DOCKER_CERT_PATH`            | Location of your authentication keys. This variable is used both by the `docker` CLI and the [`dockerd` daemon](dockerd.md)             |
+| `DOCKER_CONFIG`               | The location of your client configuration files.                                                                                      |
+| `DOCKER_CONTENT_TRUST_SERVER` | The URL of the Notary server to use. Defaults to the same URL as the registry.                                                        |
+| `DOCKER_CONTENT_TRUST`        | When set Docker uses notary to sign and verify images. Equates to `--disable-content-trust=false` for build, create, pull, push, run. |
+| `DOCKER_CONTEXT`              | Name of the `docker context` to use (overrides `DOCKER_HOST` env var and default context set with `docker context use`)               |
+| `DOCKER_DEFAULT_PLATFORM`     | Default platform for commands that take the `--platform` flag.                                                                        |
+| `DOCKER_HIDE_LEGACY_COMMANDS` | When set, Docker hides "legacy" top-level commands (such as `docker rm`, and `docker pull`) in `docker help` output, and only `Management commands` per object-type (e.g., `docker container`) are printed. This may become the default in a future release, at which point this environment-variable is removed. |
+| `DOCKER_HOST`                 | Daemon socket to connect to.                                                                                                          |
+| `DOCKER_STACK_ORCHESTRATOR`   | Configure the default orchestrator to use when using `docker stack` management commands.                                              |
+| `DOCKER_TLS_VERIFY`           | When set Docker uses TLS and verifies the remote. This variable is used both by the `docker` CLI and the [`dockerd` daemon](dockerd.md) |
 
 Because Docker is developed using Go, you can also use any environment
 variables used by the Go runtime. In particular, you may find these useful:
@@ -98,7 +90,7 @@ These Go environment variables are case-insensitive. See the
 [Go specification](http://golang.org/pkg/net/http/) for details on these
 variables.
 
-### Configuration files
+## Configuration files
 
 By default, the Docker command line stores its configuration files in a
 directory called `.docker` within your `$HOME` directory.
@@ -124,7 +116,7 @@ specified, then the `--config` option overrides the `DOCKER_CONFIG` environment
 variable. The example below overrides the `docker ps` command using a
 `config.json` file located in the `~/testconfigs/` directory.
 
-```bash
+```console
 $ docker --config ~/testconfigs/ ps
 ```
 
@@ -133,79 +125,56 @@ configuration, you can set the `DOCKER_CONFIG` environment variable in your
 shell (e.g. `~/.profile` or `~/.bashrc`). The example below sets the new
 directory to be `HOME/newdir/.docker`.
 
-```bash
-echo export DOCKER_CONFIG=$HOME/newdir/.docker > ~/.profile
+```console
+$ echo export DOCKER_CONFIG=$HOME/newdir/.docker > ~/.profile
 ```
 
-### `config.json` properties
+## Docker CLI configuration file (`config.json`) properties
 
-The `config.json` file stores a JSON encoding of several properties:
+<a name="configjson-properties"><!-- included for deep-links to old section --></a>
+
+Use the Docker CLI configuration to customize settings for the `docker` CLI. The
+configuration file uses JSON formatting, and properties:
+
+By default, configuration file is stored in `~/.docker/config.json`. Refer to the
+[change the `.docker` directory](#change-the-docker-directory) section to use a
+different location.
+
+> **Warning**
+> 
+> The configuration file and other files inside the `~/.docker` configuration
+> directory may contain sensitive information, such as authentication information
+> for proxies or, depending on your credential store, credentials for your image
+> registries. Review your configuration file's content before sharing with others,
+> and prevent committing the file to version control.
+
+### Customize the default output format for commands
+
+These fields allow you to customize the default output format for some commands
+if no `--format` flag is provided.
+
+| Property               | Description                                                                                                                                                                                                                         |
+|:-----------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `configFormat`         | Custom default format for `docker config ls` output. Refer to the [**format the output** section in the `docker config ls` documentation](config_ls.md#format-the-output) for a list of supported formatting directives.            |
+| `imagesFormat`         | Custom default format for `docker images` / `docker image ls` output. Refer to the [**format the output** section in the `docker images` documentation](images.md#format-the-output) for a list of supported formatting directives. |
+| `nodesFormat`          | Custom default format for `docker node ls` output. Refer to the [**formatting** section in the `docker node ls` documentation](node_ls.md#formatting) for a list of supported formatting directives.                                |
+| `pluginsFormat`        | Custom default format for `docker plugin ls` output. Refer to the [**formatting** section in the `docker plugin ls` documentation](plugin_ls.md#formatting) for a list of supported formatting directives.                          |
+| `psFormat`             | Custom default format for `docker ps` / `docker container ps` output. Refer to the [**formatting** section in the `docker ps` documentation](ps.md#formatting) for a list of supported formatting directives.                       |
+| `secretFormat`         | Custom default format for `docker secret ls` output. Refer to the [**format the output** section in the `docker secret ls` documentation](secret_ls.md#format-the-output) for a list of supported formatting directives.            |
+| `serviceInspectFormat` | Custom default format for `docker service inspect` output. Refer to the [**formatting** section in the `docker service inspect` documentation](service_inspect.md#formatting) for a list of supported formatting directives.        |
+| `servicesFormat`       | Custom default format for `docker service ls` output. Refer to the [**formatting** section in the `docker service ls` documentation](service_ls.md#formatting) for a list of supported formatting directives.                       |
+| `statsFormat`          | Custom default format for `docker stats` output. Refer to the [**formatting** section in the `docker stats` documentation](stats.md#formatting) for a list of supported formatting directives.                                      |
+
+
+### Custom HTTP headers
 
 The property `HttpHeaders` specifies a set of headers to include in all messages
 sent from the Docker client to the daemon. Docker does not try to interpret or
-understand these header; it simply puts them into the messages. Docker does
+understand these headers; it simply puts them into the messages. Docker does
 not allow these headers to change any headers it sets for itself.
 
-The property `psFormat` specifies the default format for `docker ps` output.
-When the `--format` flag is not provided with the `docker ps` command,
-Docker's client uses this property. If this property is not set, the client
-falls back to the default table format. For a list of supported formatting
-directives, see the
-[**Formatting** section in the `docker ps` documentation](ps.md)
 
-The property `imagesFormat` specifies the default format for `docker images` output.
-When the `--format` flag is not provided with the `docker images` command,
-Docker's client uses this property. If this property is not set, the client
-falls back to the default table format. For a list of supported formatting
-directives, see the [**Formatting** section in the `docker images` documentation](images.md)
-
-The property `pluginsFormat` specifies the default format for `docker plugin ls` output.
-When the `--format` flag is not provided with the `docker plugin ls` command,
-Docker's client uses this property. If this property is not set, the client
-falls back to the default table format. For a list of supported formatting
-directives, see the [**Formatting** section in the `docker plugin ls` documentation](plugin_ls.md)
-
-The property `servicesFormat` specifies the default format for `docker
-service ls` output. When the `--format` flag is not provided with the
-`docker service ls` command, Docker's client uses this property. If this
-property is not set, the client falls back to the default json format. For a
-list of supported formatting directives, see the
-[**Formatting** section in the `docker service ls` documentation](service_ls.md)
-
-The property `serviceInspectFormat` specifies the default format for `docker
-service inspect` output. When the `--format` flag is not provided with the
-`docker service inspect` command, Docker's client uses this property. If this
-property is not set, the client falls back to the default json format. For a
-list of supported formatting directives, see the
-[**Formatting** section in the `docker service inspect` documentation](service_inspect.md)
-
-The property `statsFormat` specifies the default format for `docker
-stats` output. When the `--format` flag is not provided with the
-`docker stats` command, Docker's client uses this property. If this
-property is not set, the client falls back to the default table
-format. For a list of supported formatting directives, see
-[**Formatting** section in the `docker stats` documentation](stats.md)
-
-The property `secretFormat` specifies the default format for `docker
-secret ls` output. When the `--format` flag is not provided with the
-`docker secret ls` command, Docker's client uses this property. If this
-property is not set, the client falls back to the default table
-format. For a list of supported formatting directives, see
-[**Formatting** section in the `docker secret ls` documentation](secret_ls.md)
-
-
-The property `nodesFormat` specifies the default format for `docker node ls` output.
-When the `--format` flag is not provided with the `docker node ls` command,
-Docker's client uses the value of `nodesFormat`. If the value of `nodesFormat` is not set,
-the client uses the default table format. For a list of supported formatting
-directives, see the [**Formatting** section in the `docker node ls` documentation](node_ls.md)
-
-The property `configFormat` specifies the default format for `docker
-config ls` output. When the `--format` flag is not provided with the
-`docker config ls` command, Docker's client uses this property. If this
-property is not set, the client falls back to the default table
-format. For a list of supported formatting directives, see
-[**Formatting** section in the `docker config ls` documentation](config_ls.md)
+### Credential store options
 
 The property `credsStore` specifies an external binary to serve as the default
 credential store. When this property is set, `docker login` will attempt to
@@ -221,11 +190,17 @@ credentials for specific registries. If this property is set, the binary
 for a specific registry. For more information, see the
 [**Credential helpers** section in the `docker login` documentation](login.md#credential-helpers)
 
+
+### Orchestrator options for docker stacks
+
 The property `stackOrchestrator` specifies the default orchestrator to use when
 running `docker stack` management commands. Valid values are `"swarm"`,
 `"kubernetes"`, and `"all"`. This property can be overridden with the
 `DOCKER_STACK_ORCHESTRATOR` environment variable, or the `--orchestrator` flag.
 
+
+### Automatic proxy configuration for containers
+
 The property `proxies` specifies proxy environment variables to be automatically
 set on containers, and set as `--build-arg` on containers used during `docker build`.
 A `"default"` set of proxies can be configured, and will be used for any docker
@@ -233,15 +208,26 @@ daemon that the client connects to, or a configuration per host (docker daemon),
 for example, "https://docker-daemon1.example.com". The following properties can
 be set for each environment:
 
-* `httpProxy` (sets the value of `HTTP_PROXY` and `http_proxy`)
-* `httpsProxy` (sets the value of `HTTPS_PROXY` and `https_proxy`)
-* `ftpProxy` (sets the value of `FTP_PROXY` and `ftp_proxy`)
-* `noProxy` (sets the value of `NO_PROXY` and `no_proxy`)
+| Property       | Description                                                                                             |
+|:---------------|:--------------------------------------------------------------------------------------------------------|
+| `httpProxy`    | Default value of `HTTP_PROXY` and `http_proxy` for containers, and as `--build-arg` on `docker build`   |
+| `httpsProxy`   | Default value of `HTTPS_PROXY` and `https_proxy` for containers, and as `--build-arg` on `docker build` |
+| `ftpProxy`     | Default value of `FTP_PROXY` and `ftp_proxy` for containers, and as `--build-arg` on `docker build`     |
+| `noProxy`      | Default value of `NO_PROXY` and `no_proxy` for containers, and as `--build-arg` on `docker build`       |
 
-> **Warning**: Proxy settings may contain sensitive information (for example,
-> if the proxy requires authentication). Environment variables are stored as
-> plain text in the container's configuration, and as such can be inspected
-> through the remote API or committed to an image when using `docker commit`.
+These settings are used to configure proxy settings for containers only, and not
+used as proxy settings for the `docker` CLI or the `dockerd` daemon. Refer to the
+[environment variables](#environment-variables) and [HTTP/HTTPS proxy](https://docs.docker.com/config/daemon/systemd/#httphttps-proxy)
+sections for configuring proxy settings for the cli and daemon.
+
+> **Warning**
+> 
+> Proxy settings may contain sensitive information (for example, if the proxy
+> requires authentication). Environment variables are stored as plain text in
+> the container's configuration, and as such can be inspected through the remote
+> API or committed to an image when using `docker commit`.
+
+### Default key-sequence to detach from containers
 
 Once attached to a container, users detach from it and leave it running using
 the using `CTRL-p CTRL-q` key sequence. This detach key sequence is customizable
@@ -261,11 +247,17 @@ Users can override your custom or the default key sequence on a per-container
 basis. To do this, the user specifies the `--detach-keys` flag with the `docker
 attach`, `docker exec`, `docker run` or `docker start` command.
 
+### CLI Plugin options
+
 The property `plugins` contains settings specific to CLI plugins. The
 key is the plugin name, while the value is a further map of options,
 which are specific to that plugin.
 
-Following is a sample `config.json` file:
+
+### Sample configuration file
+
+Following is a sample `config.json` file to illustrate the format used for
+various fields:
 
 ```json
 {% raw %}
@@ -301,14 +293,14 @@ Following is a sample `config.json` file:
   "proxies": {
     "default": {
       "httpProxy":  "http://user:pass@example.com:3128",
-      "httpsProxy": "http://user:pass@example.com:3128",
-      "noProxy":    "http://user:pass@example.com:3128",
+      "httpsProxy": "https://my-proxy.example.com:3129",
+      "noProxy":    "intra.mycorp.example.com",
       "ftpProxy":   "http://user:pass@example.com:3128"
     },
     "https://manager1.mycorp.example.com:2377": {
       "httpProxy":  "http://user:pass@example.com:3128",
-      "httpsProxy": "http://user:pass@example.com:3128"
-    },
+      "httpsProxy": "https://my-proxy.example.com:3129"
+    }
   }
 }
 {% endraw %}
@@ -336,16 +328,18 @@ list of root Certificate Authorities.
 To list the help on any command just execute the command, followed by the
 `--help` option.
 
-    $ docker run --help
+```console
+$ docker run --help
 
-    Usage: docker run [OPTIONS] IMAGE [COMMAND] [ARG...]
+Usage: docker run [OPTIONS] IMAGE [COMMAND] [ARG...]
 
-    Run a command in a new container
+Run a command in a new container
 
-    Options:
-          --add-host value             Add a custom host-to-IP mapping (host:ip) (default [])
-      -a, --attach value               Attach to STDIN, STDOUT or STDERR (default [])
-    ...
+Options:
+      --add-host value             Add a custom host-to-IP mapping (host:ip) (default [])
+  -a, --attach value               Attach to STDIN, STDOUT or STDERR (default [])
+<...>
+```
 
 ### Option types
 
@@ -366,7 +360,7 @@ container **will** run in "detached" mode, in the background.
 Options which default to `true` (e.g., `docker build --rm=true`) can only be
 set to the non-default value by explicitly setting them to `false`:
 
-```bash
+```console
 $ docker build --rm=false .
 ```
 
@@ -375,7 +369,7 @@ $ docker build --rm=false .
 You can specify options like `-a=[]` multiple times in a single command line,
 for example in these commands:
 
-```bash
+```console
 $ docker run -a stdin -a stdout -i -t ubuntu /bin/bash
 
 $ docker run -a stdin -a stdout -a stderr ubuntu /bin/ls
@@ -384,7 +378,7 @@ $ docker run -a stdin -a stdout -a stderr ubuntu /bin/ls
 Sometimes, multiple options can call for a more complex value string as for
 `-v`:
 
-```bash
+```console
 $ docker run -v /host:/container example/mysql
 ```
 

docs/reference/commandline/config.md

@@ -0,0 +1,35 @@
+---
+title: "config"
+description: "The config command description and usage"
+keywords: "config"
+---
+
+# config
+
+```markdown
+Usage:  docker config COMMAND
+
+Manage Docker configs
+
+Options:
+      --help   Print usage
+
+Commands:
+  create      Create a config from a file or STDIN
+  inspect     Display detailed information on one or more configs
+  ls          List configs
+  rm          Remove one or more configs
+
+Run 'docker config COMMAND --help' for more information on a command.
+```
+
+## Description
+
+Manage configs.
+
+## Related commands
+
+* [config create](config_create.md)
+* [config inspect](config_inspect.md)
+* [config list](config_ls.md)
+* [config rm](config_rm.md)

docs/reference/commandline/config_create.md

@@ -0,0 +1,99 @@
+---
+title: "config create"
+description: "The config create command description and usage"
+keywords: ["config, create"]
+---
+
+# config create
+
+```Markdown
+Usage:  docker config create [OPTIONS] CONFIG [file|-]
+
+Create a config from a file or STDIN as content
+
+Options:
+  -l, --label list               Config labels
+      --template-driver string   Template driver
+```
+
+## Description
+
+Creates a config using standard input or from a file for the config content.
+
+For detailed information about using configs, refer to [store configuration data using Docker Configs](https://docs.docker.com/engine/swarm/configs/).
+
+> **Note**
+>
+> This is a cluster management command, and must be executed on a swarm
+> manager node. To learn about managers and workers, refer to the
+> [Swarm mode section](https://docs.docker.com/engine/swarm/) in the
+> documentation.
+
+## Examples
+
+### Create a config
+
+```bash
+$ printf <config> | docker config create my_config -
+
+onakdyv307se2tl7nl20anokv
+
+$ docker config ls
+
+ID                          NAME                CREATED             UPDATED
+onakdyv307se2tl7nl20anokv   my_config           6 seconds ago       6 seconds ago
+```
+
+### Create a config with a file
+
+```bash
+$ docker config create my_config ./config.json
+
+dg426haahpi5ezmkkj5kyl3sn
+
+$ docker config ls
+
+ID                          NAME                CREATED             UPDATED
+dg426haahpi5ezmkkj5kyl3sn   my_config           7 seconds ago       7 seconds ago
+```
+
+### Create a config with labels
+
+```bash
+$ docker config create \
+    --label env=dev \
+    --label rev=20170324 \
+    my_config ./config.json
+
+eo7jnzguqgtpdah3cm5srfb97
+```
+
+```bash
+$ docker config inspect my_config
+
+[
+    {
+        "ID": "eo7jnzguqgtpdah3cm5srfb97",
+        "Version": {
+            "Index": 17
+        },
+        "CreatedAt": "2017-03-24T08:15:09.735271783Z",
+        "UpdatedAt": "2017-03-24T08:15:09.735271783Z",
+        "Spec": {
+            "Name": "my_config",
+            "Labels": {
+                "env": "dev",
+                "rev": "20170324"
+            },
+            "Data": "aGVsbG8K"
+        }
+    }
+]
+```
+
+
+## Related commands
+
+* [config inspect](config_inspect.md)
+* [config ls](config_ls.md)
+* [config rm](config_rm.md)

docs/reference/commandline/config_inspect.md

@@ -0,0 +1,97 @@
+---
+title: "config inspect"
+description: "The config inspect command description and usage"
+keywords: ["config, inspect"]
+---
+
+# config inspect
+
+```Markdown
+Usage:  docker config inspect [OPTIONS] CONFIG [CONFIG...]
+
+Display detailed information on one or more configs
+
+Options:
+  -f, --format string   Format the output using the given Go template
+      --help            Print usage
+```
+
+## Description
+
+Inspects the specified config.
+
+By default, this renders all results in a JSON array. If a format is specified,
+the given template will be executed for each result.
+
+Go's [text/template](http://golang.org/pkg/text/template/) package
+describes all the details of the format.
+
+For detailed information about using configs, refer to [store configuration data using Docker Configs](https://docs.docker.com/engine/swarm/configs/).
+
+> **Note**
+>
+> This is a cluster management command, and must be executed on a swarm
+> manager node. To learn about managers and workers, refer to the
+> [Swarm mode section](https://docs.docker.com/engine/swarm/) in the
+> documentation.
+
+## Examples
+
+### Inspect a config by name or ID
+
+You can inspect a config, either by its *name*, or *ID*
+
+For example, given the following config:
+
+```bash
+$ docker config ls
+
+ID                          NAME                CREATED             UPDATED
+eo7jnzguqgtpdah3cm5srfb97   my_config           3 minutes ago       3 minutes ago
+```
+
+```bash
+$ docker config inspect config.json
+```
+
+The output is in JSON format, for example:
+
+```json
+[
+  {
+    "ID": "eo7jnzguqgtpdah3cm5srfb97",
+    "Version": {
+      "Index": 17
+    },
+    "CreatedAt": "2017-03-24T08:15:09.735271783Z",
+    "UpdatedAt": "2017-03-24T08:15:09.735271783Z",
+    "Spec": {
+      "Name": "my_config",
+      "Labels": {
+        "env": "dev",
+        "rev": "20170324"
+      },
+      "Data": "aGVsbG8K"
+    }
+  }
+]
+```
+
+### Formatting
+
+You can use the --format option to obtain specific information about a
+config. The following example command outputs the creation time of the
+config.
+
+```bash
+$ docker config inspect --format='{{.CreatedAt}}' eo7jnzguqgtpdah3cm5srfb97
+
+2017-03-24 08:15:09.735271783 +0000 UTC
+```
+
+
+## Related commands
+
+* [config create](config_create.md)
+* [config ls](config_ls.md)
+* [config rm](config_rm.md)

docs/reference/commandline/config_ls.md

@@ -0,0 +1,155 @@
+---
+title: "config ls"
+description: "The config ls command description and usage"
+keywords: ["config, ls"]
+---
+
+# config ls
+
+```Markdown
+Usage:  docker config ls [OPTIONS]
+
+List configs
+
+Aliases:
+  ls, list
+
+Options:
+  -f, --filter filter   Filter output based on conditions provided
+      --format string   Pretty-print configs using a Go template
+      --help            Print usage
+  -q, --quiet           Only display IDs
+```
+
+## Description
+
+Run this command on a manager node to list the configs in the swarm.
+
+For detailed information about using configs, refer to [store configuration data using Docker Configs](https://docs.docker.com/engine/swarm/configs/).
+
+> **Note**
+>
+> This is a cluster management command, and must be executed on a swarm
+> manager node. To learn about managers and workers, refer to the
+> [Swarm mode section](https://docs.docker.com/engine/swarm/) in the
+> documentation.
+
+## Examples
+
+```bash
+$ docker config ls
+
+ID                          NAME                        CREATED             UPDATED
+6697bflskwj1998km1gnnjr38   q5s5570vtvnimefos1fyeo2u2   6 weeks ago         6 weeks ago
+9u9hk4br2ej0wgngkga6rp4hq   my_config                   5 weeks ago         5 weeks ago
+mem02h8n73mybpgqjf0kfi1n0   test_config                 3 seconds ago       3 seconds ago
+```
+
+### Filtering
+
+The filtering flag (`-f` or `--filter`) format is a `key=value` pair. If there is more
+than one filter, then pass multiple flags (e.g., `--filter "foo=bar" --filter "bif=baz"`)
+
+The currently supported filters are:
+
+- [id](#id) (config's ID)
+- [label](#label) (`label=<key>` or `label=<key>=<value>`)
+- [name](#name) (config's name)
+
+#### id
+
+The `id` filter matches all or prefix of a config's id.
+
+```bash
+$ docker config ls -f "id=6697bflskwj1998km1gnnjr38"
+
+ID                          NAME                        CREATED             UPDATED
+6697bflskwj1998km1gnnjr38   q5s5570vtvnimefos1fyeo2u2   6 weeks ago         6 weeks ago
+```
+
+#### label
+
+The `label` filter matches configs based on the presence of a `label` alone or
+a `label` and a value.
+
+The following filter matches all configs with a `project` label regardless of
+its value:
+
+```bash
+$ docker config ls --filter label=project
+
+ID                          NAME                        CREATED             UPDATED
+mem02h8n73mybpgqjf0kfi1n0   test_config                 About an hour ago   About an hour ago
+```
+
+The following filter matches only services with the `project` label with the
+`project-a` value.
+
+```bash
+$ docker service ls --filter label=project=test
+
+ID                          NAME                        CREATED             UPDATED
+mem02h8n73mybpgqjf0kfi1n0   test_config                 About an hour ago   About an hour ago
+```
+
+#### name
+
+The `name` filter matches on all or prefix of a config's name.
+
+The following filter matches config with a name containing a prefix of `test`.
+
+```bash
+$ docker config ls --filter name=test_config
+
+ID                          NAME                        CREATED             UPDATED
+mem02h8n73mybpgqjf0kfi1n0   test_config                 About an hour ago   About an hour ago
+```
+
+### Format the output
+
+The formatting option (`--format`) pretty prints configs output
+using a Go template.
+
+Valid placeholders for the Go template are listed below:
+
+| Placeholder  | Description                                                                          |
+| ------------ | ------------------------------------------------------------------------------------ |
+| `.ID`        | Config ID                                                                            |
+| `.Name`      | Config name                                                                          |
+| `.CreatedAt` | Time when the config was created                                                     |
+| `.UpdatedAt` | Time when the config was updated                                                     |
+| `.Labels`    | All labels assigned to the config                                                    |
+| `.Label`     | Value of a specific label for this config. For example `{{.Label "my-label"}}`       |
+
+When using the `--format` option, the `config ls` command will either
+output the data exactly as the template declares or, when using the
+`table` directive, will include column headers as well.
+
+The following example uses a template without headers and outputs the
+`ID` and `Name` entries separated by a colon (`:`) for all images:
+
+```bash
+$ docker config ls --format "{{.ID}}: {{.Name}}"
+
+77af4d6b9913: config-1
+b6fa739cedf5: config-2
+78a85c484f71: config-3
+```
+
+To list all configs with their name and created date in a table format you
+can use:
+
+```bash
+$ docker config ls --format "table {{.ID}}\t{{.Name}}\t{{.CreatedAt}}"
+
+ID                  NAME                      CREATED
+77af4d6b9913        config-1                  5 minutes ago
+b6fa739cedf5        config-2                  3 hours ago
+78a85c484f71        config-3                  10 days ago
+```
+
+## Related commands
+
+* [config create](config_create.md)
+* [config inspect](config_inspect.md)
+* [config rm](config_rm.md)

docs/reference/commandline/config_rm.md

@@ -0,0 +1,53 @@
+---
+title: "config rm"
+description: "The config rm command description and usage"
+keywords: ["config, rm"]
+---
+
+# config rm
+
+```Markdown
+Usage:  docker config rm CONFIG [CONFIG...]
+
+Remove one or more configs
+
+Aliases:
+  rm, remove
+
+Options:
+      --help   Print usage
+```
+
+## Description
+
+Removes the specified configs from the swarm.
+
+For detailed information about using configs, refer to [store configuration data using Docker Configs](https://docs.docker.com/engine/swarm/configs/).
+
+> **Note**
+>
+> This is a cluster management command, and must be executed on a swarm
+> manager node. To learn about managers and workers, refer to the
+> [Swarm mode section](https://docs.docker.com/engine/swarm/) in the
+> documentation.
+
+## Examples
+
+This example removes a config:
+
+```bash
+$ docker config rm my_config
+sapth4csdo5b6wz2p5uimh5xg
+```
+
+> **Warning**
+>
+> Unlike `docker rm`, this command does not ask for confirmation before removing
+> a config.
+
+
+## Related commands
+
+* [config create](config_create.md)
+* [config inspect](config_inspect.md)
+* [config ls](config_ls.md)

docs/reference/commandline/dockerd.md

@@ -1203,8 +1203,8 @@ command line or Docker's Engine API are allowed or denied by the plugin.
 If you have multiple plugins installed, each plugin, in order, must
 allow the request for it to complete.
 
-For information about how to create an authorization plugin, see [authorization
-plugin](../../extend/plugins_authorization.md) section in the Docker extend section of this documentation.
+For information about how to create an authorization plugin, refer to the
+[authorization plugin](../../extend/plugins_authorization.md) section.
 
 
 ### Daemon user namespace options
@@ -1232,10 +1232,16 @@ Docker supports softlinks for the Docker data directory (`/var/lib/docker`) and
 for `/var/lib/docker/tmp`. The `DOCKER_TMPDIR` and the data directory can be
 set like this:
 
-    DOCKER_TMPDIR=/mnt/disk2/tmp /usr/local/bin/dockerd -D -g /var/lib/docker -H unix:// > /var/lib/docker-machine/docker.log 2>&1
-    # or
-    export DOCKER_TMPDIR=/mnt/disk2/tmp
-    /usr/local/bin/dockerd -D -g /var/lib/docker -H unix:// > /var/lib/docker-machine/docker.log 2>&1
+```console
+$ DOCKER_TMPDIR=/mnt/disk2/tmp /usr/local/bin/dockerd -D -g /var/lib/docker -H unix:// > /var/lib/docker-machine/docker.log 2>&1
+```
+
+or
+
+```console
+$ export DOCKER_TMPDIR=/mnt/disk2/tmp
+$ /usr/local/bin/dockerd -D -g /var/lib/docker -H unix:// > /var/lib/docker-machine/docker.log 2>&1
+````
 
 #### Default cgroup parent
 
@@ -1400,6 +1406,10 @@ This is a full example of the allowed configuration options on Linux:
   "log-driver": "json-file",
   "log-level": "",
   "log-opts": {
+    "cache-disabled": "false",
+    "cache-max-file": "5",
+    "cache-max-size": "20m",
+    "cache-compress": "true",
     "env": "os,customer",
     "labels": "somelabel",
     "max-file": "5",
@@ -1560,7 +1570,9 @@ previously configured cluster configurations.
 
 ### Run multiple daemons
 
-> **Note:** Running multiple daemons on a single host is considered as "experimental". The user should be aware of
+> **Note:**
+>
+> Running multiple daemons on a single host is considered as "experimental". The user should be aware of
 > unsolved problems. This solution may not work properly in some cases. Solutions are currently under development
 > and will be delivered in the near future.
 
@@ -1612,7 +1624,7 @@ The `--tls*` options enable use of specific certificates for individual daemons.
 
 Example script for a separate “bootstrap” instance of the Docker daemon without network:
 
-```bash
+```console
 $ sudo dockerd \
         -H unix:///var/run/docker-bootstrap.sock \
         -p /var/run/docker-bootstrap.pid \

docs/reference/commandline/pull.md

@@ -160,7 +160,7 @@ Digest can also be used in the `FROM` of a Dockerfile, for example:
 
 ```dockerfile
 FROM ubuntu@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2
-LABEL maintainer="some maintainer <maintainer@example.com>"
+LABEL org.opencontainers.image.authors="some maintainer <maintainer@example.com>"
 ```
 
 > **Note**

docs/reference/commandline/rm.md

@@ -58,13 +58,33 @@ The main process inside the container referenced under the link `redis` will rec
 
 ### Remove all stopped containers
 
-```bash
-$ docker rm $(docker ps -a -q)
+Use the [`docker container prune`](container_prune.md) command to remove all
+stopped containers, or refer to the [`docker system prune`](system_prune.md)
+command to remove unused containers in addition to other Docker resources, such
+as (unused) images and networks.
+
+Alternatively, you can use the `docker ps` with the `-q` / `--quiet` option to
+generate a list of container IDs to remove, and use that list as argument for
+the `docker rm` command.
+
+Combining commands can be more flexible, but is less portable as it depends
+on features provided by the shell, and the exact syntax may differ depending on
+what shell is used. To use this approach on Windows, consider using PowerShell
+or Bash.
+
+The example below uses `docker ps -q` to print the IDs of all containers that
+have exited (`--filter status=exited`), and removes those containers with
+the `docker rm` command:
+
+```console
+$ docker rm $(docker ps --filter status=exited -q)
 ```
 
-This command deletes all stopped containers. The command
-`docker ps -a -q` above returns all existing container IDs and passes them to
-the `rm` command which deletes them. Running containers are not deleted.
+Or, using the `xargs` Linux utility;
+
+```console
+$ docker ps --filter status=exited -q | xargs docker rm
+```
 
 ### Remove a container and its volumes
 

docs/reference/commandline/service_ls.md

@@ -43,7 +43,7 @@ ID            NAME      MODE            REPLICAS             IMAGE
 c8wgl7q4ndfd  frontend  replicated      5/5                  nginx:alpine
 dmu1ept4cxcf  redis     replicated      3/3                  redis:3.0.6
 iwe3278osahj  mongo     global          7/7                  mongo:3.3
-hh08h9uu8uwr  job       replicated-job  1/1 (3/5 completed)  nginx:latest        
+hh08h9uu8uwr  job       replicated-job  1/1 (3/5 completed)  nginx:latest
 ```
 
 The `REPLICAS` column shows both the *actual* and *desired* number of tasks for

docs/reference/commandline/stop.md

@@ -19,7 +19,9 @@ Options:
 ## Description
 
 The main process inside the container will receive `SIGTERM`, and after a grace
-period, `SIGKILL`.
+period, `SIGKILL`. The first signal can be changed with the `STOPSIGNAL`
+instruction in the container's Dockerfile, or the `--stop-signal` option to
+`docker run`.
 
 ## Examples
 

docs/reference/run.md

@@ -587,7 +587,7 @@ $ docker inspect -f "{{ .State.StartedAt }}" my-container
 
 Combining `--restart` (restart policy) with the `--rm` (clean up) flag results
 in an error. On container restart, attached clients are disconnected. See the
-examples on using the [`--rm` (clean up)](#clean-up-rm) flag later in this page.
+examples on using the [`--rm` (clean up)](#clean-up---rm) flag later in this page.
 
 ### Examples
 

experimental/README.md

@@ -39,12 +39,5 @@ Checkpoint and restore support for Containers.
 Metrics (Prometheus) output for basic container, image, and daemon operations.
 
  * [External graphdriver plugins](../docs/extend/plugins_graphdriver.md)
- * [Ipvlan Network Drivers](vlan-networks.md)
  * [Checkpoint & Restore](checkpoint-restore.md)
  * [Docker build with --squash argument](../docs/reference/commandline/build.md#squash-an-images-layers---squash-experimental)
-
-## How to comment on an experimental feature
-
-Each feature's documentation includes a list of proposal pull requests or PRs associated with the feature. If you want to comment on or suggest a change to a feature, please add it to the existing feature PR.
-
-Issues or problems with a feature? Inquire for help on the `#docker` IRC channel or on the [Docker Google group](https://groups.google.com/forum/#!forum/docker-user).

experimental/vlan-networks.md

@@ -1,632 +0,0 @@
-# Ipvlan Network Driver
-
-### Getting Started
-
-The Ipvlan driver is currently in experimental mode in order to incubate Docker
-users use cases and vet the implementation to ensure a hardened, production ready
-driver in a future release. Libnetwork now gives users total control over both
-IPv4 and IPv6 addressing. The VLAN driver builds on top of that in giving
-operators complete control of layer 2 VLAN tagging and even Ipvlan L3 routing
-for users interested in underlay network integration. For overlay deployments
-that abstract away physical constraints see the
-[multi-host overlay](https://docs.docker.com/network/network-tutorial-overlay/)
-driver.
-
-Ipvlan is a new twist on the tried and true network virtualization technique.
-The Linux implementations are extremely lightweight because rather than using
-the traditional Linux bridge for isolation, they are simply associated to a Linux
-Ethernet interface or sub-interface to enforce separation between networks and
-connectivity to the physical network.
-
-Ipvlan offers a number of unique features and plenty of room for further
-innovations with the various modes. Two high level advantages of these approaches
-are, the positive performance implications of bypassing the Linux bridge and the
-simplicity of having fewer moving parts. Removing the bridge that traditionally
-resides in between the Docker host NIC and container interface leaves a simple
-setup consisting of container interfaces, attached directly to the Docker host
-interface. This result is easy access for external facing services as there is
-no need for port mappings in these scenarios.
-
-### Pre-Requisites
-
-- The examples on this page are all single host and require using Docker
-  experimental features to be enabled.
-- All of the examples can be performed on a single host running Docker. Any
-  example using a sub-interface like `eth0.10` can be replaced with `eth0` or
-  any other valid parent interface on the Docker host. Sub-interfaces with a `.`
-  are created on the fly. `-o parent` interfaces can also be left out of the
-  `docker network create` all together and the driver will create a `dummy`
-  interface that will enable local host connectivity to perform the examples.
-- Kernel requirements:
-    - To check your current kernel version, use `uname -r`
-    - Ipvlan Linux kernel v4.2+ (support for earlier kernels exists but is buggy)
-
-### Ipvlan L2 Mode Example Usage
-
-An example of the ipvlan `L2` mode topology is shown in the following image.
-The driver is specified with `-d driver_name` option. In this case `-d ipvlan`.
-
-![Simple Ipvlan L2 Mode Example](images/ipvlan_l2_simple.png)
-
-The parent interface in the next example `-o parent=eth0` is configured as follows:
-
-```bash
-$ ip addr show eth0
-3: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
-    inet 192.168.1.250/24 brd 192.168.1.255 scope global eth0
-```
-
-Use the network from the host's interface as the `--subnet` in the
-`docker network create`. The container will be attached to the same network as
-the host interface as set via the `-o parent=` option.
-
-Create the ipvlan network and run a container attaching to it:
-
-```bash
-# Ipvlan  (-o ipvlan_mode= Defaults to L2 mode if not specified)
-$ docker network  create -d ipvlan \
-    --subnet=192.168.1.0/24 \ 
-    --gateway=192.168.1.1 \
-    -o ipvlan_mode=l2 \
-    -o parent=eth0 db_net
-
-# Start a container on the db_net network
-$ docker  run --net=db_net -it --rm alpine /bin/sh
-
-# NOTE: the containers can NOT ping the underlying host interfaces as
-# they are intentionally filtered by Linux for additional isolation.
-```
-
-The default mode for Ipvlan is `l2`. If `-o ipvlan_mode=` are left unspecified,
-the default mode will be used. Similarly, if the `--gateway` is left empty, the
-first usable address on the network will be set as the gateway. For example, if
-the subnet provided in the network create is `--subnet=192.168.1.0/24` then the
-gateway the container receives is `192.168.1.1`.
-
-To help understand how this mode interacts with other hosts, the following
-figure shows the same layer 2 segment between two Docker hosts that applies to
-and Ipvlan L2 mode.
-
-![Multiple Ipvlan Hosts](images/macvlan-bridge-ipvlan-l2.png)
-
-The following will create the exact same network as the network `db_net` created
-prior, with the driver defaults for `--gateway=192.168.1.1` and `-o ipvlan_mode=l2`.
-
-```bash
-# Ipvlan  (-o ipvlan_mode= Defaults to L2 mode if not specified)
-$ docker network  create -d ipvlan \
-    --subnet=192.168.1.0/24 \ 
-    -o parent=eth0 db_net_ipv
-
-# Start a container with an explicit name in daemon mode
-$ docker  run --net=db_net_ipv --name=ipv1 -itd alpine /bin/sh
-
-# Start a second container and ping using the container name
-# to see the docker included name resolution functionality
-$ docker  run --net=db_net_ipv --name=ipv2 -it --rm alpine /bin/sh
-$ ping -c 4 ipv1
-
-# NOTE: the containers can NOT ping the underlying host interfaces as
-# they are intentionally filtered by Linux for additional isolation.
-```
-
-The drivers also support the `--internal` flag that will completely isolate
-containers on a network from any communications external to that network. Since
-network isolation is tightly coupled to the network's parent interface the result
-of leaving the `-o parent=` option off of a `docker network create` is the exact
-same as the `--internal` option. If the parent interface is not specified or the
-`--internal` flag is used, a netlink type `dummy` parent interface is created
-for the user and used as the parent interface effectively isolating the network
-completely.
-
-The following two `docker network create` examples result in identical networks
-that you can attach container to:
-
-```bash
-# Empty '-o parent=' creates an isolated network
-$ docker network  create -d ipvlan \
-    --subnet=192.168.10.0/24 isolated1
-
-# Explicit '--internal' flag is the same:
-$ docker network  create -d ipvlan \
-    --subnet=192.168.11.0/24 --internal isolated2
-
-# Even the '--subnet=' can be left empty and the default 
-# IPAM subnet of 172.18.0.0/16 will be assigned
-$ docker network  create -d ipvlan isolated3
-
-$ docker run --net=isolated1 --name=cid1 -it --rm alpine /bin/sh
-$ docker run --net=isolated2 --name=cid2 -it --rm alpine /bin/sh
-$ docker run --net=isolated3 --name=cid3 -it --rm alpine /bin/sh
-
-# To attach to any use `docker exec` and start a shell
-$ docker exec -it cid1 /bin/sh
-$ docker exec -it cid2 /bin/sh
-$ docker exec -it cid3 /bin/sh
-```
-
-### Ipvlan 802.1q Trunk L2 Mode Example Usage
-
-Architecturally, Ipvlan L2 mode trunking is the same as Macvlan with regard to
-gateways and L2 path isolation. There are nuances that can be advantageous for
-CAM table pressure in ToR switches, one MAC per port and MAC exhaustion on a
-host's parent NIC to name a few. The 802.1q trunk scenario looks the same. Both
-modes adhere to tagging standards and have seamless integration with the physical
-network for underlay integration and hardware vendor plugin integrations.
-
-Hosts on the same VLAN are typically on the same subnet and almost always are
-grouped together based on their security policy. In most scenarios, a multi-tier
-application is tiered into different subnets because the security profile of each
-process requires some form of isolation. For example, hosting your credit card
-processing on the same virtual network as the frontend webserver would be a
-regulatory compliance issue, along with circumventing the long standing best
-practice of layered defense in depth architectures. VLANs or the equivocal VNI
-(Virtual Network Identifier) when using the Overlay driver, are the first step
-in isolating tenant traffic.
-
-![Docker VLANs in Depth](images/vlans-deeper-look.png)
-
-The Linux sub-interface tagged with a vlan can either already exist or will be
-created when you call a `docker network create`. `docker network rm` will delete
-the sub-interface. Parent interfaces such as `eth0` are not deleted, only
-sub-interfaces with a netlink parent index > 0.
-
-For the driver to add/delete the vlan sub-interfaces the format needs to be
-`interface_name.vlan_tag`. Other sub-interface naming can be used as the
-specified parent, but the link will not be deleted automatically when
-`docker network rm` is invoked. 
-
-The option to use either existing parent vlan sub-interfaces or let Docker manage
-them enables the user to either completely manage the Linux interfaces and
-networking or let Docker create and delete the Vlan parent sub-interfaces
-(netlink `ip link`) with no effort from the user.
-
-For example: use `eth0.10` to denote a sub-interface of `eth0` tagged with the
-vlan id of `10`. The equivalent `ip link` command would be
-`ip link add link eth0 name eth0.10 type vlan id 10`.
-
-The example creates the vlan tagged networks and then start two containers to
-test connectivity between containers. Different Vlans cannot ping one another
-without a router routing between the two networks. The default namespace is not
-reachable per ipvlan design in order to isolate container namespaces from the
-underlying host.
-
-**Vlan ID 20**
-
-In the first network tagged and isolated by the Docker host, `eth0.20` is the
-parent interface tagged with vlan id `20` specified with `-o parent=eth0.20`.
-Other naming formats can be used, but the links need to be added and deleted
-manually using `ip link` or Linux configuration files. As long as the `-o parent`
-exists anything can be used if compliant with Linux netlink.
-
-```bash
-# now add networks and hosts as you would normally by attaching to the master (sub)interface that is tagged
-$ docker network  create  -d ipvlan \
-    --subnet=192.168.20.0/24 \
-    --gateway=192.168.20.1 \
-    -o parent=eth0.20 ipvlan20
-
-# in two separate terminals, start a Docker container and the containers can now ping one another.
-$ docker run --net=ipvlan20 -it --name ivlan_test1 --rm alpine /bin/sh
-$ docker run --net=ipvlan20 -it --name ivlan_test2 --rm alpine /bin/sh
-```
-
-**Vlan ID 30**
-
-In the second network, tagged and isolated by the Docker host, `eth0.30` is the
-parent interface tagged with vlan id `30` specified with `-o parent=eth0.30`. The
-`ipvlan_mode=` defaults to l2 mode `ipvlan_mode=l2`. It can also be explicitly
-set with the same result as shown in the next example.
-
-```bash
-# now add networks and hosts as you would normally by attaching to the master (sub)interface that is tagged.
-$ docker network  create  -d ipvlan \
-    --subnet=192.168.30.0/24 \
-    --gateway=192.168.30.1 \
-    -o parent=eth0.30 \
-    -o ipvlan_mode=l2 ipvlan30
-
-# in two separate terminals, start a Docker container and the containers can now ping one another.
-$ docker run --net=ipvlan30 -it --name ivlan_test3 --rm alpine /bin/sh
-$ docker run --net=ipvlan30 -it --name ivlan_test4 --rm alpine /bin/sh
-```
-
-The gateway is set inside of the container as the default gateway. That gateway
-would typically be an external router on the network.
-
-```bash
-$$ ip route
-  default via 192.168.30.1 dev eth0
-  192.168.30.0/24 dev eth0  src 192.168.30.2
-```
-
-Example: Multi-Subnet Ipvlan L2 Mode starting two containers on the same subnet
-and pinging one another. In order for the `192.168.114.0/24` to reach
-`192.168.116.0/24` it requires an external router in L2 mode. L3 mode can route
-between subnets that share a common `-o parent=`. 
-
-Secondary addresses on network routers are common as an address space becomes
-exhausted to add another secondary to an L3 vlan interface or commonly referred
-to as a "switched virtual interface" (SVI).
-
-```bash
-$ docker network  create  -d ipvlan \
-    --subnet=192.168.114.0/24 --subnet=192.168.116.0/24 \
-    --gateway=192.168.114.254  --gateway=192.168.116.254 \
-     -o parent=eth0.114 \
-     -o ipvlan_mode=l2 ipvlan114
-     
-$ docker run --net=ipvlan114 --ip=192.168.114.10 -it --rm alpine /bin/sh
-$ docker run --net=ipvlan114 --ip=192.168.114.11 -it --rm alpine /bin/sh
-```
-
-A key takeaway is, operators have the ability to map their physical network into
-their virtual network for integrating containers into their environment with no
-operational overhauls required. NetOps simply drops an 802.1q trunk into the
-Docker host. That virtual link would be the `-o parent=` passed in the network
-creation. For untagged (non-VLAN) links, it is as simple as `-o parent=eth0` or
-for 802.1q trunks with VLAN IDs each network gets mapped to the corresponding
-VLAN/Subnet from the network.
-
-An example being, NetOps provides VLAN ID and the associated subnets for VLANs
-being passed on the Ethernet link to the Docker host server. Those values are
-simply plugged into the `docker network create` commands when provisioning the
-Docker networks. These are persistent configurations that are applied every time
-the Docker engine starts which alleviates having to manage often complex
-configuration files. The network interfaces can also be managed manually by
-being pre-created and docker networking will never modify them, simply use them
-as parent interfaces. Example mappings from NetOps to Docker network commands
-are as follows:
-
-- VLAN: 10, Subnet: 172.16.80.0/24, Gateway: 172.16.80.1
-    - `--subnet=172.16.80.0/24 --gateway=172.16.80.1 -o parent=eth0.10` 
-- VLAN: 20, IP subnet: 172.16.50.0/22, Gateway: 172.16.50.1
-    - `--subnet=172.16.50.0/22 --gateway=172.16.50.1 -o parent=eth0.20 ` 
-- VLAN: 30, Subnet: 10.1.100.0/16, Gateway: 10.1.100.1
-    - `--subnet=10.1.100.0/16 --gateway=10.1.100.1 -o parent=eth0.30` 
-
-### IPVlan L3 Mode Example
-
-IPVlan will require routes to be distributed to each endpoint. The driver only
-builds the Ipvlan L3 mode port and attaches the container to the interface. Route
-distribution throughout a cluster is beyond the initial implementation of this
-single host scoped driver. In L3 mode, the Docker host is very similar to a
-router starting new networks in the container. They are on networks that the
-upstream network will not know about without route distribution. For those
-curious how Ipvlan L3 will fit into container networking see the following
-examples.
-
-![Docker Ipvlan L2 Mode](images/ipvlan-l3.png)
-
-Ipvlan L3 mode drops all broadcast and multicast traffic. This reason alone
-makes Ipvlan L3 mode a prime candidate for those looking for massive scale and
-predictable network integrations. It is predictable and in turn will lead to
-greater uptimes because there is no bridging involved. Bridging loops have been
-responsible for high profile outages that can be hard to pinpoint depending on
-the size of the failure domain. This is due to the cascading nature of BPDUs
-(Bridge Port Data Units) that are flooded throughout a broadcast domain (VLAN)
-to find and block topology loops. Eliminating bridging domains, or at the least,
-keeping them isolated to a pair of ToRs (top of rack switches) will reduce hard
-to troubleshoot bridging instabilities. Ipvlan L2 modes is well suited for
-isolated VLANs only trunked into a pair of ToRs that can provide a loop-free
-non-blocking fabric. The next step further is to route at the edge via Ipvlan L3
-mode that reduces a failure domain to a local host only. 
-
-- L3 mode needs to be on a separate subnet as the default namespace since it
-  requires a netlink route in the default namespace pointing to the Ipvlan parent
-  interface.
-- The parent interface used in this example is `eth0` and it is on the subnet
- `192.168.1.0/24`. Notice the `docker network` is **not** on the same subnet
-  as `eth0`.
-- Unlike ipvlan l2 modes, different subnets/networks can ping one another as
-  long as they share the same parent interface `-o parent=`.
-
-```bash
-$$ ip a show eth0
-3: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
-    link/ether 00:50:56:39:45:2e brd ff:ff:ff:ff:ff:ff
-    inet 192.168.1.250/24 brd 192.168.1.255 scope global eth0
-```
-
-- A traditional gateway doesn't mean much to an L3 mode Ipvlan interface since
-  there is no broadcast traffic allowed. Because of that, the container default
-  gateway simply points to the containers `eth0` device. See below for CLI output
-  of `ip route` or `ip -6 route` from inside an L3 container for details.
-
-The mode ` -o ipvlan_mode=l3` must be explicitly specified since the default
-ipvlan mode is `l2`.
-
-The following example does not specify a parent interface. The network drivers
-will create a dummy type link for the user rather than rejecting the network
-creation and isolating containers from only communicating with one another.
-
-```bash
-# Create the Ipvlan L3 network
-$ docker network  create  -d ipvlan \
-    --subnet=192.168.214.0/24 \
-    --subnet=10.1.214.0/24 \
-     -o ipvlan_mode=l3 ipnet210
-
-# Test 192.168.214.0/24 connectivity
-$ docker run --net=ipnet210 --ip=192.168.214.10 -itd alpine /bin/sh
-$ docker run --net=ipnet210 --ip=10.1.214.10 -itd alpine /bin/sh
-
-# Test L3 connectivity from 10.1.214.0/24 to 192.168.212.0/24
-$ docker run --net=ipnet210 --ip=192.168.214.9 -it --rm alpine ping -c 2 10.1.214.10
-
-# Test L3 connectivity from 192.168.212.0/24 to 10.1.214.0/24
-$ docker run --net=ipnet210 --ip=10.1.214.9 -it --rm alpine ping -c 2 192.168.214.10
-
-```
-
-> **Note**
->
-> Notice that there is no `--gateway=` option in the network create. The field
-> is ignored if one is specified `l3` mode. Take a look at the container routing
-> table from inside of the container:
->
-> ```bash
-> # Inside an L3 mode container
-> $$ ip route
->  default dev eth0
->   192.168.214.0/24 dev eth0  src 192.168.214.10
-> ```
-
-In order to ping the containers from a remote Docker host or the container be
-able to ping a remote host, the remote host or the physical network in between
-need to have a route pointing to the host IP address of the container's Docker
-host eth interface. More on this as we evolve the Ipvlan `L3` story.
-
-### Dual Stack IPv4 IPv6 Ipvlan L2 Mode
-
-- Not only does Libnetwork give you complete control over IPv4 addressing, but
-it also gives you total control over IPv6 addressing as well as feature parity
-between the two address families.
-
-- The next example will start with IPv6 only. Start two containers on the same
-VLAN `139` and ping one another. Since the IPv4 subnet is not specified, the
-default IPAM will provision a default IPv4 subnet. That subnet is isolated
-unless the upstream network is explicitly routing it on VLAN `139`.
-
-```bash
-# Create a v6 network
-$ docker network create -d ipvlan \
-    --subnet=2001:db8:abc2::/64 --gateway=2001:db8:abc2::22 \
-    -o parent=eth0.139 v6ipvlan139
-    
-# Start a container on the network
-$ docker run --net=v6ipvlan139 -it --rm alpine /bin/sh
-```
-
-View the container eth0 interface and v6 routing table:
-
-```bash
-# Inside the IPv6 container
-$$ ip a show eth0
-75: eth0@if55: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default
-    link/ether 00:50:56:2b:29:40 brd ff:ff:ff:ff:ff:ff
-    inet 172.18.0.2/16 scope global eth0
-       valid_lft forever preferred_lft forever
-    inet6 2001:db8:abc4::250:56ff:fe2b:2940/64 scope link
-       valid_lft forever preferred_lft forever
-    inet6 2001:db8:abc2::1/64 scope link nodad
-       valid_lft forever preferred_lft forever
-       
-$$ ip -6 route
-2001:db8:abc4::/64 dev eth0  proto kernel  metric 256
-2001:db8:abc2::/64 dev eth0  proto kernel  metric 256
-default via 2001:db8:abc2::22 dev eth0  metric 1024
-```
-
-Start a second container and ping the first container's v6 address. 
-
-```bash
-# Test L2 connectivity over IPv6
-$ docker run --net=v6ipvlan139 -it --rm alpine /bin/sh
-
-# Inside the second IPv6 container
-$$ ip a show eth0
-75: eth0@if55: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default
-    link/ether 00:50:56:2b:29:40 brd ff:ff:ff:ff:ff:ff
-    inet 172.18.0.3/16 scope global eth0
-       valid_lft forever preferred_lft forever
-    inet6 2001:db8:abc4::250:56ff:fe2b:2940/64 scope link tentative dadfailed
-       valid_lft forever preferred_lft forever
-    inet6 2001:db8:abc2::2/64 scope link nodad
-       valid_lft forever preferred_lft forever
-
-$$ ping6 2001:db8:abc2::1
-PING 2001:db8:abc2::1 (2001:db8:abc2::1): 56 data bytes
-64 bytes from 2001:db8:abc2::1%eth0: icmp_seq=0 ttl=64 time=0.044 ms
-64 bytes from 2001:db8:abc2::1%eth0: icmp_seq=1 ttl=64 time=0.058 ms
-
-2 packets transmitted, 2 packets received, 0% packet loss
-round-trip min/avg/max/stddev = 0.044/0.051/0.058/0.000 ms
-```
-
-The next example with setup a dual stack IPv4/IPv6 network with an example
-VLAN ID of `140`.
-
-Next create a network with two IPv4 subnets and one IPv6 subnets, all of which
-have explicit gateways:
-
-```bash
-$ docker network  create  -d ipvlan \
-    --subnet=192.168.140.0/24 --subnet=192.168.142.0/24 \
-    --gateway=192.168.140.1  --gateway=192.168.142.1 \
-    --subnet=2001:db8:abc9::/64 --gateway=2001:db8:abc9::22 \
-     -o parent=eth0.140 \
-     -o ipvlan_mode=l2 ipvlan140
-```
-
-Start a container and view eth0 and both v4 & v6 routing tables:
-
-```bash
-$ docker run --net=ipvlan140 --ip6=2001:db8:abc2::51 -it --rm alpine /bin/sh
-
-$ ip a show eth0
-78: eth0@if77: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default
-    link/ether 00:50:56:2b:29:40 brd ff:ff:ff:ff:ff:ff
-    inet 192.168.140.2/24 scope global eth0
-       valid_lft forever preferred_lft forever
-    inet6 2001:db8:abc4::250:56ff:fe2b:2940/64 scope link
-       valid_lft forever preferred_lft forever
-    inet6 2001:db8:abc9::1/64 scope link nodad
-       valid_lft forever preferred_lft forever
-
-$$ ip route
-default via 192.168.140.1 dev eth0
-192.168.140.0/24 dev eth0  proto kernel  scope link  src 192.168.140.2
-
-$$ ip -6 route
-2001:db8:abc4::/64 dev eth0  proto kernel  metric 256
-2001:db8:abc9::/64 dev eth0  proto kernel  metric 256
-default via 2001:db8:abc9::22 dev eth0  metric 1024
-```
-
-Start a second container with a specific `--ip4` address and ping the first host
-using IPv4 packets:
-
-```bash
-$ docker run --net=ipvlan140 --ip=192.168.140.10 -it --rm alpine /bin/sh
-```
-
-> **Note**
->
-> Different subnets on the same parent interface in Ipvlan `L2` mode cannot ping
-> one another. That requires a router to proxy-arp the requests with a secondary
-> subnet. However, Ipvlan `L3` will route the unicast traffic between disparate
-> subnets as long as they share the same `-o parent` parent link.
-
-### Dual Stack IPv4 IPv6 Ipvlan L3 Mode 
-
-**Example:** IpVlan L3 Mode Dual Stack IPv4/IPv6, Multi-Subnet w/ 802.1q Vlan Tag:118
-
-As in all of the examples, a tagged VLAN interface does not have to be used. The
-sub-interfaces can be swapped with `eth0`, `eth1`, `bond0` or any other valid
-interface on the host other then the `lo` loopback.
-
-The primary difference you will see is that L3 mode does not create a default
-route with a next-hop but rather sets a default route pointing to `dev eth` only
-since ARP/Broadcasts/Multicast are all filtered by Linux as per the design. Since
-the parent interface is essentially acting as a router, the parent interface IP
-and subnet needs to be different from the container networks. That is the opposite
-of bridge and L2 modes, which need to be on the same subnet (broadcast domain)
-in order to forward broadcast and multicast packets.
-
-```bash
-# Create an IPv6+IPv4 Dual Stack Ipvlan L3 network 
-# Gateways for both v4 and v6 are set to a dev e.g. 'default dev eth0'
-$ docker network  create  -d ipvlan \
-    --subnet=192.168.110.0/24 \
-    --subnet=192.168.112.0/24 \
-    --subnet=2001:db8:abc6::/64 \
-     -o parent=eth0 \
-     -o ipvlan_mode=l3 ipnet110
-
-
-# Start a few of containers on the network (ipnet110) 
-# in separate terminals and check connectivity
-$ docker run --net=ipnet110 -it --rm alpine /bin/sh
-# Start a second container specifying the v6 address
-$ docker run --net=ipnet110 --ip6=2001:db8:abc6::10 -it --rm alpine /bin/sh
-# Start a third specifying the IPv4 address
-$ docker run --net=ipnet110 --ip=192.168.112.30 -it --rm alpine /bin/sh
-# Start a 4th specifying both the IPv4 and IPv6 addresses
-$ docker run --net=ipnet110 --ip6=2001:db8:abc6::50 --ip=192.168.112.50 -it --rm alpine /bin/sh
-```
-
-Interface and routing table outputs are as follows:
-
-```bash
-$$ ip a show eth0
-63: eth0@if59: <BROADCAST,MULTICAST,NOARP,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN group default
-    link/ether 00:50:56:2b:29:40 brd ff:ff:ff:ff:ff:ff
-    inet 192.168.112.2/24 scope global eth0
-       valid_lft forever preferred_lft forever
-    inet6 2001:db8:abc4::250:56ff:fe2b:2940/64 scope link
-       valid_lft forever preferred_lft forever
-    inet6 2001:db8:abc6::10/64 scope link nodad
-       valid_lft forever preferred_lft forever
-     
-# Note the default route is simply the eth device because ARPs are filtered.
-$$ ip route
-  default dev eth0  scope link
-  192.168.112.0/24 dev eth0  proto kernel  scope link  src 192.168.112.2
-
-$$ ip -6 route
-2001:db8:abc4::/64 dev eth0  proto kernel  metric 256
-2001:db8:abc6::/64 dev eth0  proto kernel  metric 256
-default dev eth0  metric 1024
-```
-
-> *Note*
->
-> There may be a bug when specifying `--ip6=` addresses when you delete a
-> container with a specified v6 address and then start a new container with the
-> same v6 address it throws the following like the address isn't properly being
-> released to the v6 pool. It will fail to unmount the container and be left dead.
-
-```console
-docker: Error response from daemon: Address already in use.
-```
-
-### Manually Creating 802.1q Links
-
-**Vlan ID 40**
-
-If a user does not want the driver to create the vlan sub-interface it simply
-needs to exist prior to the `docker network create`. If you have sub-interface
-naming that is not `interface.vlan_id` it is honored in the `-o parent=` option
-again as long as the interface exists and is up.
-
-Links, when manually created, can be named anything as long as they exist when
-the network is created. Manually created links do not get deleted regardless of
-the name when the network is deleted with `docker network rm`.
-
-```bash
-# create a new sub-interface tied to dot1q vlan 40
-$ ip link add link eth0 name eth0.40 type vlan id 40
-
-# enable the new sub-interface
-$ ip link set eth0.40 up
-
-# now add networks and hosts as you would normally by attaching to the master (sub)interface that is tagged
-$ docker network  create  -d ipvlan \
-   --subnet=192.168.40.0/24 \
-   --gateway=192.168.40.1 \
-   -o parent=eth0.40 ipvlan40
-
-# in two separate terminals, start a Docker container and the containers can now ping one another.
-$ docker run --net=ipvlan40 -it --name ivlan_test5 --rm alpine /bin/sh
-$ docker run --net=ipvlan40 -it --name ivlan_test6 --rm alpine /bin/sh
-```
-
-**Example:** Vlan sub-interface manually created with any name:
-
-```bash
-# create a new sub interface tied to dot1q vlan 40
-$ ip link add link eth0 name foo type vlan id 40
-
-# enable the new sub-interface
-$ ip link set foo up
-
-# now add networks and hosts as you would normally by attaching to the master (sub)interface that is tagged
-$ docker network  create  -d ipvlan \
-    --subnet=192.168.40.0/24 --gateway=192.168.40.1 \
-    -o parent=foo ipvlan40
-
-# in two separate terminals, start a Docker container and the containers can now ping one another.
-$ docker run --net=ipvlan40 -it --name ivlan_test5 --rm alpine /bin/sh
-$ docker run --net=ipvlan40 -it --name ivlan_test6 --rm alpine /bin/sh
-```
-
-Manually created links can be cleaned up with:
-
-```bash
-$ ip link del foo
-```
-
-As with all of the Libnetwork drivers, they can be mixed and matched, even as
-far as running 3rd party ecosystem drivers in parallel for maximum flexibility
-to the Docker user.

man/Dockerfile.5.md

@@ -438,8 +438,12 @@ A Dockerfile is similar to a Makefile.
   * `NO_PROXY`
   * `no_proxy`
 
-  To use these, simply pass them on the command line using the `--build-arg
-  <varname>=<value>` flag.
+  To use these, pass them on the command line using `--build-arg` flag, for
+  example:
+
+  ```
+  $ docker build --build-arg HTTPS_PROXY=https://my-proxy.example.com .
+  ```
 
 **ONBUILD**
   -- `ONBUILD [INSTRUCTION]`

man/src/image/pull.md

@@ -111,7 +111,7 @@ pull the above image by digest, run the following command:
 Digest can also be used in the `FROM` of a Dockerfile, for example:
 
     FROM ubuntu@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2
-    LABEL maintainer="some maintainer <maintainer@example.com>"
+    LABEL org.opencontainers.image.authors="some maintainer <maintainer@example.com>"
 
 > **Note**: Using this feature "pins" an image to a specific version in time.
 > Docker will therefore not pull updated versions of an image, which may include 

scripts/build/.variables

@@ -1,8 +1,10 @@
-#!/usr/bin/env bash
+#!/usr/bin/env sh
 set -eu
 
+TARGET=${TARGET:-"build"}
+
 PLATFORM=${PLATFORM:-}
-VERSION=${VERSION:-"unknown-version"}
+VERSION=${VERSION:-$(git describe --match 'v[0-9]*' --dirty='.m' --always --tags | sed 's/^v//' 2>/dev/null || echo "unknown-version" )}
 GITCOMMIT=${GITCOMMIT:-$(git rev-parse --short HEAD 2> /dev/null || true)}
 BUILDTIME=${BUILDTIME:-$(date -u +"%Y-%m-%dT%H:%M:%SZ")}
 
@@ -20,15 +22,15 @@ export LDFLAGS="\
     ${LDFLAGS:-} \
 "
 
-GOOS="${GOOS:-$(go env GOHOSTOS)}"
-GOARCH="${GOARCH:-$(go env GOHOSTARCH)}"
+GOOS="$(go env GOOS)"
+GOARCH="$(go env GOARCH)"
 if [ "${GOARCH}" = "arm" ]; then
-	GOARM="${GOARM:-$(go env GOHOSTARM)}"
+	GOARM="$(go env GOARM)"
 fi
 
-TARGET="build/docker-$GOOS-$GOARCH"
+TARGET="$TARGET/docker-${GOOS}-${GOARCH}"
 if [ "${GOARCH}" = "arm" ] && [ -n "${GOARM}" ]; then
-	TARGET="${TARGET}-v${GOARM}"
+    TARGET="${TARGET}-v${GOARM}"
 fi
 
 if [ "${GOOS}" = "windows" ]; then

scripts/build/binary

@@ -1,14 +1,80 @@
-#!/usr/bin/env bash
+#!/usr/bin/env sh
 #
 # Build a static binary for the host OS/ARCH
 #
 
-set -eu -o pipefail
+set -eu
 
-source ./scripts/build/.variables
+: "${CGO_ENABLED=}"
+: "${GO_LINKMODE=static}"
+: "${GO_BUILDMODE=}"
+: "${GO_BUILDTAGS=}"
+: "${GO_STRIP=}"
 
-echo "Building statically linked $TARGET"
-export CGO_ENABLED=0
-go build -o "${TARGET}" --ldflags "${LDFLAGS}" "${SOURCE}"
+. ./scripts/build/.variables
 
-ln -sf "$(basename "${TARGET}")" build/docker
+if [ -z "$CGO_ENABLED" ]; then
+    case "$(go env GOOS)" in
+        linux)
+            case "$(go env GOARCH)" in
+                amd64|arm64|arm|s390x)
+                    CGO_ENABLED=1
+                ;;
+                *)
+                    CGO_ENABLED=0
+                ;;
+            esac
+        ;;
+        darwin|windows)
+            CGO_ENABLED=1
+        ;;
+        *)
+            CGO_ENABLED=0
+        ;;
+    esac
+fi
+export CGO_ENABLED
+if [ "$CGO_ENABLED" = "1" ] && [ "$(go env GOOS)" != "windows" ]; then
+    case "$(go env GOARCH)" in
+        mips*|ppc64)
+            # pie build mode is not supported on mips architectures
+            ;;
+        *)
+            GO_BUILDMODE="-buildmode=pie"
+            ;;
+    esac
+    GO_BUILDTAGS="$GO_BUILDTAGS pkcs11"
+fi
+
+if [ "$CGO_ENABLED" = "1" ] && [ "$GO_LINKMODE" = "static" ] && [ "$(go env GOOS)" = "linux" ]; then
+    LDFLAGS="$LDFLAGS -extldflags -static"
+fi
+
+if [ -n "$GO_STRIP" ]; then
+    LDFLAGS="$LDFLAGS -s -w"
+fi
+
+if [ "$(go env GOOS)" = "windows" ]; then
+    # Generate a Windows file version of the form major,minor,patch,build
+    VERSION_QUAD=$(printf "%s" "$VERSION" | sed -re 's/^([0-9.]*).*$/\1/' | tr . , | sed -re 's/^[0-9]+$/\0,0/' | sed -re 's/^[0-9]+,[0-9]+$/\0,0/' | sed -re 's/^[0-9]+,[0-9]+,[0-9]+$/\0,0/')
+
+    set --
+    [ -n "$VERSION" ]      && set -- "$@" -D "DOCKER_VERSION=\"$VERSION\""
+    [ -n "$VERSION_QUAD" ] && set -- "$@" -D "DOCKER_VERSION_QUAD=$VERSION_QUAD"
+    [ -n "$GITCOMMIT" ]    && set -- "$@" -D "DOCKER_COMMIT=\"$GITCOMMIT\""
+
+    windres=$($(go env CC) --print-prog-name=windres)
+
+    target="$(dirname "$0")/../../cli/winresources/rsrc_$(go env GOARCH).syso"
+    mkdir -p "$(dirname "${target}")"
+    "$windres" -i "$(dirname "$0")/../winresources/docker.rc" -o "$target" "$@"
+    echo "package winresources" > "$(dirname "${target}")/stub_windows.go"
+fi
+
+echo "Building $GO_LINKMODE $(basename "${TARGET}")"
+
+export GO111MODULE=auto
+
+go build -o "${TARGET}" -tags "${GO_BUILDTAGS}" --ldflags "${LDFLAGS}" ${GO_BUILDMODE} "${SOURCE}"
+
+ln -sf "$(basename "${TARGET}")" "$(dirname "${TARGET}")/docker"

scripts/build/cross

@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-#
-# Build a binary for all supported platforms
-#
-
-set -eu -o pipefail
-
-BUILDDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-export SHELL=bash
-
-jobs=(
-    "$BUILDDIR/windows" \
-    "$BUILDDIR/osx" \
-    "GOOS=linux GOARCH=amd64   $BUILDDIR/binary" \
-    "GOOS=linux GOARCH=arm     $BUILDDIR/binary" \
-    "GOOS=linux GOARCH=ppc64le $BUILDDIR/binary" \
-    "GOOS=linux GOARCH=s390x   $BUILDDIR/binary" \
-)
-
-# Outside of circleCI run all at once. On circleCI run two at a time because
-# each container has access to two cores.
-group=${CROSS_GROUP-"all"}
-
-if [ "$group" = "all" ]; then
-
-    echo "Building binaries for all platforms"
-    parallel ::: "${jobs[@]}"
-    exit 0
-
-fi
-
-declare -i start="$group*2"
-parallel ::: "${jobs[@]:$start:2}"

scripts/build/dynbinary

@@ -1,24 +0,0 @@
-#!/usr/bin/env bash
-#
-# Build a dynamically linked binary for the host OS/ARCH
-#
-
-set -eu -o pipefail
-
-source ./scripts/build/.variables
-
-echo "Building dynamically linked $TARGET"
-export CGO_ENABLED=1
-case "$(go env GOARCH)" in
-        mips*|ppc64)
-                # pie build mode is not supported on mips architectures
-                GO_BUILDMODE=""
-                ;;
-        *)
-                GO_BUILDMODE="-buildmode=pie"
-                ;;
-esac
-
-go build -o "${TARGET}" -tags pkcs11 --ldflags "${LDFLAGS}" ${GO_BUILDMODE} "${SOURCE}"
-
-ln -sf "$(basename "${TARGET}")" build/docker

scripts/build/osx

@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-#
-# Build an osx binary from linux
-#
-
-set -eu -o pipefail
-
-source ./scripts/build/.variables
-
-export CGO_ENABLED=1
-export GOOS=darwin
-export GOARCH=amd64
-export CC=o64-clang
-export CXX=o64-clang++
-export LDFLAGS="$LDFLAGS -linkmode external -s"
-export LDFLAGS_STATIC_DOCKER='-extld='${CC}
-
-# Override TARGET
-TARGET="build/docker-$GOOS-$GOARCH"
-
-echo "Building $TARGET"
-go build -o "${TARGET}" -tags pkcs11 --ldflags "${LDFLAGS}" "${SOURCE}"

scripts/build/windows

@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-#
-# Build a windows binary from linux
-#
-
-set -eu -o pipefail
-
-source ./scripts/build/.variables
-
-export CC=x86_64-w64-mingw32-gcc
-export CGO_ENABLED=1
-export GOOS=windows
-export GOARCH=amd64
-
-# Override TARGET
-TARGET="build/docker-$GOOS-$GOARCH.exe"
-
-echo "Generating windows resources"
-go generate ./cli/winresources
-
-echo "Building $TARGET"
-# TODO: -tags pkcs11
-go build -o "${TARGET}" --ldflags "${LDFLAGS}" "${SOURCE}"

scripts/gen/windows-resources

@@ -1,45 +0,0 @@
-#!/usr/bin/env bash
-#
-# Compile the Windows resources into the sources
-#
-
-set -eu -o pipefail
-
-SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-# shellcheck source=/go/src/github.com/docker/cli/scripts/build/.variables
-source "$SCRIPTDIR"/../build/.variables
-
-RESOURCES=$SCRIPTDIR/../winresources
-
-TEMPDIR=$(mktemp -d)
-trap 'rm -rf $TEMPDIR' EXIT
-
-if [ "$(go env GOHOSTOS)" = "windows" ]; then
-	WINDRES=windres
-else
-	# Cross compiling
-	WINDRES=x86_64-w64-mingw32-windres
-fi
-
-# Generate a Windows file version of the form major,minor,patch,build (with any part optional)
-VERSION_QUAD=$(printf "%s" "$VERSION" | sed -re 's/^([0-9.]*).*$/\1/' | tr . ,)
-
-# Pass version and commit information into the resource compiler
-defs=
-[ -n "$VERSION" ]      && defs+=( "-D DOCKER_VERSION=\"$VERSION\"")
-[ -n "$VERSION_QUAD" ] && defs+=( "-D DOCKER_VERSION_QUAD=$VERSION_QUAD")
-[ -n "$GITCOMMIT" ]    && defs+=( "-D DOCKER_COMMIT=\"$GITCOMMIT\"")
-
-makeres() {
-	# shellcheck disable=SC2086
-	"$WINDRES" \
-		-i "$RESOURCES/$1" \
-		-o "$3" \
-		-F "$2" \
-		--use-temp-file \
-		-I "$TEMPDIR" \
-		${defs[*]}
-}
-
-makeres docker.rc pe-x86-64 rsrc_amd64.syso
-makeres docker.rc pe-i386   rsrc_386.syso

vendor.conf

@@ -1,5 +1,4 @@
 cloud.google.com/go                                 ceeb313ad77b789a7fa5287b36a1d127b69b7093 # v0.44.3
-github.com/agl/ed25519                              5312a61534124124185d41f09206b9fef1d88403
 github.com/Azure/go-ansiterm                        d6e3b3328b783f23731bc4d058875b0371ff8109
 github.com/beorn7/perks                             37c8de3658fcb183f997c4e13e8337516ab753e6 # v1.0.1
 github.com/cespare/xxhash/v2                        d7df74196a9e781ede915320c11c378c1b2f3a1f # v2.1.1
@@ -68,7 +67,7 @@ github.com/shurcooL/sanitized_anchor_name           7bfe4c7ecddb3666a94b053b422c
 github.com/sirupsen/logrus                          6699a89a232f3db797f2e280639854bbc4b89725 # v1.7.0
 github.com/spf13/cobra                              86f8bfd7fef868a174e1b606783bd7f5c82ddf8f # v1.1.1
 github.com/spf13/pflag                              2e9d26c8c37aae03e3f9d4e90b7116f5accb7cab # v1.0.5
-github.com/theupdateframework/notary                d6e1431feb32348e0650bf7551ac5cffd01d857b # v0.6.1
+github.com/theupdateframework/notary                bf96a202a09a312ae005cd312fc06ff4d2c166ce # v0.7.0-21-gbf96a202
 github.com/tonistiigi/fsutil                        0834f99b7b85462efb69b4f571a4fa3ca7da5ac9
 github.com/tonistiigi/go-rosetta                    f79598599c5d34ea253b56a1d7c89bc6a96de7db
 github.com/tonistiigi/units                         6950e57a87eaf136bbe44ef2ec8e75b9e3569de2

vendor/github.com/agl/ed25519/LICENSE

@@ -1,27 +0,0 @@
-Copyright (c) 2012 The Go Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-   * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-   * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-   * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/agl/ed25519/ed25519.go

@@ -1,127 +0,0 @@
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package ed25519 implements the Ed25519 signature algorithm. See
-// http://ed25519.cr.yp.to/.
-package ed25519
-
-// This code is a port of the public domain, "ref10" implementation of ed25519
-// from SUPERCOP.
-
-import (
-	"crypto/sha512"
-	"crypto/subtle"
-	"io"
-
-	"github.com/agl/ed25519/edwards25519"
-)
-
-const (
-	PublicKeySize  = 32
-	PrivateKeySize = 64
-	SignatureSize  = 64
-)
-
-// GenerateKey generates a public/private key pair using randomness from rand.
-func GenerateKey(rand io.Reader) (publicKey *[PublicKeySize]byte, privateKey *[PrivateKeySize]byte, err error) {
-	privateKey = new([64]byte)
-	publicKey = new([32]byte)
-	_, err = io.ReadFull(rand, privateKey[:32])
-	if err != nil {
-		return nil, nil, err
-	}
-
-	h := sha512.New()
-	h.Write(privateKey[:32])
-	digest := h.Sum(nil)
-
-	digest[0] &= 248
-	digest[31] &= 127
-	digest[31] |= 64
-
-	var A edwards25519.ExtendedGroupElement
-	var hBytes [32]byte
-	copy(hBytes[:], digest)
-	edwards25519.GeScalarMultBase(&A, &hBytes)
-	A.ToBytes(publicKey)
-
-	copy(privateKey[32:], publicKey[:])
-	return
-}
-
-// Sign signs the message with privateKey and returns a signature.
-func Sign(privateKey *[PrivateKeySize]byte, message []byte) *[SignatureSize]byte {
-	h := sha512.New()
-	h.Write(privateKey[:32])
-
-	var digest1, messageDigest, hramDigest [64]byte
-	var expandedSecretKey [32]byte
-	h.Sum(digest1[:0])
-	copy(expandedSecretKey[:], digest1[:])
-	expandedSecretKey[0] &= 248
-	expandedSecretKey[31] &= 63
-	expandedSecretKey[31] |= 64
-
-	h.Reset()
-	h.Write(digest1[32:])
-	h.Write(message)
-	h.Sum(messageDigest[:0])
-
-	var messageDigestReduced [32]byte
-	edwards25519.ScReduce(&messageDigestReduced, &messageDigest)
-	var R edwards25519.ExtendedGroupElement
-	edwards25519.GeScalarMultBase(&R, &messageDigestReduced)
-
-	var encodedR [32]byte
-	R.ToBytes(&encodedR)
-
-	h.Reset()
-	h.Write(encodedR[:])
-	h.Write(privateKey[32:])
-	h.Write(message)
-	h.Sum(hramDigest[:0])
-	var hramDigestReduced [32]byte
-	edwards25519.ScReduce(&hramDigestReduced, &hramDigest)
-
-	var s [32]byte
-	edwards25519.ScMulAdd(&s, &hramDigestReduced, &expandedSecretKey, &messageDigestReduced)
-
-	signature := new([64]byte)
-	copy(signature[:], encodedR[:])
-	copy(signature[32:], s[:])
-	return signature
-}
-
-// Verify returns true iff sig is a valid signature of message by publicKey.
-func Verify(publicKey *[PublicKeySize]byte, message []byte, sig *[SignatureSize]byte) bool {
-	if sig[63]&224 != 0 {
-		return false
-	}
-
-	var A edwards25519.ExtendedGroupElement
-	if !A.FromBytes(publicKey) {
-		return false
-	}
-	edwards25519.FeNeg(&A.X, &A.X)
-	edwards25519.FeNeg(&A.T, &A.T)
-
-	h := sha512.New()
-	h.Write(sig[:32])
-	h.Write(publicKey[:])
-	h.Write(message)
-	var digest [64]byte
-	h.Sum(digest[:0])
-
-	var hReduced [32]byte
-	edwards25519.ScReduce(&hReduced, &digest)
-
-	var R edwards25519.ProjectiveGroupElement
-	var b [32]byte
-	copy(b[:], sig[32:])
-	edwards25519.GeDoubleScalarMultVartime(&R, &hReduced, &A, &b)
-
-	var checkR [32]byte
-	R.ToBytes(&checkR)
-	return subtle.ConstantTimeCompare(sig[:32], checkR[:]) == 1
-}

vendor/github.com/theupdateframework/notary/README.md

@@ -21,7 +21,7 @@ for more information.
 
 Notary aims to make the internet more secure by making it easy for people to
 publish and verify content. We often rely on TLS to secure our communications
-with a web server which is inherently flawed, as any compromise of the server
+with a web server, which is inherently flawed, as any compromise of the server
 enables malicious content to be substituted for the legitimate content.
 
 With Notary, publishers can sign their content offline using keys kept highly
@@ -46,11 +46,16 @@ Notary is based on [The Update Framework](https://www.theupdateframework.com/),
 
 ## Security
 
+Any security vulnerabilities can be reported to security@docker.com.
+
 See Notary's [service architecture docs](docs/service_architecture.md#threat-model) for more information about our threat model, which details the varying survivability and severities for key compromise as well as mitigations.
 
-Notary's last security audit was on July 31, 2015 by NCC ([results](docs/resources/ncc_docker_notary_audit_2015_07_31.pdf)).
+### Security Audits
 
-Any security vulnerabilities can be reported to security@docker.com.
+Notary has had two public security audits:
+
+* [August 7, 2018 by Cure53](docs/resources/cure53_tuf_notary_audit_2018_08_07.pdf) covering TUF and Notary
+* [July 31, 2015 by NCC](docs/resources/ncc_docker_notary_audit_2015_07_31.pdf) covering Notary
 
 # Getting started with the Notary CLI
 
@@ -65,7 +70,7 @@ For more advanced usage, see the
 
 To use the CLI against a local Notary server rather than against Docker Hub:
 
-1. Ensure that you have [docker and docker-compose](http://docs.docker.com/compose/install/) installed.
+1. Ensure that you have [docker and docker-compose](https://docs.docker.com/compose/install/) installed.
 1. `git clone https://github.com/theupdateframework/notary.git` and from the cloned repository path,
     start up a local Notary server and signer and copy the config file and testing certs to your
     local Notary config directory:
@@ -88,6 +93,20 @@ URL is specified already in the configuration, file you copied.
 You can also leave off the `-d ~/.docker/trust` argument if you do not care
 to use `notary` with Docker images.
 
+## Upgrading dependencies
+
+To prevent mistakes in vendoring the go modules a buildscript has been added to properly vendor the modules using the correct version of Go to mitigate differences in CI and development environment.
+
+Following procedure should be executed to upgrade a dependency. Preferably keep dependency upgrades in a separate commit from your code changes.
+
+```bash
+go get -u github.com/spf13/viper
+buildscripts/circle-validate-vendor.sh
+git add .
+git commit -m "Upgraded github.com/spf13/viper"
+```
+
+The `buildscripts/circle-validate-vendor.sh` runs `go mod tidy` and `go mod vendor` using the given version of Go to prevent differences if you are for example running on a different version of Go.
 
 ## Building Notary
 
@@ -97,25 +116,20 @@ branch and contains features for the next release.
 
 Prerequisites:
 
-- Go >= 1.7.1
-    - Fedora: `dnf install golang`
-- libtool development headers installed
-    - Ubuntu: `apt-get install libltdl-dev`
-    - CentOS/RedHat: `yum install libtool-ltdl-devel`
-    - Fedora: `dnf install libtool-ltdl-devel`
-    - Mac OS ([Homebrew](http://brew.sh/)): `brew install libtool`
+* Go >= 1.12
 
 Set [```GOPATH```](https://golang.org/doc/code.html#GOPATH). Then, run:
 
 ```bash
+$ export GO111MODULE=on
 $ go get github.com/theupdateframework/notary
-# build with pcks11 support by default to support yubikey
+# build with pkcs11 support by default to support yubikey
 $ go install -tags pkcs11 github.com/theupdateframework/notary/cmd/notary
 $ notary
 ```
 
 To build the server and signer, run `docker-compose build`.
 
-
 ## License
+
 [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Ftheupdateframework%2Fnotary.svg?type=large)](https://app.fossa.io/projects/git%2Bgithub.com%2Ftheupdateframework%2Fnotary?ref=badge_large)

vendor/github.com/theupdateframework/notary/client/changelist/file_changelist.go

@@ -35,7 +35,10 @@ func getFileNames(dirName string) ([]os.FileInfo, error) {
 	if err != nil {
 		return fileInfos, err
 	}
-	defer dir.Close()
+	defer func() {
+		_ = dir.Close()
+	}()
+
 	dirListing, err = dir.Readdir(0)
 	if err != nil {
 		return fileInfos, err
@@ -89,7 +92,7 @@ func (cl FileChangelist) Add(c Change) error {
 		return err
 	}
 	filename := fmt.Sprintf("%020d_%s.change", time.Now().UnixNano(), uuid.Generate())
-	return ioutil.WriteFile(filepath.Join(cl.dir, filename), cJSON, 0644)
+	return ioutil.WriteFile(filepath.Join(cl.dir, filename), cJSON, 0600)
 }
 
 // Remove deletes the changes found at the given indices
@@ -120,7 +123,10 @@ func (cl FileChangelist) Clear(archive string) error {
 	if err != nil {
 		return err
 	}
-	defer dir.Close()
+	defer func() {
+		_ = dir.Close()
+	}()
+
 	files, err := dir.Readdir(0)
 	if err != nil {
 		return err

vendor/github.com/theupdateframework/notary/client/changelist/interface.go

@@ -20,7 +20,7 @@ type Changelist interface {
 	// Remove deletes the changes corresponding with the indices given
 	Remove(idxs []int) error
 
-	// Close syncronizes any pending writes to the underlying
+	// Close synchronizes any pending writes to the underlying
 	// storage and closes the file/connection
 	Close() error
 

vendor/github.com/theupdateframework/notary/client/client.go

@@ -7,10 +7,8 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
-	"net/url"
 	"os"
 	"path/filepath"
-	"regexp"
 	"time"
 
 	canonicaljson "github.com/docker/go/canonical/json"
@@ -39,7 +37,6 @@ func init() {
 
 // repository stores all the information needed to operate on a notary repository.
 type repository struct {
-	baseDir        string
 	gun            data.GUN
 	baseURL        string
 	changelist     changelist.Changelist
@@ -56,7 +53,8 @@ type repository struct {
 // NewFileCachedRepository is a wrapper for NewRepository that initializes
 // a file cache from the provided repository, local config information and a crypto service.
 // It also retrieves the remote store associated to the base directory under where all the
-// trust files will be stored and the specified GUN.
+// trust files will be stored (This is normally defaults to "~/.notary" or "~/.docker/trust"
+// when enabling Docker content trust) and the specified GUN.
 //
 // In case of a nil RoundTripper, a default offline store is used instead.
 func NewFileCachedRepository(baseDir string, gun data.GUN, baseURL string, rt http.RoundTripper,
@@ -90,16 +88,13 @@ func NewFileCachedRepository(baseDir string, gun data.GUN, baseURL string, rt ht
 		return nil, err
 	}
 
-	return NewRepository(baseDir, gun, baseURL, remoteStore, cache, trustPinning, cryptoService, cl)
+	return NewRepository(gun, baseURL, remoteStore, cache, trustPinning, cryptoService, cl)
 }
 
 // NewRepository is the base method that returns a new notary repository.
-// It takes the base directory under where all the trust files will be stored
-// (This is normally defaults to "~/.notary" or "~/.docker/trust" when enabling
-// docker content trust).
 // It expects an initialized cache. In case of a nil remote store, a default
 // offline store is used.
-func NewRepository(baseDir string, gun data.GUN, baseURL string, remoteStore store.RemoteStore, cache store.MetadataStore,
+func NewRepository(gun data.GUN, baseURL string, remoteStore store.RemoteStore, cache store.MetadataStore,
 	trustPinning trustpinning.TrustPinConfig, cryptoService signed.CryptoService, cl changelist.Changelist) (Repository, error) {
 
 	// Repo's remote store is either a valid remote store or an OfflineStore
@@ -114,7 +109,6 @@ func NewRepository(baseDir string, gun data.GUN, baseURL string, remoteStore sto
 	nRepo := &repository{
 		gun:            gun,
 		baseURL:        baseURL,
-		baseDir:        baseDir,
 		changelist:     cl,
 		cache:          cache,
 		remoteStore:    remoteStore,
@@ -131,20 +125,62 @@ func (r *repository) GetGUN() data.GUN {
 	return r.gun
 }
 
-// Target represents a simplified version of the data TUF operates on, so external
-// applications don't have to depend on TUF data types.
-type Target struct {
-	Name   string                    // the name of the target
-	Hashes data.Hashes               // the hash of the target
-	Length int64                     // the size in bytes of the target
-	Custom *canonicaljson.RawMessage // the custom data provided to describe the file at TARGETPATH
+func (r *repository) updateTUF(forWrite bool) error {
+	repo, invalid, err := LoadTUFRepo(TUFLoadOptions{
+		GUN:                    r.gun,
+		TrustPinning:           r.trustPinning,
+		CryptoService:          r.cryptoService,
+		Cache:                  r.cache,
+		RemoteStore:            r.remoteStore,
+		AlwaysCheckInitialized: forWrite,
+	})
+	if err != nil {
+		return err
+	}
+	r.tufRepo = repo
+	r.invalid = invalid
+	return nil
 }
 
-// TargetWithRole represents a Target that exists in a particular role - this is
-// produced by ListTargets and GetTargetByName
-type TargetWithRole struct {
-	Target
-	Role data.RoleName
+// ListTargets calls update first before listing targets
+func (r *repository) ListTargets(roles ...data.RoleName) ([]*TargetWithRole, error) {
+	if err := r.updateTUF(false); err != nil {
+		return nil, err
+	}
+	return NewReadOnly(r.tufRepo).ListTargets(roles...)
+}
+
+// GetTargetByName calls update first before getting target by name
+func (r *repository) GetTargetByName(name string, roles ...data.RoleName) (*TargetWithRole, error) {
+	if err := r.updateTUF(false); err != nil {
+		return nil, err
+	}
+	return NewReadOnly(r.tufRepo).GetTargetByName(name, roles...)
+}
+
+// GetAllTargetMetadataByName calls update first before getting targets by name
+func (r *repository) GetAllTargetMetadataByName(name string) ([]TargetSignedStruct, error) {
+	if err := r.updateTUF(false); err != nil {
+		return nil, err
+	}
+	return NewReadOnly(r.tufRepo).GetAllTargetMetadataByName(name)
+
+}
+
+// ListRoles calls update first before getting roles
+func (r *repository) ListRoles() ([]RoleWithSignatures, error) {
+	if err := r.updateTUF(false); err != nil {
+		return nil, err
+	}
+	return NewReadOnly(r.tufRepo).ListRoles()
+}
+
+// GetDelegationRoles calls update first before getting all delegation roles
+func (r *repository) GetDelegationRoles() ([]data.Role, error) {
+	if err := r.updateTUF(false); err != nil {
+		return nil, err
+	}
+	return NewReadOnly(r.tufRepo).GetDelegationRoles()
 }
 
 // NewTarget is a helper method that returns a Target
@@ -493,167 +529,6 @@ func (r *repository) RemoveTarget(targetName string, roles ...data.RoleName) err
 	return addChange(r.changelist, template, roles...)
 }
 
-// ListTargets lists all targets for the current repository. The list of
-// roles should be passed in order from highest to lowest priority.
-//
-// IMPORTANT: if you pass a set of roles such as [ "targets/a", "targets/x"
-// "targets/a/b" ], even though "targets/a/b" is part of the "targets/a" subtree
-// its entries will be strictly shadowed by those in other parts of the "targets/a"
-// subtree and also the "targets/x" subtree, as we will defer parsing it until
-// we explicitly reach it in our iteration of the provided list of roles.
-func (r *repository) ListTargets(roles ...data.RoleName) ([]*TargetWithRole, error) {
-	if err := r.Update(false); err != nil {
-		return nil, err
-	}
-
-	if len(roles) == 0 {
-		roles = []data.RoleName{data.CanonicalTargetsRole}
-	}
-	targets := make(map[string]*TargetWithRole)
-	for _, role := range roles {
-		// Define an array of roles to skip for this walk (see IMPORTANT comment above)
-		skipRoles := utils.RoleNameSliceRemove(roles, role)
-
-		// Define a visitor function to populate the targets map in priority order
-		listVisitorFunc := func(tgt *data.SignedTargets, validRole data.DelegationRole) interface{} {
-			// We found targets so we should try to add them to our targets map
-			for targetName, targetMeta := range tgt.Signed.Targets {
-				// Follow the priority by not overriding previously set targets
-				// and check that this path is valid with this role
-				if _, ok := targets[targetName]; ok || !validRole.CheckPaths(targetName) {
-					continue
-				}
-				targets[targetName] = &TargetWithRole{
-					Target: Target{
-						Name:   targetName,
-						Hashes: targetMeta.Hashes,
-						Length: targetMeta.Length,
-						Custom: targetMeta.Custom,
-					},
-					Role: validRole.Name,
-				}
-			}
-			return nil
-		}
-
-		r.tufRepo.WalkTargets("", role, listVisitorFunc, skipRoles...)
-	}
-
-	var targetList []*TargetWithRole
-	for _, v := range targets {
-		targetList = append(targetList, v)
-	}
-
-	return targetList, nil
-}
-
-// GetTargetByName returns a target by the given name. If no roles are passed
-// it uses the targets role and does a search of the entire delegation
-// graph, finding the first entry in a breadth first search of the delegations.
-// If roles are passed, they should be passed in descending priority and
-// the target entry found in the subtree of the highest priority role
-// will be returned.
-// See the IMPORTANT section on ListTargets above. Those roles also apply here.
-func (r *repository) GetTargetByName(name string, roles ...data.RoleName) (*TargetWithRole, error) {
-	if err := r.Update(false); err != nil {
-		return nil, err
-	}
-
-	if len(roles) == 0 {
-		roles = append(roles, data.CanonicalTargetsRole)
-	}
-	var resultMeta data.FileMeta
-	var resultRoleName data.RoleName
-	var foundTarget bool
-	for _, role := range roles {
-		// Define an array of roles to skip for this walk (see IMPORTANT comment above)
-		skipRoles := utils.RoleNameSliceRemove(roles, role)
-
-		// Define a visitor function to find the specified target
-		getTargetVisitorFunc := func(tgt *data.SignedTargets, validRole data.DelegationRole) interface{} {
-			if tgt == nil {
-				return nil
-			}
-			// We found the target and validated path compatibility in our walk,
-			// so we should stop our walk and set the resultMeta and resultRoleName variables
-			if resultMeta, foundTarget = tgt.Signed.Targets[name]; foundTarget {
-				resultRoleName = validRole.Name
-				return tuf.StopWalk{}
-			}
-			return nil
-		}
-		// Check that we didn't error, and that we assigned to our target
-		if err := r.tufRepo.WalkTargets(name, role, getTargetVisitorFunc, skipRoles...); err == nil && foundTarget {
-			return &TargetWithRole{Target: Target{Name: name, Hashes: resultMeta.Hashes, Length: resultMeta.Length, Custom: resultMeta.Custom}, Role: resultRoleName}, nil
-		}
-	}
-	return nil, ErrNoSuchTarget(name)
-
-}
-
-// TargetSignedStruct is a struct that contains a Target, the role it was found in, and the list of signatures for that role
-type TargetSignedStruct struct {
-	Role       data.DelegationRole
-	Target     Target
-	Signatures []data.Signature
-}
-
-//ErrNoSuchTarget is returned when no valid trust data is found.
-type ErrNoSuchTarget string
-
-func (f ErrNoSuchTarget) Error() string {
-	return fmt.Sprintf("No valid trust data for %s", string(f))
-}
-
-// GetAllTargetMetadataByName searches the entire delegation role tree to find the specified target by name for all
-// roles, and returns a list of TargetSignedStructs for each time it finds the specified target.
-// If given an empty string for a target name, it will return back all targets signed into the repository in every role
-func (r *repository) GetAllTargetMetadataByName(name string) ([]TargetSignedStruct, error) {
-	if err := r.Update(false); err != nil {
-		return nil, err
-	}
-
-	var targetInfoList []TargetSignedStruct
-
-	// Define a visitor function to find the specified target
-	getAllTargetInfoByNameVisitorFunc := func(tgt *data.SignedTargets, validRole data.DelegationRole) interface{} {
-		if tgt == nil {
-			return nil
-		}
-		// We found a target and validated path compatibility in our walk,
-		// so add it to our list if we have a match
-		// if we have an empty name, add all targets, else check if we have it
-		var targetMetaToAdd data.Files
-		if name == "" {
-			targetMetaToAdd = tgt.Signed.Targets
-		} else {
-			if meta, ok := tgt.Signed.Targets[name]; ok {
-				targetMetaToAdd = data.Files{name: meta}
-			}
-		}
-
-		for targetName, resultMeta := range targetMetaToAdd {
-			targetInfo := TargetSignedStruct{
-				Role:       validRole,
-				Target:     Target{Name: targetName, Hashes: resultMeta.Hashes, Length: resultMeta.Length, Custom: resultMeta.Custom},
-				Signatures: tgt.Signatures,
-			}
-			targetInfoList = append(targetInfoList, targetInfo)
-		}
-		// continue walking to all child roles
-		return nil
-	}
-
-	// Check that we didn't error, and that we found the target at least once
-	if err := r.tufRepo.WalkTargets(name, "", getAllTargetInfoByNameVisitorFunc); err != nil {
-		return nil, err
-	}
-	if len(targetInfoList) == 0 {
-		return nil, ErrNoSuchTarget(name)
-	}
-	return targetInfoList, nil
-}
-
 // GetChangelist returns the list of the repository's unpublished changes
 func (r *repository) GetChangelist() (changelist.Changelist, error) {
 	return r.changelist, nil
@@ -671,51 +546,6 @@ func (r *repository) getRemoteStore() store.RemoteStore {
 	return r.remoteStore
 }
 
-// RoleWithSignatures is a Role with its associated signatures
-type RoleWithSignatures struct {
-	Signatures []data.Signature
-	data.Role
-}
-
-// ListRoles returns a list of RoleWithSignatures objects for this repo
-// This represents the latest metadata for each role in this repo
-func (r *repository) ListRoles() ([]RoleWithSignatures, error) {
-	// Update to latest repo state
-	if err := r.Update(false); err != nil {
-		return nil, err
-	}
-
-	// Get all role info from our updated keysDB, can be empty
-	roles := r.tufRepo.GetAllLoadedRoles()
-
-	var roleWithSigs []RoleWithSignatures
-
-	// Populate RoleWithSignatures with Role from keysDB and signatures from TUF metadata
-	for _, role := range roles {
-		roleWithSig := RoleWithSignatures{Role: *role, Signatures: nil}
-		switch role.Name {
-		case data.CanonicalRootRole:
-			roleWithSig.Signatures = r.tufRepo.Root.Signatures
-		case data.CanonicalTargetsRole:
-			roleWithSig.Signatures = r.tufRepo.Targets[data.CanonicalTargetsRole].Signatures
-		case data.CanonicalSnapshotRole:
-			roleWithSig.Signatures = r.tufRepo.Snapshot.Signatures
-		case data.CanonicalTimestampRole:
-			roleWithSig.Signatures = r.tufRepo.Timestamp.Signatures
-		default:
-			if !data.IsDelegation(role.Name) {
-				continue
-			}
-			if _, ok := r.tufRepo.Targets[role.Name]; ok {
-				// We'll only find a signature if we've published any targets with this delegation
-				roleWithSig.Signatures = r.tufRepo.Targets[role.Name].Signatures
-			}
-		}
-		roleWithSigs = append(roleWithSigs, roleWithSig)
-	}
-	return roleWithSigs, nil
-}
-
 // Publish pushes the local changes in signed material to the remote notary-server
 // Conceptually it performs an operation similar to a `git rebase`
 func (r *repository) Publish() error {
@@ -736,7 +566,7 @@ func (r *repository) Publish() error {
 func (r *repository) publish(cl changelist.Changelist) error {
 	var initialPublish bool
 	// update first before publishing
-	if err := r.Update(true); err != nil {
+	if err := r.updateTUF(true); err != nil {
 		// If the remote is not aware of the repo, then this is being published
 		// for the first time.  Try to initialize the repository before publishing.
 		if _, ok := err.(ErrRepositoryNotExist); ok {
@@ -863,7 +693,14 @@ func (r *repository) oldKeysForLegacyClientSupport(legacyVersions int, initialPu
 	}
 	oldKeys := make(map[string]data.PublicKey)
 
-	c, err := r.bootstrapClient(true)
+	c, err := bootstrapClient(TUFLoadOptions{
+		GUN:                    r.gun,
+		TrustPinning:           r.trustPinning,
+		CryptoService:          r.cryptoService,
+		Cache:                  r.cache,
+		RemoteStore:            r.remoteStore,
+		AlwaysCheckInitialized: true,
+	})
 	// require a server connection to fetch old roots
 	if err != nil {
 		return nil, err
@@ -1003,135 +840,6 @@ func (r *repository) saveMetadata(ignoreSnapshot bool) error {
 	return r.cache.Set(data.CanonicalSnapshotRole.String(), snapshotJSON)
 }
 
-// returns a properly constructed ErrRepositoryNotExist error based on this
-// repo's information
-func (r *repository) errRepositoryNotExist() error {
-	host := r.baseURL
-	parsed, err := url.Parse(r.baseURL)
-	if err == nil {
-		host = parsed.Host // try to exclude the scheme and any paths
-	}
-	return ErrRepositoryNotExist{remote: host, gun: r.gun}
-}
-
-// Update bootstraps a trust anchor (root.json) before updating all the
-// metadata from the repo.
-func (r *repository) Update(forWrite bool) error {
-	c, err := r.bootstrapClient(forWrite)
-	if err != nil {
-		if _, ok := err.(store.ErrMetaNotFound); ok {
-			return r.errRepositoryNotExist()
-		}
-		return err
-	}
-	repo, invalid, err := c.Update()
-	if err != nil {
-		// notFound.Resource may include a version or checksum so when the role is root,
-		// it will be root, <version>.root or root.<checksum>.
-		notFound, ok := err.(store.ErrMetaNotFound)
-		isRoot, _ := regexp.MatchString(`\.?`+data.CanonicalRootRole.String()+`\.?`, notFound.Resource)
-		if ok && isRoot {
-			return r.errRepositoryNotExist()
-		}
-		return err
-	}
-	// we can be assured if we are at this stage that the repo we built is good
-	// no need to test the following function call for an error as it will always be fine should the repo be good- it is!
-	r.tufRepo = repo
-	r.invalid = invalid
-	warnRolesNearExpiry(repo)
-	return nil
-}
-
-// bootstrapClient attempts to bootstrap a root.json to be used as the trust
-// anchor for a repository. The checkInitialized argument indicates whether
-// we should always attempt to contact the server to determine if the repository
-// is initialized or not. If set to true, we will always attempt to download
-// and return an error if the remote repository errors.
-//
-// Populates a tuf.RepoBuilder with this root metadata. If the root metadata
-// downloaded is a newer version than what is on disk, then intermediate
-// versions will be downloaded and verified in order to rotate trusted keys
-// properly. Newer root metadata must always be signed with the previous
-// threshold and keys.
-//
-// Fails if the remote server is reachable and does not know the repo
-// (i.e. before the first r.Publish()), in which case the error is
-// store.ErrMetaNotFound, or if the root metadata (from whichever source is used)
-// is not trusted.
-//
-// Returns a TUFClient for the remote server, which may not be actually
-// operational (if the URL is invalid but a root.json is cached).
-func (r *repository) bootstrapClient(checkInitialized bool) (*tufClient, error) {
-	minVersion := 1
-	// the old root on disk should not be validated against any trust pinning configuration
-	// because if we have an old root, it itself is the thing that pins trust
-	oldBuilder := tuf.NewRepoBuilder(r.gun, r.GetCryptoService(), trustpinning.TrustPinConfig{})
-
-	// by default, we want to use the trust pinning configuration on any new root that we download
-	newBuilder := tuf.NewRepoBuilder(r.gun, r.GetCryptoService(), r.trustPinning)
-
-	// Try to read root from cache first. We will trust this root until we detect a problem
-	// during update which will cause us to download a new root and perform a rotation.
-	// If we have an old root, and it's valid, then we overwrite the newBuilder to be one
-	// preloaded with the old root or one which uses the old root for trust bootstrapping.
-	if rootJSON, err := r.cache.GetSized(data.CanonicalRootRole.String(), store.NoSizeLimit); err == nil {
-		// if we can't load the cached root, fail hard because that is how we pin trust
-		if err := oldBuilder.Load(data.CanonicalRootRole, rootJSON, minVersion, true); err != nil {
-			return nil, err
-		}
-
-		// again, the root on disk is the source of trust pinning, so use an empty trust
-		// pinning configuration
-		newBuilder = tuf.NewRepoBuilder(r.gun, r.GetCryptoService(), trustpinning.TrustPinConfig{})
-
-		if err := newBuilder.Load(data.CanonicalRootRole, rootJSON, minVersion, false); err != nil {
-			// Ok, the old root is expired - we want to download a new one.  But we want to use the
-			// old root to verify the new root, so bootstrap a new builder with the old builder
-			// but use the trustpinning to validate the new root
-			minVersion = oldBuilder.GetLoadedVersion(data.CanonicalRootRole)
-			newBuilder = oldBuilder.BootstrapNewBuilderWithNewTrustpin(r.trustPinning)
-		}
-	}
-
-	remote := r.getRemoteStore()
-
-	if !newBuilder.IsLoaded(data.CanonicalRootRole) || checkInitialized {
-		// remoteErr was nil and we were not able to load a root from cache or
-		// are specifically checking for initialization of the repo.
-
-		// if remote store successfully set up, try and get root from remote
-		// We don't have any local data to determine the size of root, so try the maximum (though it is restricted at 100MB)
-		tmpJSON, err := remote.GetSized(data.CanonicalRootRole.String(), store.NoSizeLimit)
-		if err != nil {
-			// we didn't have a root in cache and were unable to load one from
-			// the server. Nothing we can do but error.
-			return nil, err
-		}
-
-		if !newBuilder.IsLoaded(data.CanonicalRootRole) {
-			// we always want to use the downloaded root if we couldn't load from cache
-			if err := newBuilder.Load(data.CanonicalRootRole, tmpJSON, minVersion, false); err != nil {
-				return nil, err
-			}
-
-			err = r.cache.Set(data.CanonicalRootRole.String(), tmpJSON)
-			if err != nil {
-				// if we can't write cache we should still continue, just log error
-				logrus.Errorf("could not save root to cache: %s", err.Error())
-			}
-		}
-	}
-
-	// We can only get here if remoteErr != nil (hence we don't download any new root),
-	// and there was no root on disk
-	if !newBuilder.IsLoaded(data.CanonicalRootRole) {
-		return nil, ErrRepoNotInitialized{}
-	}
-
-	return newTufClient(oldBuilder, newBuilder, remote, r.cache), nil
-}
-
 // RotateKey removes all existing keys associated with the role. If no keys are
 // specified in keyList, then this creates and adds one new key or delegates
 // managing the key to the server. If key(s) are specified by keyList, then they are
@@ -1273,7 +981,7 @@ func DeleteTrustData(baseDir string, gun data.GUN, URL string, rt http.RoundTrip
 	if deleteRemote {
 		remote, err := getRemoteStore(URL, gun, rt)
 		if err != nil {
-			logrus.Error("unable to instantiate a remote store: %v", err)
+			logrus.Errorf("unable to instantiate a remote store: %v", err)
 			return err
 		}
 		if err := remote.RemoveAll(); err != nil {

vendor/github.com/theupdateframework/notary/client/delegations.go

@@ -7,7 +7,6 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/theupdateframework/notary"
 	"github.com/theupdateframework/notary/client/changelist"
-	store "github.com/theupdateframework/notary/storage"
 	"github.com/theupdateframework/notary/tuf/data"
 	"github.com/theupdateframework/notary/tuf/utils"
 )
@@ -77,7 +76,7 @@ func (r *repository) AddDelegationPaths(name data.RoleName, paths []string) erro
 }
 
 // RemoveDelegationKeysAndPaths creates changelist entries to remove provided delegation key IDs and paths.
-// This method composes RemoveDelegationPaths and RemoveDelegationKeys (each creates one changelist if called).
+// This method composes RemoveDelegationPaths and RemoveDelegationKeys (each creates one changelist entry if called).
 func (r *repository) RemoveDelegationKeysAndPaths(name data.RoleName, keyIDs, paths []string) error {
 	if len(paths) > 0 {
 		err := r.RemoveDelegationPaths(name, paths)
@@ -201,41 +200,6 @@ func newDeleteDelegationChange(name data.RoleName, content []byte) *changelist.T
 	)
 }
 
-// GetDelegationRoles returns the keys and roles of the repository's delegations
-// Also converts key IDs to canonical key IDs to keep consistent with signing prompts
-func (r *repository) GetDelegationRoles() ([]data.Role, error) {
-	// Update state of the repo to latest
-	if err := r.Update(false); err != nil {
-		return nil, err
-	}
-
-	// All top level delegations (ex: targets/level1) are stored exclusively in targets.json
-	_, ok := r.tufRepo.Targets[data.CanonicalTargetsRole]
-	if !ok {
-		return nil, store.ErrMetaNotFound{Resource: data.CanonicalTargetsRole.String()}
-	}
-
-	// make a copy for traversing nested delegations
-	allDelegations := []data.Role{}
-
-	// Define a visitor function to populate the delegations list and translate their key IDs to canonical IDs
-	delegationCanonicalListVisitor := func(tgt *data.SignedTargets, validRole data.DelegationRole) interface{} {
-		// For the return list, update with a copy that includes canonicalKeyIDs
-		// These aren't validated by the validRole
-		canonicalDelegations, err := translateDelegationsToCanonicalIDs(tgt.Signed.Delegations)
-		if err != nil {
-			return err
-		}
-		allDelegations = append(allDelegations, canonicalDelegations...)
-		return nil
-	}
-	err := r.tufRepo.WalkTargets("", "", delegationCanonicalListVisitor)
-	if err != nil {
-		return nil, err
-	}
-	return allDelegations, nil
-}
-
 func translateDelegationsToCanonicalIDs(delegationInfo data.Delegations) ([]data.Role, error) {
 	canonicalDelegations := make([]data.Role, len(delegationInfo.Roles))
 	// Do a copy by value to ensure local delegation metadata is untouched

vendor/github.com/theupdateframework/notary/client/interface.go

@@ -6,42 +6,145 @@ import (
 	"github.com/theupdateframework/notary/tuf/signed"
 )
 
-// Repository represents the set of options that must be supported over a TUF repo.
-type Repository interface {
-	// General management operations
-	Initialize(rootKeyIDs []string, serverManagedRoles ...data.RoleName) error
-	InitializeWithCertificate(rootKeyIDs []string, rootCerts []data.PublicKey, serverManagedRoles ...data.RoleName) error
-	Publish() error
-
-	// Target Operations
-	AddTarget(target *Target, roles ...data.RoleName) error
-	RemoveTarget(targetName string, roles ...data.RoleName) error
+// ReadOnly represents the set of options that must be supported over a TUF repo for
+// reading
+type ReadOnly interface {
+	// ListTargets lists all targets for the current repository. The list of
+	// roles should be passed in order from highest to lowest priority.
+	//
+	// IMPORTANT: if you pass a set of roles such as [ "targets/a", "targets/x"
+	// "targets/a/b" ], even though "targets/a/b" is part of the "targets/a" subtree
+	// its entries will be strictly shadowed by those in other parts of the "targets/a"
+	// subtree and also the "targets/x" subtree, as we will defer parsing it until
+	// we explicitly reach it in our iteration of the provided list of roles.
 	ListTargets(roles ...data.RoleName) ([]*TargetWithRole, error)
+
+	// GetTargetByName returns a target by the given name. If no roles are passed
+	// it uses the targets role and does a search of the entire delegation
+	// graph, finding the first entry in a breadth first search of the delegations.
+	// If roles are passed, they should be passed in descending priority and
+	// the target entry found in the subtree of the highest priority role
+	// will be returned.
+	// See the IMPORTANT section on ListTargets above. Those roles also apply here.
 	GetTargetByName(name string, roles ...data.RoleName) (*TargetWithRole, error)
+
+	// GetAllTargetMetadataByName searches the entire delegation role tree to find
+	// the specified target by name for all roles, and returns a list of
+	// TargetSignedStructs for each time it finds the specified target.
+	// If given an empty string for a target name, it will return back all targets
+	// signed into the repository in every role
 	GetAllTargetMetadataByName(name string) ([]TargetSignedStruct, error)
 
-	// Changelist operations
+	// ListRoles returns a list of RoleWithSignatures objects for this repo
+	// This represents the latest metadata for each role in this repo
+	ListRoles() ([]RoleWithSignatures, error)
+
+	// GetDelegationRoles returns the keys and roles of the repository's delegations
+	// Also converts key IDs to canonical key IDs to keep consistent with signing prompts
+	GetDelegationRoles() ([]data.Role, error)
+}
+
+// Repository represents the set of options that must be supported over a TUF repo
+// for both reading and writing.
+type Repository interface {
+	ReadOnly
+
+	// ------------------- Publishing operations -------------------
+
+	// GetGUN returns the GUN associated with the repository
+	GetGUN() data.GUN
+
+	// SetLegacyVersion sets the number of versions back to fetch roots to sign with
+	SetLegacyVersions(int)
+
+	// ----- General management operations -----
+
+	// Initialize creates a new repository by using rootKey as the root Key for the
+	// TUF repository. The remote store/server must be reachable (and is asked to
+	// generate a timestamp key and possibly other serverManagedRoles), but the
+	// created repository result is only stored on local cache, not published to
+	// the remote store. To do that, use r.Publish() eventually.
+	Initialize(rootKeyIDs []string, serverManagedRoles ...data.RoleName) error
+
+	// InitializeWithCertificate initializes the repository with root keys and their
+	// corresponding certificates
+	InitializeWithCertificate(rootKeyIDs []string, rootCerts []data.PublicKey, serverManagedRoles ...data.RoleName) error
+
+	// Publish pushes the local changes in signed material to the remote notary-server
+	// Conceptually it performs an operation similar to a `git rebase`
+	Publish() error
+
+	// ----- Target Operations -----
+
+	// AddTarget creates new changelist entries to add a target to the given roles
+	// in the repository when the changelist gets applied at publish time.
+	// If roles are unspecified, the default role is "targets"
+	AddTarget(target *Target, roles ...data.RoleName) error
+
+	// RemoveTarget creates new changelist entries to remove a target from the given
+	// roles in the repository when the changelist gets applied at publish time.
+	// If roles are unspecified, the default role is "target".
+	RemoveTarget(targetName string, roles ...data.RoleName) error
+
+	// ----- Changelist operations -----
+
+	// GetChangelist returns the list of the repository's unpublished changes
 	GetChangelist() (changelist.Changelist, error)
 
-	// Role operations
-	ListRoles() ([]RoleWithSignatures, error)
-	GetDelegationRoles() ([]data.Role, error)
+	// ----- Role operations -----
+
+	// AddDelegation creates changelist entries to add provided delegation public keys and paths.
+	// This method composes AddDelegationRoleAndKeys and AddDelegationPaths (each creates one changelist if called).
 	AddDelegation(name data.RoleName, delegationKeys []data.PublicKey, paths []string) error
+
+	// AddDelegationRoleAndKeys creates a changelist entry to add provided delegation public keys.
+	// This method is the simplest way to create a new delegation, because the delegation must have at least
+	// one key upon creation to be valid since we will reject the changelist while validating the threshold.
 	AddDelegationRoleAndKeys(name data.RoleName, delegationKeys []data.PublicKey) error
+
+	// AddDelegationPaths creates a changelist entry to add provided paths to an existing delegation.
+	// This method cannot create a new delegation itself because the role must meet the key threshold upon
+	// creation.
 	AddDelegationPaths(name data.RoleName, paths []string) error
+
+	// RemoveDelegationKeysAndPaths creates changelist entries to remove provided delegation key IDs and
+	// paths. This method composes RemoveDelegationPaths and RemoveDelegationKeys (each creates one
+	// changelist entry if called).
 	RemoveDelegationKeysAndPaths(name data.RoleName, keyIDs, paths []string) error
+
+	// RemoveDelegationRole creates a changelist to remove all paths and keys from a role, and delete the
+	// role in its entirety.
 	RemoveDelegationRole(name data.RoleName) error
+
+	// RemoveDelegationPaths creates a changelist entry to remove provided paths from an existing delegation.
 	RemoveDelegationPaths(name data.RoleName, paths []string) error
+
+	// RemoveDelegationKeys creates a changelist entry to remove provided keys from an existing delegation.
+	// When this changelist is applied, if the specified keys are the only keys left in the role,
+	// the role itself will be deleted in its entirety.
+	// It can also delete a key from all delegations under a parent using a name
+	// with a wildcard at the end.
 	RemoveDelegationKeys(name data.RoleName, keyIDs []string) error
+
+	// ClearDelegationPaths creates a changelist entry to remove all paths from an existing delegation.
 	ClearDelegationPaths(name data.RoleName) error
 
-	// Witness and other re-signing operations
+	// ----- Witness and other re-signing operations -----
+
+	// Witness creates change objects to witness (i.e. re-sign) the given
+	// roles on the next publish. One change is created per role
 	Witness(roles ...data.RoleName) ([]data.RoleName, error)
 
-	// Key Operations
+	// ----- Key Operations -----
+
+	// RotateKey removes all existing keys associated with the role. If no keys are
+	// specified in keyList, then this creates and adds one new key or delegates
+	// managing the key to the server. If key(s) are specified by keyList, then they are
+	// used for signing the role.
+	// These changes are staged in a changelist until publish is called.
 	RotateKey(role data.RoleName, serverManagesKey bool, keyList []string) error
 
+	// GetCryptoService is the getter for the repository's CryptoService, which is used
+	// to sign all updates.
 	GetCryptoService() signed.CryptoService
-	SetLegacyVersions(int)
-	GetGUN() data.GUN
 }

vendor/github.com/theupdateframework/notary/client/reader.go

@@ -0,0 +1,257 @@
+package client
+
+import (
+	"fmt"
+
+	canonicaljson "github.com/docker/go/canonical/json"
+	store "github.com/theupdateframework/notary/storage"
+	"github.com/theupdateframework/notary/tuf"
+	"github.com/theupdateframework/notary/tuf/data"
+	"github.com/theupdateframework/notary/tuf/utils"
+)
+
+// Target represents a simplified version of the data TUF operates on, so external
+// applications don't have to depend on TUF data types.
+type Target struct {
+	Name   string                    // the name of the target
+	Hashes data.Hashes               // the hash of the target
+	Length int64                     // the size in bytes of the target
+	Custom *canonicaljson.RawMessage // the custom data provided to describe the file at TARGETPATH
+}
+
+// TargetWithRole represents a Target that exists in a particular role - this is
+// produced by ListTargets and GetTargetByName
+type TargetWithRole struct {
+	Target
+	Role data.RoleName
+}
+
+// TargetSignedStruct is a struct that contains a Target, the role it was found in, and the list of signatures for that role
+type TargetSignedStruct struct {
+	Role       data.DelegationRole
+	Target     Target
+	Signatures []data.Signature
+}
+
+//ErrNoSuchTarget is returned when no valid trust data is found.
+type ErrNoSuchTarget string
+
+func (f ErrNoSuchTarget) Error() string {
+	return fmt.Sprintf("No valid trust data for %s", string(f))
+}
+
+// RoleWithSignatures is a Role with its associated signatures
+type RoleWithSignatures struct {
+	Signatures []data.Signature
+	data.Role
+}
+
+// NewReadOnly is the base method that returns a new notary repository for reading.
+// It expects an initialized cache. In case of a nil remote store, a default
+// offline store is used.
+func NewReadOnly(repo *tuf.Repo) ReadOnly {
+	return &reader{tufRepo: repo}
+}
+
+type reader struct {
+	tufRepo *tuf.Repo
+}
+
+// ListTargets lists all targets for the current repository. The list of
+// roles should be passed in order from highest to lowest priority.
+//
+// IMPORTANT: if you pass a set of roles such as [ "targets/a", "targets/x"
+// "targets/a/b" ], even though "targets/a/b" is part of the "targets/a" subtree
+// its entries will be strictly shadowed by those in other parts of the "targets/a"
+// subtree and also the "targets/x" subtree, as we will defer parsing it until
+// we explicitly reach it in our iteration of the provided list of roles.
+func (r *reader) ListTargets(roles ...data.RoleName) ([]*TargetWithRole, error) {
+	if len(roles) == 0 {
+		roles = []data.RoleName{data.CanonicalTargetsRole}
+	}
+	targets := make(map[string]*TargetWithRole)
+	for _, role := range roles {
+		// Define an array of roles to skip for this walk (see IMPORTANT comment above)
+		skipRoles := utils.RoleNameSliceRemove(roles, role)
+
+		// Define a visitor function to populate the targets map in priority order
+		listVisitorFunc := func(tgt *data.SignedTargets, validRole data.DelegationRole) interface{} {
+			// We found targets so we should try to add them to our targets map
+			for targetName, targetMeta := range tgt.Signed.Targets {
+				// Follow the priority by not overriding previously set targets
+				// and check that this path is valid with this role
+				if _, ok := targets[targetName]; ok || !validRole.CheckPaths(targetName) {
+					continue
+				}
+				targets[targetName] = &TargetWithRole{
+					Target: Target{
+						Name:   targetName,
+						Hashes: targetMeta.Hashes,
+						Length: targetMeta.Length,
+						Custom: targetMeta.Custom,
+					},
+					Role: validRole.Name,
+				}
+			}
+			return nil
+		}
+
+		r.tufRepo.WalkTargets("", role, listVisitorFunc, skipRoles...)
+	}
+
+	var targetList []*TargetWithRole
+	for _, v := range targets {
+		targetList = append(targetList, v)
+	}
+
+	return targetList, nil
+}
+
+// GetTargetByName returns a target by the given name. If no roles are passed
+// it uses the targets role and does a search of the entire delegation
+// graph, finding the first entry in a breadth first search of the delegations.
+// If roles are passed, they should be passed in descending priority and
+// the target entry found in the subtree of the highest priority role
+// will be returned.
+// See the IMPORTANT section on ListTargets above. Those roles also apply here.
+func (r *reader) GetTargetByName(name string, roles ...data.RoleName) (*TargetWithRole, error) {
+	if len(roles) == 0 {
+		roles = append(roles, data.CanonicalTargetsRole)
+	}
+	var resultMeta data.FileMeta
+	var resultRoleName data.RoleName
+	var foundTarget bool
+	for _, role := range roles {
+		// Define an array of roles to skip for this walk (see IMPORTANT comment above)
+		skipRoles := utils.RoleNameSliceRemove(roles, role)
+
+		// Define a visitor function to find the specified target
+		getTargetVisitorFunc := func(tgt *data.SignedTargets, validRole data.DelegationRole) interface{} {
+			if tgt == nil {
+				return nil
+			}
+			// We found the target and validated path compatibility in our walk,
+			// so we should stop our walk and set the resultMeta and resultRoleName variables
+			if resultMeta, foundTarget = tgt.Signed.Targets[name]; foundTarget {
+				resultRoleName = validRole.Name
+				return tuf.StopWalk{}
+			}
+			return nil
+		}
+		// Check that we didn't error, and that we assigned to our target
+		if err := r.tufRepo.WalkTargets(name, role, getTargetVisitorFunc, skipRoles...); err == nil && foundTarget {
+			return &TargetWithRole{Target: Target{Name: name, Hashes: resultMeta.Hashes, Length: resultMeta.Length, Custom: resultMeta.Custom}, Role: resultRoleName}, nil
+		}
+	}
+	return nil, ErrNoSuchTarget(name)
+
+}
+
+// GetAllTargetMetadataByName searches the entire delegation role tree to find the specified target by name for all
+// roles, and returns a list of TargetSignedStructs for each time it finds the specified target.
+// If given an empty string for a target name, it will return back all targets signed into the repository in every role
+func (r *reader) GetAllTargetMetadataByName(name string) ([]TargetSignedStruct, error) {
+	var targetInfoList []TargetSignedStruct
+
+	// Define a visitor function to find the specified target
+	getAllTargetInfoByNameVisitorFunc := func(tgt *data.SignedTargets, validRole data.DelegationRole) interface{} {
+		if tgt == nil {
+			return nil
+		}
+		// We found a target and validated path compatibility in our walk,
+		// so add it to our list if we have a match
+		// if we have an empty name, add all targets, else check if we have it
+		var targetMetaToAdd data.Files
+		if name == "" {
+			targetMetaToAdd = tgt.Signed.Targets
+		} else {
+			if meta, ok := tgt.Signed.Targets[name]; ok {
+				targetMetaToAdd = data.Files{name: meta}
+			}
+		}
+
+		for targetName, resultMeta := range targetMetaToAdd {
+			targetInfo := TargetSignedStruct{
+				Role:       validRole,
+				Target:     Target{Name: targetName, Hashes: resultMeta.Hashes, Length: resultMeta.Length, Custom: resultMeta.Custom},
+				Signatures: tgt.Signatures,
+			}
+			targetInfoList = append(targetInfoList, targetInfo)
+		}
+		// continue walking to all child roles
+		return nil
+	}
+
+	// Check that we didn't error, and that we found the target at least once
+	if err := r.tufRepo.WalkTargets(name, "", getAllTargetInfoByNameVisitorFunc); err != nil {
+		return nil, err
+	}
+	if len(targetInfoList) == 0 {
+		return nil, ErrNoSuchTarget(name)
+	}
+	return targetInfoList, nil
+}
+
+// ListRoles returns a list of RoleWithSignatures objects for this repo
+// This represents the latest metadata for each role in this repo
+func (r *reader) ListRoles() ([]RoleWithSignatures, error) {
+	// Get all role info from our updated keysDB, can be empty
+	roles := r.tufRepo.GetAllLoadedRoles()
+
+	var roleWithSigs []RoleWithSignatures
+
+	// Populate RoleWithSignatures with Role from keysDB and signatures from TUF metadata
+	for _, role := range roles {
+		roleWithSig := RoleWithSignatures{Role: *role, Signatures: nil}
+		switch role.Name {
+		case data.CanonicalRootRole:
+			roleWithSig.Signatures = r.tufRepo.Root.Signatures
+		case data.CanonicalTargetsRole:
+			roleWithSig.Signatures = r.tufRepo.Targets[data.CanonicalTargetsRole].Signatures
+		case data.CanonicalSnapshotRole:
+			roleWithSig.Signatures = r.tufRepo.Snapshot.Signatures
+		case data.CanonicalTimestampRole:
+			roleWithSig.Signatures = r.tufRepo.Timestamp.Signatures
+		default:
+			if !data.IsDelegation(role.Name) {
+				continue
+			}
+			if _, ok := r.tufRepo.Targets[role.Name]; ok {
+				// We'll only find a signature if we've published any targets with this delegation
+				roleWithSig.Signatures = r.tufRepo.Targets[role.Name].Signatures
+			}
+		}
+		roleWithSigs = append(roleWithSigs, roleWithSig)
+	}
+	return roleWithSigs, nil
+}
+
+// GetDelegationRoles returns the keys and roles of the repository's delegations
+// Also converts key IDs to canonical key IDs to keep consistent with signing prompts
+func (r *reader) GetDelegationRoles() ([]data.Role, error) {
+	// All top level delegations (ex: targets/level1) are stored exclusively in targets.json
+	_, ok := r.tufRepo.Targets[data.CanonicalTargetsRole]
+	if !ok {
+		return nil, store.ErrMetaNotFound{Resource: data.CanonicalTargetsRole.String()}
+	}
+
+	// make a copy for traversing nested delegations
+	allDelegations := []data.Role{}
+
+	// Define a visitor function to populate the delegations list and translate their key IDs to canonical IDs
+	delegationCanonicalListVisitor := func(tgt *data.SignedTargets, validRole data.DelegationRole) interface{} {
+		// For the return list, update with a copy that includes canonicalKeyIDs
+		// These aren't validated by the validRole
+		canonicalDelegations, err := translateDelegationsToCanonicalIDs(tgt.Signed.Delegations)
+		if err != nil {
+			return err
+		}
+		allDelegations = append(allDelegations, canonicalDelegations...)
+		return nil
+	}
+	err := r.tufRepo.WalkTargets("", "", delegationCanonicalListVisitor)
+	if err != nil {
+		return nil, err
+	}
+	return allDelegations, nil
+}

vendor/github.com/theupdateframework/notary/client/tufclient.go

@@ -3,9 +3,11 @@ package client
 import (
 	"encoding/json"
 	"fmt"
+	"regexp"
 
 	"github.com/sirupsen/logrus"
 	"github.com/theupdateframework/notary"
+	"github.com/theupdateframework/notary/cryptoservice"
 	store "github.com/theupdateframework/notary/storage"
 	"github.com/theupdateframework/notary/trustpinning"
 	"github.com/theupdateframework/notary/tuf"
@@ -21,16 +23,6 @@ type tufClient struct {
 	newBuilder tuf.RepoBuilder
 }
 
-// newTufClient initialized a tufClient with the given repo, remote source of content, and cache
-func newTufClient(oldBuilder, newBuilder tuf.RepoBuilder, remote store.RemoteStore, cache store.MetadataStore) *tufClient {
-	return &tufClient{
-		oldBuilder: oldBuilder,
-		newBuilder: newBuilder,
-		remote:     remote,
-		cache:      cache,
-	}
-}
-
 // Update performs an update to the TUF repo as defined by the TUF spec
 func (c *tufClient) Update() (*tuf.Repo, *tuf.Repo, error) {
 	// 1. Get timestamp
@@ -139,7 +131,7 @@ func (c *tufClient) updateRoot() error {
 
 	// Write newest to cache
 	if err := c.cache.Set(data.CanonicalRootRole.String(), raw); err != nil {
-		logrus.Debugf("unable to write %s to cache: %d.%s", newestVersion, data.CanonicalRootRole, err)
+		logrus.Debugf("unable to write %d.%s to cache: %s", newestVersion, data.CanonicalRootRole, err)
 	}
 	logrus.Debugf("finished updating root files")
 	return nil
@@ -323,3 +315,149 @@ func (c *tufClient) tryLoadRemote(consistentInfo tuf.ConsistentInfo, old []byte)
 	}
 	return raw, nil
 }
+
+// TUFLoadOptions are provided to LoadTUFRepo, which loads a TUF repo from cache,
+// from a remote store, or both
+type TUFLoadOptions struct {
+	GUN                    data.GUN
+	TrustPinning           trustpinning.TrustPinConfig
+	CryptoService          signed.CryptoService
+	Cache                  store.MetadataStore
+	RemoteStore            store.RemoteStore
+	AlwaysCheckInitialized bool
+}
+
+// bootstrapClient attempts to bootstrap a root.json to be used as the trust
+// anchor for a repository. The checkInitialized argument indicates whether
+// we should always attempt to contact the server to determine if the repository
+// is initialized or not. If set to true, we will always attempt to download
+// and return an error if the remote repository errors.
+//
+// Populates a tuf.RepoBuilder with this root metadata. If the root metadata
+// downloaded is a newer version than what is on disk, then intermediate
+// versions will be downloaded and verified in order to rotate trusted keys
+// properly. Newer root metadata must always be signed with the previous
+// threshold and keys.
+//
+// Fails if the remote server is reachable and does not know the repo
+// (i.e. before any metadata has been published), in which case the error is
+// store.ErrMetaNotFound, or if the root metadata (from whichever source is used)
+// is not trusted.
+//
+// Returns a TUFClient for the remote server, which may not be actually
+// operational (if the URL is invalid but a root.json is cached).
+func bootstrapClient(l TUFLoadOptions) (*tufClient, error) {
+	minVersion := 1
+	// the old root on disk should not be validated against any trust pinning configuration
+	// because if we have an old root, it itself is the thing that pins trust
+	oldBuilder := tuf.NewRepoBuilder(l.GUN, l.CryptoService, trustpinning.TrustPinConfig{})
+
+	// by default, we want to use the trust pinning configuration on any new root that we download
+	newBuilder := tuf.NewRepoBuilder(l.GUN, l.CryptoService, l.TrustPinning)
+
+	// Try to read root from cache first. We will trust this root until we detect a problem
+	// during update which will cause us to download a new root and perform a rotation.
+	// If we have an old root, and it's valid, then we overwrite the newBuilder to be one
+	// preloaded with the old root or one which uses the old root for trust bootstrapping.
+	if rootJSON, err := l.Cache.GetSized(data.CanonicalRootRole.String(), store.NoSizeLimit); err == nil {
+		// if we can't load the cached root, fail hard because that is how we pin trust
+		if err := oldBuilder.Load(data.CanonicalRootRole, rootJSON, minVersion, true); err != nil {
+			return nil, err
+		}
+
+		// again, the root on disk is the source of trust pinning, so use an empty trust
+		// pinning configuration
+		newBuilder = tuf.NewRepoBuilder(l.GUN, l.CryptoService, trustpinning.TrustPinConfig{})
+
+		if err := newBuilder.Load(data.CanonicalRootRole, rootJSON, minVersion, false); err != nil {
+			// Ok, the old root is expired - we want to download a new one.  But we want to use the
+			// old root to verify the new root, so bootstrap a new builder with the old builder
+			// but use the trustpinning to validate the new root
+			minVersion = oldBuilder.GetLoadedVersion(data.CanonicalRootRole)
+			newBuilder = oldBuilder.BootstrapNewBuilderWithNewTrustpin(l.TrustPinning)
+		}
+	}
+
+	if !newBuilder.IsLoaded(data.CanonicalRootRole) || l.AlwaysCheckInitialized {
+		// remoteErr was nil and we were not able to load a root from cache or
+		// are specifically checking for initialization of the repo.
+
+		// if remote store successfully set up, try and get root from remote
+		// We don't have any local data to determine the size of root, so try the maximum (though it is restricted at 100MB)
+		tmpJSON, err := l.RemoteStore.GetSized(data.CanonicalRootRole.String(), store.NoSizeLimit)
+		if err != nil {
+			// we didn't have a root in cache and were unable to load one from
+			// the server. Nothing we can do but error.
+			return nil, err
+		}
+
+		if !newBuilder.IsLoaded(data.CanonicalRootRole) {
+			// we always want to use the downloaded root if we couldn't load from cache
+			if err := newBuilder.Load(data.CanonicalRootRole, tmpJSON, minVersion, false); err != nil {
+				return nil, err
+			}
+
+			err = l.Cache.Set(data.CanonicalRootRole.String(), tmpJSON)
+			if err != nil {
+				// if we can't write cache we should still continue, just log error
+				logrus.Errorf("could not save root to cache: %s", err.Error())
+			}
+		}
+	}
+
+	// We can only get here if remoteErr != nil (hence we don't download any new root),
+	// and there was no root on disk
+	if !newBuilder.IsLoaded(data.CanonicalRootRole) {
+		return nil, ErrRepoNotInitialized{}
+	}
+
+	return &tufClient{
+		oldBuilder: oldBuilder,
+		newBuilder: newBuilder,
+		remote:     l.RemoteStore,
+		cache:      l.Cache,
+	}, nil
+}
+
+// LoadTUFRepo bootstraps a trust anchor (root.json) from cache (if provided) before updating
+// all the metadata for the repo from the remote (if provided). It loads a TUF repo from cache,
+// from a remote store, or both.
+func LoadTUFRepo(options TUFLoadOptions) (*tuf.Repo, *tuf.Repo, error) {
+	// set some sane defaults, so nothing has to be provided necessarily
+	if options.RemoteStore == nil {
+		options.RemoteStore = store.OfflineStore{}
+	}
+	if options.Cache == nil {
+		options.Cache = store.NewMemoryStore(nil)
+	}
+	if options.CryptoService == nil {
+		options.CryptoService = cryptoservice.EmptyService
+	}
+
+	c, err := bootstrapClient(options)
+	if err != nil {
+		if _, ok := err.(store.ErrMetaNotFound); ok {
+			return nil, nil, ErrRepositoryNotExist{
+				remote: options.RemoteStore.Location(),
+				gun:    options.GUN,
+			}
+		}
+		return nil, nil, err
+	}
+	repo, invalid, err := c.Update()
+	if err != nil {
+		// notFound.Resource may include a version or checksum so when the role is root,
+		// it will be root, <version>.root or root.<checksum>.
+		notFound, ok := err.(store.ErrMetaNotFound)
+		isRoot, _ := regexp.MatchString(`\.?`+data.CanonicalRootRole.String()+`\.?`, notFound.Resource)
+		if ok && isRoot {
+			return nil, nil, ErrRepositoryNotExist{
+				remote: options.RemoteStore.Location(),
+				gun:    options.GUN,
+			}
+		}
+		return nil, nil, err
+	}
+	warnRolesNearExpiry(repo)
+	return repo, invalid, nil
+}

vendor/github.com/theupdateframework/notary/cryptoservice/crypto_service.go

@@ -21,6 +21,9 @@ var (
 	// ErrRootKeyNotEncrypted is returned if a root key being imported is
 	// unencrypted
 	ErrRootKeyNotEncrypted = errors.New("only encrypted root keys may be imported")
+
+	// EmptyService is an empty crypto service
+	EmptyService = NewCryptoService()
 )
 
 // CryptoService implements Sign and Create, holding a specific GUN and keystore to

vendor/github.com/theupdateframework/notary/fips.go

@@ -3,7 +3,7 @@ package notary
 import (
 	"crypto"
 	// Need to import md5 so can test availability.
-	_ "crypto/md5"
+	_ "crypto/md5" // #nosec
 )
 
 // FIPSEnabled returns true if running in FIPS mode.

vendor/github.com/theupdateframework/notary/go.mod

@@ -0,0 +1,58 @@
+module github.com/theupdateframework/notary
+
+go 1.12
+
+require (
+	github.com/BurntSushi/toml v0.3.1 // indirect
+	github.com/Shopify/logrus-bugsnag v0.0.0-20170309145241-6dbc35f2c30d
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bitly/go-simplejson v0.5.0 // indirect
+	github.com/bugsnag/bugsnag-go v1.0.5
+	github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b // indirect
+	github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 // indirect
+	github.com/cloudflare/cfssl v0.0.0-20180223231731-4e2dcbde5004 // 1.3.1
+	github.com/denisenkom/go-mssqldb v0.0.0-20191128021309-1d7a30a10f73 // indirect
+	github.com/docker/distribution v2.7.1+incompatible
+	github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c
+	github.com/docker/go-connections v0.4.0
+	github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916 // indirect
+	github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
+	github.com/dvsekhvalnov/jose2go v0.0.0-20170216131308-f21a8cedbbae
+	github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 // indirect
+	github.com/go-sql-driver/mysql v1.5.0
+	github.com/gogo/protobuf v1.0.0 // indirect
+	github.com/golang/protobuf v1.3.4
+	github.com/google/certificate-transparency-go v1.0.10-0.20180222191210-5ab67e519c93 // indirect
+	github.com/gorilla/mux v1.7.0
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/jinzhu/gorm v0.0.0-20170222002820-5409931a1bb8
+	github.com/jinzhu/inflection v0.0.0-20170102125226-1c35d901db3d // indirect
+	github.com/jinzhu/now v1.1.1 // indirect
+	github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 // indirect
+	github.com/lib/pq v1.9.0
+	github.com/magiconair/properties v1.5.3 // indirect
+	github.com/mattn/go-sqlite3 v1.6.0
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/miekg/pkcs11 v1.0.3
+	github.com/mitchellh/mapstructure v1.0.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.1 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/prometheus/client_golang v0.9.0-pre1.0.20180209125602-c332b6f63c06
+	github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5 // indirect
+	github.com/prometheus/common v0.0.0-20180110214958-89604d197083 // indirect
+	github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7 // indirect
+	github.com/sirupsen/logrus v1.8.1
+	github.com/spf13/cast v0.0.0-20150508191742-4d07383ffe94 // indirect
+	github.com/spf13/cobra v0.0.1
+	github.com/spf13/jwalterweatherman v0.0.0-20141219030609-3d60171a6431 // indirect
+	github.com/spf13/pflag v1.0.0 // indirect
+	github.com/spf13/viper v0.0.0-20150530192845-be5ff3e4840c
+	github.com/stretchr/testify v1.5.1
+	golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9
+	golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3
+	golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e
+	golang.org/x/term v0.0.0-20201117132131-f5c789dd3221
+	google.golang.org/grpc v1.0.5
+	gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.1
+)

vendor/github.com/theupdateframework/notary/passphrase/passphrase.go

@@ -12,7 +12,7 @@ import (
 	"strings"
 
 	"github.com/theupdateframework/notary"
-	"golang.org/x/crypto/ssh/terminal"
+	"golang.org/x/term"
 )
 
 const (
@@ -49,7 +49,7 @@ var (
 // Upon successful passphrase retrievals, the passphrase will be cached such that
 // subsequent prompts will produce the same passphrase.
 func PromptRetriever() notary.PassRetriever {
-	if !terminal.IsTerminal(int(os.Stdin.Fd())) {
+	if !term.IsTerminal(int(os.Stdin.Fd())) {
 		return func(string, string, bool, int) (string, bool, error) {
 			return "", false, ErrNoInput
 		}
@@ -200,8 +200,8 @@ func GetPassphrase(in *bufio.Reader) ([]byte, error) {
 		err        error
 	)
 
-	if terminal.IsTerminal(int(os.Stdin.Fd())) {
-		passphrase, err = terminal.ReadPassword(int(os.Stdin.Fd()))
+	if term.IsTerminal(int(os.Stdin.Fd())) {
+		passphrase, err = term.ReadPassword(int(os.Stdin.Fd()))
 	} else {
 		passphrase, err = in.ReadBytes('\n')
 	}

vendor/github.com/theupdateframework/notary/storage/filestore.go

@@ -137,14 +137,16 @@ func (f *FilesystemStore) GetSized(name string, size int64) ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
-	file, err := os.OpenFile(p, os.O_RDONLY, notary.PrivNoExecPerms)
+	file, err := os.Open(p)
 	if err != nil {
 		if os.IsNotExist(err) {
 			err = ErrMetaNotFound{Resource: name}
 		}
 		return nil, err
 	}
-	defer file.Close()
+	defer func() {
+		_ = file.Close()
+	}()
 
 	if size == NoSizeLimit {
 		size = notary.MaxDownloadSize

vendor/github.com/theupdateframework/notary/storage/httpstore.go

@@ -111,6 +111,18 @@ type HTTPStore struct {
 	roundTrip     http.RoundTripper
 }
 
+// NewNotaryServerStore returns a new HTTPStore against a URL which should represent a notary
+// server
+func NewNotaryServerStore(serverURL string, gun data.GUN, roundTrip http.RoundTripper) (RemoteStore, error) {
+	return NewHTTPStore(
+		serverURL+"/v2/"+gun.String()+"/_trust/tuf/",
+		"",
+		"json",
+		"key",
+		roundTrip,
+	)
+}
+
 // NewHTTPStore initializes a new store against a URL and a number of configuration options.
 //
 // In case of a nil `roundTrip`, a default offline store is used instead.
@@ -363,5 +375,5 @@ func (s HTTPStore) RotateKey(role data.RoleName) ([]byte, error) {
 
 // Location returns a human readable name for the storage location
 func (s HTTPStore) Location() string {
-	return s.baseURL.String()
+	return s.baseURL.Host
 }

vendor/github.com/theupdateframework/notary/storage/interfaces.go

@@ -15,6 +15,7 @@ type MetadataStore interface {
 	SetMulti(map[string][]byte) error
 	RemoveAll() error
 	Remove(name string) error
+	Location() string
 }
 
 // PublicKeyStore must be implemented by a key service

vendor/github.com/theupdateframework/notary/trustmanager/yubikey/pkcs11_darwin.go

@@ -6,4 +6,6 @@ var possiblePkcs11Libs = []string{
 	"/usr/local/lib/libykcs11.dylib",
 	"/usr/local/docker/lib/libykcs11.dylib",
 	"/usr/local/docker-experimental/lib/libykcs11.dylib",
+	// default location on arm64
+	"/opt/homebrew/lib/libykcs11.dylib",
 }

vendor/github.com/theupdateframework/notary/tuf/builder.go

@@ -359,7 +359,7 @@ func (rb *repoBuilder) GenerateSnapshot(prev *data.SignedSnapshot) ([]byte, int,
 
 	// loadedNotChecksummed should currently contain the root awaiting checksumming,
 	// since it has to have been loaded.  Since the snapshot was generated using
-	// the root and targets data (there may not be any) that that have been loaded,
+	// the root and targets data (there may not be any) that have been loaded,
 	// remove all of them from rb.loadedNotChecksummed
 	for tgtName := range rb.repo.Targets {
 		delete(rb.loadedNotChecksummed, data.RoleName(tgtName))

vendor/github.com/theupdateframework/notary/tuf/data/keys.go

@@ -12,9 +12,9 @@ import (
 	"io"
 	"math/big"
 
-	"github.com/agl/ed25519"
 	"github.com/docker/go/canonical/json"
 	"github.com/sirupsen/logrus"
+	"golang.org/x/crypto/ed25519"
 )
 
 // PublicKey is the necessary interface for public keys
@@ -484,9 +484,10 @@ func (k RSAPrivateKey) Sign(rand io.Reader, msg []byte, opts crypto.SignerOpts)
 
 // Sign creates an ed25519 signature
 func (k ED25519PrivateKey) Sign(rand io.Reader, msg []byte, opts crypto.SignerOpts) (signature []byte, err error) {
-	priv := [ed25519.PrivateKeySize]byte{}
-	copy(priv[:], k.private[ed25519.PublicKeySize:])
-	return ed25519.Sign(&priv, msg)[:], nil
+	priv := make([]byte, ed25519.PrivateKeySize)
+	// The ed25519 key is serialized as public key then private key, so just use private key here.
+	copy(priv, k.private[ed25519.PublicKeySize:])
+	return ed25519.Sign(ed25519.PrivateKey(priv), msg)[:], nil
 }
 
 // Sign on an UnknownPrivateKey raises an error because the client does not

vendor/github.com/theupdateframework/notary/tuf/data/roles.go

@@ -55,7 +55,7 @@ func (e ErrInvalidRole) Error() string {
 
 // ValidRole only determines the name is semantically
 // correct. For target delegated roles, it does NOT check
-// the the appropriate parent roles exist.
+// the appropriate parent roles exist.
 func ValidRole(name RoleName) bool {
 	if IsDelegation(name) {
 		return true

vendor/github.com/theupdateframework/notary/tuf/data/snapshot.go

@@ -59,7 +59,7 @@ func IsValidSnapshotStructure(s Snapshot) error {
 	return nil
 }
 
-// NewSnapshot initilizes a SignedSnapshot with a given top level root
+// NewSnapshot initializes a SignedSnapshot with a given top level root
 // and targets objects
 func NewSnapshot(root *Signed, targets *Signed) (*SignedSnapshot, error) {
 	logrus.Debug("generating new snapshot...")

vendor/github.com/theupdateframework/notary/tuf/data/targets.go

@@ -54,7 +54,7 @@ func isValidTargetsStructure(t Targets, roleName RoleName) error {
 	return nil
 }
 
-// NewTargets intiializes a new empty SignedTargets object
+// NewTargets initializes a new empty SignedTargets object
 func NewTargets() *SignedTargets {
 	return &SignedTargets{
 		Signatures: make([]Signature, 0),

vendor/github.com/theupdateframework/notary/tuf/data/types.go

@@ -186,7 +186,7 @@ type FileMeta struct {
 
 // Equals returns true if the other FileMeta object is equivalent to this one
 func (f FileMeta) Equals(o FileMeta) bool {
-	if o.Length != f.Length || len(f.Hashes) != len(f.Hashes) {
+	if o.Length != f.Length || len(o.Hashes) != len(f.Hashes) {
 		return false
 	}
 	if f.Custom == nil && o.Custom != nil || f.Custom != nil && o.Custom == nil {

vendor/github.com/theupdateframework/notary/tuf/signed/interface.go

@@ -39,7 +39,7 @@ type CryptoService interface {
 	KeyService
 }
 
-// Verifier defines an interface for verfying signatures. An implementer
+// Verifier defines an interface for verifying signatures. An implementer
 // of this interface should verify signatures for one and only one
 // signing scheme.
 type Verifier interface {

vendor/github.com/theupdateframework/notary/tuf/signed/sign.go

@@ -87,7 +87,8 @@ func Sign(service CryptoService, s *data.Signed, signingKeys []data.PublicKey,
 		})
 	}
 
-	for _, sig := range s.Signatures {
+	for i := range s.Signatures {
+		sig := s.Signatures[i]
 		if _, ok := signingKeyIDs[sig.KeyID]; ok {
 			// key is in the set of key IDs for which a signature has been created
 			continue

vendor/github.com/theupdateframework/notary/tuf/signed/verifiers.go

@@ -10,9 +10,9 @@ import (
 	"fmt"
 	"math/big"
 
-	"github.com/agl/ed25519"
 	"github.com/sirupsen/logrus"
 	"github.com/theupdateframework/notary/tuf/data"
+	"golang.org/x/crypto/ed25519"
 )
 
 const (
@@ -39,26 +39,26 @@ func (v Ed25519Verifier) Verify(key data.PublicKey, sig []byte, msg []byte) erro
 	if key.Algorithm() != data.ED25519Key {
 		return ErrInvalidKeyType{}
 	}
-	var sigBytes [ed25519.SignatureSize]byte
+	sigBytes := make([]byte, ed25519.SignatureSize)
 	if len(sig) != ed25519.SignatureSize {
 		logrus.Debugf("signature length is incorrect, must be %d, was %d.", ed25519.SignatureSize, len(sig))
 		return ErrInvalid
 	}
-	copy(sigBytes[:], sig)
+	copy(sigBytes, sig)
 
-	var keyBytes [ed25519.PublicKeySize]byte
+	keyBytes := make([]byte, ed25519.PublicKeySize)
 	pub := key.Public()
 	if len(pub) != ed25519.PublicKeySize {
 		logrus.Errorf("public key is incorrect size, must be %d, was %d.", ed25519.PublicKeySize, len(pub))
 		return ErrInvalidKeyLength{msg: fmt.Sprintf("ed25519 public key must be %d bytes.", ed25519.PublicKeySize)}
 	}
-	n := copy(keyBytes[:], key.Public())
+	n := copy(keyBytes, key.Public())
 	if n < ed25519.PublicKeySize {
 		logrus.Errorf("failed to copy the key, must have %d bytes, copied %d bytes.", ed25519.PublicKeySize, n)
 		return ErrInvalid
 	}
 
-	if !ed25519.Verify(&keyBytes, msg, &sigBytes) {
+	if !ed25519.Verify(ed25519.PublicKey(keyBytes), msg, sigBytes) {
 		logrus.Debugf("failed ed25519 verification")
 		return ErrInvalid
 	}

vendor/github.com/theupdateframework/notary/tuf/utils/pkcs8.go

@@ -39,7 +39,7 @@ import (
 	"crypto/elliptic"
 	"crypto/rand"
 	"crypto/rsa"
-	"crypto/sha1"
+	"crypto/sha1" // #nosec
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"encoding/asn1"

vendor/github.com/theupdateframework/notary/tuf/utils/utils.go

@@ -30,7 +30,7 @@ func RoleNameSliceContains(ss []data.RoleName, s data.RoleName) bool {
 	return false
 }
 
-// RoleNameSliceRemove removes the the given RoleName from the slice, returning a new slice
+// RoleNameSliceRemove removes the given RoleName from the slice, returning a new slice
 func RoleNameSliceRemove(ss []data.RoleName, s data.RoleName) []data.RoleName {
 	res := []data.RoleName{}
 	for _, v := range ss {

vendor/github.com/theupdateframework/notary/tuf/utils/x509.go

@@ -16,10 +16,10 @@ import (
 	"math/big"
 	"time"
 
-	"github.com/agl/ed25519"
 	"github.com/sirupsen/logrus"
 	"github.com/theupdateframework/notary"
 	"github.com/theupdateframework/notary/tuf/data"
+	"golang.org/x/crypto/ed25519"
 )
 
 // CanonicalKeyID returns the ID of the public bytes version of a TUF key.

vendor/github.com/theupdateframework/notary/vendor.conf

@@ -1,59 +0,0 @@
-github.com/Shopify/logrus-bugsnag   6dbc35f2c30d1e37549f9673dd07912452ab28a5
-github.com/sirupsen/logrus          f006c2ac4710855cf0f916dd6b77acf6b048dc6e  # v1.0.3
-github.com/agl/ed25519              278e1ec8e8a6e017cd07577924d6766039146ced
-github.com/bugsnag/bugsnag-go       13fd6b8acda029830ef9904df6b63be0a83369d0
-github.com/bugsnag/panicwrap	    e2c28503fcd0675329da73bf48b33404db873782
-github.com/bugsnag/osext	    0dd3f918b21bec95ace9dc86c7e70266cfc5c702
-github.com/docker/distribution      edc3ab29cdff8694dd6feb85cfeb4b5f1b38ed9c
-github.com/opencontainers/go-digest a6d0ee40d4207ea02364bd3b9e8e77b9159ba1eb
-github.com/docker/go-connections    7395e3f8aa162843a74ed6d48e79627d9792ac55
-github.com/docker/go                d30aec9fd63c35133f8f79c3412ad91a3b08be06
-github.com/dvsekhvalnov/jose2go     f21a8cedbbae609f623613ec8f81125c243212e6   # v1.3
-github.com/go-sql-driver/mysql      a0583e0143b1624142adab07e0e97fe106d99561   # v1.3
-github.com/gorilla/mux              53c1911da2b537f792e7cafcb446b05ffe33b996   # v1.6.1
-github.com/jinzhu/gorm              5409931a1bb87e484d68d649af9367c207713ea2
-github.com/jinzhu/inflection        1c35d901db3da928c72a72d8458480cc9ade058f
-github.com/lib/pq                   0dad96c0b94f8dee039aa40467f767467392a0af
-github.com/mattn/go-sqlite3         6c771bb9887719704b210e87e934f08be014bdb1   # v1.6.0
-github.com/miekg/pkcs11             5f6e0d0dad6f472df908c8e968a98ef00c9224bb
-github.com/prometheus/client_golang 449ccefff16c8e2b7229f6be1921ba22f62461fe
-github.com/prometheus/client_model  fa8ad6fec33561be4280a8f0514318c79d7f6cb6   # model-0.0.2-12-gfa8ad6f
-github.com/prometheus/procfs	    b1afdc266f54247f5dc725544f5d351a8661f502
-github.com/prometheus/common	    4fdc91a58c9d3696b982e8a680f4997403132d44
-github.com/golang/protobuf          c3cefd437628a0b7d31b34fe44b3a7a540e98527
-github.com/spf13/cobra              7b2c5ac9fc04fc5efafb60700713d4fa609b777b   # v0.0.1
-github.com/spf13/viper              be5ff3e4840cf692388bde7a057595a474ef379e
-golang.org/x/crypto                 76eec36fa14229c4b25bb894c2d0e591527af429
-golang.org/x/net                    6a513affb38dc9788b449d59ffed099b8de18fa0
-golang.org/x/sys                    314a259e304ff91bd6985da2a7149bbf91237993
-google.golang.org/grpc              708a7f9f3283aa2d4f6132d287d78683babe55c8   # v1.0.5
-github.com/pkg/errors 		    839d9e913e063e28dfd0e6c7b7512793e0a48be9
-
-github.com/spf13/pflag		    e57e3eeb33f795204c1ca35f56c44f83227c6e66  # v1.0.0
-github.com/spf13/cast		    4d07383ffe94b5e5a6fa3af9211374a4507a0184
-gopkg.in/yaml.v2 		    5420a8b6744d3b0345ab293f6fcba19c978f1183  # v2.2.1
-gopkg.in/fatih/pool.v2		    cba550ebf9bce999a02e963296d4bc7a486cb715
-github.com/gorilla/context	    14f550f51af52180c2eefed15e5fd18d63c0a64a  # unused
-github.com/spf13/jwalterweatherman  3d60171a64319ef63c78bd45bd60e6eab1e75f8b
-github.com/mitchellh/mapstructure   2caf8efc93669b6c43e0441cdc6aed17546c96f3
-github.com/magiconair/properties    624009598839a9432bd97bb75552389422357723  # v1.5.3
-github.com/kr/text		    6807e777504f54ad073ecef66747de158294b639
-github.com/kr/pretty		    bc9499caa0f45ee5edb2f0209fbd61fbf3d9018f  # go.weekly.2011-12-22-18-gbc9499c
-github.com/hailocab/go-hostpool	    e80d13ce29ede4452c43dea11e79b9bc8a15b478
-github.com/docker/libtrust	    aabc10ec26b754e797f9028f4589c5b7bd90dc20
-github.com/beorn7/perks		    b965b613227fddccbfffe13eae360ed3fa822f8d
-github.com/BurntSushi/toml	    a368813c5e648fee92e5f6c30e3944ff9d5e8895
-
-github.com/matttproud/golang_protobuf_extensions	d0c3fe89de86839aecf2e0579c40ba3bb336a453
-github.com/inconshreveable/mousetrap			76626ae9c91c4f2a10f34cad8ce83ea42c93bb75
-
-gopkg.in/dancannon/gorethink.v3        e324d6ad938205da6c1e8a0179dc97a5b1a92185  https://github.com/docker/gorethink # v3.0.0-logrus
-# dependencies of gorethink.v3
-gopkg.in/gorethink/gorethink.v2        ac5be4ae8538d44ae8843b97fc9f90860cb48a85  https://github.com/docker/gorethink # v2.2.2-logrus
-github.com/cenk/backoff             32cd0c5b3aef12c76ed64aaf678f6c79736be7dc  # v1.0.0
-
-# Testing requirements
-github.com/stretchr/testify                     089c7181b8c728499929ff09b62d3fdd8df8adff
-github.com/cloudflare/cfssl                     4e2dcbde500472449917533851bf4bae9bdff562 # v1.3.1
-github.com/google/certificate-transparency-go   5ab67e519c93568ac3ee50fd6772a5bcf8aa460d
-github.com/gogo/protobuf                        1adfc126b41513cc696b209667c8656ea7aac67c # v1.0.0