toolshed/docker-cli
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge component 'engine' from git@github.com:moby/moby master

Browse Source
This commit is contained in:
Andrew Hsu
2017-06-24 00:04:09 +00:00
parent 22a8e5a387 5b78ab86a3
commit 4302a48fed
975 changed files with 30673 additions and 53098 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
components/engine/CHANGELOG.md
View File

@ -5,6 +5,92 @@ information on the list of deprecated flags and APIs please have a look at
https://docs.docker.com/engine/deprecated/ where target removal dates can also
be found.
## 17.05.0-ce (2017-05-04)
### Builder
+ Add multi-stage build support [#31257](https://github.com/docker/docker/pull/31257) [#32063](https://github.com/docker/docker/pull/32063)
+ Allow using build-time args (`ARG`) in `FROM` [#31352](https://github.com/docker/docker/pull/31352)
+ Add an option for specifying build target [#32496](https://github.com/docker/docker/pull/32496)
* Accept `-f -` to read Dockerfile from `stdin`, but use local context for building [#31236](https://github.com/docker/docker/pull/31236)
* The values of default build time arguments (e.g `HTTP_PROXY`) are no longer displayed in docker image history unless a corresponding `ARG` instruction is written in the Dockerfile. [#31584](https://github.com/docker/docker/pull/31584)
- Fix setting command if a custom shell is used in a parent image [#32236](https://github.com/docker/docker/pull/32236)
- Fix `docker build --label` when the label includes single quotes and a space [#31750](https://github.com/docker/docker/pull/31750)
### Client
* Add `--mount` flag to `docker run` and `docker create` [#32251](https://github.com/docker/docker/pull/32251)
* Add `--type=secret` to `docker inspect` [#32124](https://github.com/docker/docker/pull/32124)
* Add `--format` option to `docker secret ls` [#31552](https://github.com/docker/docker/pull/31552)
* Add `--filter` option to `docker secret ls` [#30810](https://github.com/docker/docker/pull/30810)
* Add `--filter scope=<swarm|local>` to `docker network ls` [#31529](https://github.com/docker/docker/pull/31529)
* Add `--cpus` support to `docker update` [#31148](https://github.com/docker/docker/pull/31148)
* Add label filter to `docker system prune` and other `prune` commands [#30740](https://github.com/docker/docker/pull/30740)
* `docker stack rm` now accepts multiple stacks as input [#32110](https://github.com/docker/docker/pull/32110)
* Improve `docker version --format` option when the client has downgraded the API version [#31022](https://github.com/docker/docker/pull/31022)
* Prompt when using an encrypted client certificate to connect to a docker daemon [#31364](https://github.com/docker/docker/pull/31364)
* Display created tags on successful `docker build` [#32077](https://github.com/docker/docker/pull/32077)
* Cleanup compose convert error messages [#32087](https://github.com/moby/moby/pull/32087)
### Contrib
+ Add support for building docker debs for Ubuntu 17.04 Zesty on amd64 [#32435](https://github.com/docker/docker/pull/32435)
### Daemon
- Fix `--api-cors-header` being ignored if `--api-enable-cors` is not set [#32174](https://github.com/docker/docker/pull/32174)
- Cleanup docker tmp dir on start [#31741](https://github.com/docker/docker/pull/31741)
- Deprecate `--graph` flag in favor or `--data-root` [#28696](https://github.com/docker/docker/pull/28696)
### Logging
+ Add support for logging driver plugins [#28403](https://github.com/docker/docker/pull/28403)
* Add support for showing logs of individual tasks to `docker service logs`, and add `/task/{id}/logs` REST endpoint [#32015](https://github.com/docker/docker/pull/32015)
* Add `--log-opt env-regex` option to match environment variables using a regular expression [#27565](https://github.com/docker/docker/pull/27565)
### Networking
+ Allow user to replace, and customize the ingress network [#31714](https://github.com/docker/docker/pull/31714)
- Fix UDP traffic in containers not working after the container is restarted [#32505](https://github.com/docker/docker/pull/32505)
- Fix files being written to `/var/lib/docker` if a different data-root is set [#32505](https://github.com/docker/docker/pull/32505)
### Runtime
- Ensure health probe is stopped when a container exits [#32274](https://github.com/docker/docker/pull/32274)
### Swarm Mode
+ Add update/rollback order for services (`--update-order` / `--rollback-order`) [#30261](https://github.com/docker/docker/pull/30261)
+ Add support for synchronous `service create` and `service update` [#31144](https://github.com/docker/docker/pull/31144)
+ Add support for "grace periods" on healthchecks through the `HEALTHCHECK --start-period` and `--health-start-period` flag to
`docker service create`, `docker service update`, `docker create`, and `docker run` to support containers with an initial startup
time [#28938](https://github.com/docker/docker/pull/28938)
* `docker service create` now omits fields that are not specified by the user, when possible. This will allow defaults to be applied inside the manager [#32284](https://github.com/docker/docker/pull/32284)
* `docker service inspect` now shows default values for fields that are not specified by the user [#32284](https://github.com/docker/docker/pull/32284)
* Move `docker service logs` out of experimental [#32462](https://github.com/docker/docker/pull/32462)
* Add support for Credential Spec and SELinux to services to the API [#32339](https://github.com/docker/docker/pull/32339)
* Add `--entrypoint` flag to `docker service create` and `docker service update` [#29228](https://github.com/docker/docker/pull/29228)
* Add `--network-add` and `--network-rm` to `docker service update` [#32062](https://github.com/docker/docker/pull/32062)
* Add `--credential-spec` flag to `docker service create` and `docker service update` [#32339](https://github.com/docker/docker/pull/32339)
* Add `--filter mode=<global|replicated>` to `docker service ls` [#31538](https://github.com/docker/docker/pull/31538)
* Resolve network IDs on the client side, instead of in the daemon when creating services [#32062](https://github.com/docker/docker/pull/32062)
* Add `--format` option to `docker node ls` [#30424](https://github.com/docker/docker/pull/30424)
* Add `--prune` option to `docker stack deploy` to remove services that are no longer defined in the docker-compose file [#31302](https://github.com/docker/docker/pull/31302)
* Add `PORTS` column for `docker service ls` when using `ingress` mode [#30813](https://github.com/docker/docker/pull/30813)
- Fix unnescessary re-deploying of tasks when environment-variables are used [#32364](https://github.com/docker/docker/pull/32364)
- Fix `docker stack deploy` not supporting `endpoint_mode` when deploying from a docker compose file [#32333](https://github.com/docker/docker/pull/32333)
- Proceed with startup if cluster component cannot be created to allow recovering from a broken swarm setup [#31631](https://github.com/docker/docker/pull/31631)
### Security
* Allow setting SELinux type or MCS labels when using `--ipc=container:` or `--ipc=host` [#30652](https://github.com/docker/docker/pull/30652)
### Deprecation
- Deprecate `--api-enable-cors` daemon flag. This flag was marked deprecated in Docker 1.6.0 but not listed in deprecated features [#32352](https://github.com/docker/docker/pull/32352)
- Remove Ubuntu 12.04 (Precise Pangolin) as supported platform. Ubuntu 12.04 is EOL, and no longer receives updates [#32520](https://github.com/docker/docker/pull/32520)
## 17.04.0-ce (2017-04-05)
### Builder

30
components/engine/Dockerfile
View File

@ -40,7 +40,6 @@ RUN apt-get update && apt-get install -y \
bsdmainutils \
btrfs-tools \
build-essential \
clang \
cmake \
createrepo \
curl \
@ -52,7 +51,6 @@ RUN apt-get update && apt-get install -y \
less \
libapparmor-dev \
libcap-dev \
libltdl-dev \
libnl-3-dev \
libprotobuf-c0-dev \
libprotobuf-dev \
@ -90,17 +88,6 @@ RUN cd /usr/local/lvm2 \
&& make install_device-mapper
# See https://git.fedorahosted.org/cgit/lvm2.git/tree/INSTALL
# Configure the container for OSX cross compilation
ENV OSX_SDK MacOSX10.11.sdk
ENV OSX_CROSS_COMMIT a9317c18a3a457ca0a657f08cc4d0d43c6cf8953
RUN set -x \
&& export OSXCROSS_PATH="/osxcross" \
&& git clone https://github.com/tpoechtrager/osxcross.git $OSXCROSS_PATH \
&& ( cd $OSXCROSS_PATH && git checkout -q $OSX_CROSS_COMMIT) \
&& curl -sSL https://s3.dockerproject.org/darwin/v2/${OSX_SDK}.tar.xz -o "${OSXCROSS_PATH}/tarballs/${OSX_SDK}.tar.xz" \
&& UNATTENDED=yes OSX_VERSION_MIN=10.6 ${OSXCROSS_PATH}/build.sh
ENV PATH /osxcross/target/bin:$PATH
# Install seccomp: the version shipped upstream is too old
ENV SECCOMP_VERSION 2.3.2
RUN set -x \
@ -127,14 +114,6 @@ RUN curl -fsSL "https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz" \
ENV PATH /go/bin:/usr/local/go/bin:$PATH
ENV GOPATH /go
# Compile Go for cross compilation
ENV DOCKER_CROSSPLATFORMS \
linux/386 linux/arm \
darwin/amd64 \
freebsd/amd64 freebsd/386 freebsd/arm \
windows/amd64 windows/386 \
solaris/amd64
# Dependency for golint
ENV GO_TOOLS_COMMIT 823804e1ae08dbb14eb807afc7db9993bc9e3cc3
RUN git clone https://github.com/golang/tools.git /go/src/golang.org/x/tools \
@ -186,7 +165,7 @@ RUN set -x \
&& rm -rf "$GOPATH"
# Get the "docker-py" source so we can run their integration tests
ENV DOCKER_PY_COMMIT 4a08d04aef0595322e1b5ac7c52f28a931da85a5
ENV DOCKER_PY_COMMIT a962578e515185cf06506050b2200c0b81aa84ef
# To run integration tests docker-pycreds is required.
# Before running the integration tests conftest.py is
# loaded which results in loads auth.py that
@ -215,16 +194,13 @@ RUN useradd --create-home --gid docker unprivilegeduser
VOLUME /var/lib/docker
WORKDIR /go/src/github.com/docker/docker
ENV DOCKER_BUILDTAGS apparmor pkcs11 seccomp selinux
ENV DOCKER_BUILDTAGS apparmor seccomp selinux
# Let us use a .bashrc file
RUN ln -sfv $PWD/.bashrc ~/.bashrc
# Add integration helps to bashrc
RUN echo "source $PWD/hack/make/.integration-test-helpers" >> /etc/bash.bashrc
# Register Docker's bash completion.
RUN ln -sv $PWD/contrib/completion/bash/docker /etc/bash_completion.d/docker
# Get useful and necessary Hub images so we can "docker load" locally instead of pulling
COPY contrib/download-frozen-image-v2.sh /go/src/github.com/docker/docker/contrib/
RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \
@ -238,7 +214,7 @@ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \
# Please edit hack/dockerfile/install-binaries.sh to update them.
COPY hack/dockerfile/binaries-commits /tmp/binaries-commits
COPY hack/dockerfile/install-binaries.sh /tmp/install-binaries.sh
RUN /tmp/install-binaries.sh tomlv vndr runc containerd tini proxy bindata dockercli
RUN /tmp/install-binaries.sh tomlv vndr runc containerd tini proxy dockercli
ENV PATH=/usr/local/cli:$PATH
# Wrap all commands in the "docker-in-docker" script to allow nested containers

5
components/engine/Dockerfile.aarch64
View File

@ -37,7 +37,6 @@ RUN apt-get update && apt-get install -y \
libapparmor-dev \
libc6-dev \
libcap-dev \
libltdl-dev \
libsystemd-dev \
libyaml-dev \
mercurial \
@ -142,7 +141,7 @@ RUN set -x \
&& rm -rf "$GOPATH"
# Get the "docker-py" source so we can run their integration tests
ENV DOCKER_PY_COMMIT 4a08d04aef0595322e1b5ac7c52f28a931da85a5
ENV DOCKER_PY_COMMIT a962578e515185cf06506050b2200c0b81aa84ef
# Before running the integration tests conftest.py is
# loaded which results in loads auth.py that
# imports the docker-pycreds module.
@ -171,7 +170,7 @@ RUN useradd --create-home --gid docker unprivilegeduser
VOLUME /var/lib/docker
WORKDIR /go/src/github.com/docker/docker
ENV DOCKER_BUILDTAGS apparmor pkcs11 seccomp selinux
ENV DOCKER_BUILDTAGS apparmor seccomp selinux
# Let us use a .bashrc file
RUN ln -sfv $PWD/.bashrc ~/.bashrc

5
components/engine/Dockerfile.armhf
View File

@ -39,7 +39,6 @@ RUN apt-get update && apt-get install -y \
net-tools \
libapparmor-dev \
libcap-dev \
libltdl-dev \
libsystemd-journal-dev \
libtool \
mercurial \
@ -137,7 +136,7 @@ RUN set -x \
&& rm -rf "$GOPATH"
# Get the "docker-py" source so we can run their integration tests
ENV DOCKER_PY_COMMIT e2655f658408f9ad1f62abdef3eb6ed43c0cf324
ENV DOCKER_PY_COMMIT a962578e515185cf06506050b2200c0b81aa84ef
RUN git clone https://github.com/docker/docker-py.git /docker-py \
&& cd /docker-py \
&& git checkout -q $DOCKER_PY_COMMIT \
@ -152,7 +151,7 @@ RUN useradd --create-home --gid docker unprivilegeduser
VOLUME /var/lib/docker
WORKDIR /go/src/github.com/docker/docker
ENV DOCKER_BUILDTAGS apparmor pkcs11 seccomp selinux
ENV DOCKER_BUILDTAGS apparmor seccomp selinux
# Let us use a .bashrc file
RUN ln -sfv $PWD/.bashrc ~/.bashrc

5
components/engine/Dockerfile.ppc64le
View File

@ -40,7 +40,6 @@ RUN apt-get update && apt-get install -y \
net-tools \
libapparmor-dev \
libcap-dev \
libltdl-dev \
libsystemd-journal-dev \
libtool \
mercurial \
@ -143,7 +142,7 @@ RUN set -x \
&& rm -rf "$GOPATH"
# Get the "docker-py" source so we can run their integration tests
ENV DOCKER_PY_COMMIT e2655f658408f9ad1f62abdef3eb6ed43c0cf324
ENV DOCKER_PY_COMMIT a962578e515185cf06506050b2200c0b81aa84ef
RUN git clone https://github.com/docker/docker-py.git /docker-py \
&& cd /docker-py \
&& git checkout -q $DOCKER_PY_COMMIT \
@ -158,7 +157,7 @@ RUN useradd --create-home --gid docker unprivilegeduser
VOLUME /var/lib/docker
WORKDIR /go/src/github.com/docker/docker
ENV DOCKER_BUILDTAGS apparmor pkcs11 seccomp selinux
ENV DOCKER_BUILDTAGS apparmor seccomp selinux
# Let us use a .bashrc file
RUN ln -sfv $PWD/.bashrc ~/.bashrc

3
components/engine/Dockerfile.s390x
View File

@ -36,7 +36,6 @@ RUN apt-get update && apt-get install -y \
net-tools \
libapparmor-dev \
libcap-dev \
libltdl-dev \
libsystemd-journal-dev \
libtool \
mercurial \
@ -136,7 +135,7 @@ RUN set -x \
&& rm -rf "$GOPATH"
# Get the "docker-py" source so we can run their integration tests
ENV DOCKER_PY_COMMIT e2655f658408f9ad1f62abdef3eb6ed43c0cf324
ENV DOCKER_PY_COMMIT a962578e515185cf06506050b2200c0b81aa84ef
RUN git clone https://github.com/docker/docker-py.git /docker-py \
&& cd /docker-py \
&& git checkout -q $DOCKER_PY_COMMIT \

1
components/engine/Dockerfile.solaris
View File

@ -15,6 +15,5 @@ RUN pkg install --accept \
developer/gcc-*
ENV GOPATH /go/:/usr/lib/gocode/1.5/
ENV DOCKER_CROSSPLATFORMS solaris/amd64
WORKDIR /go/src/github.com/docker/docker
COPY . /go/src/github.com/docker/docker

11
components/engine/Makefile
View File

@ -17,7 +17,7 @@ export DOCKER_GITCOMMIT
# to allow things like `make KEEPBUNDLE=1 binary` easily
# `project/PACKAGERS.md` have some limited documentation of some of these
DOCKER_ENVS := \
$(if $(DOCKER_CROSSPLATFORMS), -e DOCKER_CROSSPLATFORMS) \
-e DOCKER_CROSSPLATFORMS \
-e BUILD_APT_MIRROR \
-e BUILDFLAGS \
-e KEEPBUNDLE \
@ -134,12 +134,6 @@ init-go-pkg-cache:
install: ## install the linux binaries
KEEPBUNDLE=1 hack/make.sh install-binary
manpages: ## Generate man pages from go source and markdown
docker build ${DOCKER_BUILD_ARGS} -t docker-manpage-dev -f "man/$(DOCKERFILE)" ./man
docker run --rm \
-v $(PWD):/go/src/github.com/docker/docker/ \
docker-manpage-dev
rpm: build ## build the rpm packages
$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary build-rpm
@ -149,9 +143,6 @@ run: build ## run the docker daemon in a container
shell: build ## start a shell inside the build env
$(DOCKER_RUN_DOCKER) bash
yaml-docs-gen: build ## generate documentation YAML files consumed by docs repo
$(DOCKER_RUN_DOCKER) sh -c 'hack/make.sh yaml-docs-generator && ( root=$$(pwd); cd bundles/latest/yaml-docs-generator; mkdir docs; ./yaml-docs-generator --root $${root} --target $$(pwd)/docs )'
test: build ## run the unit, integration and docker-py tests
$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary cross test-unit test-integration-cli test-docker-py

4
components/engine/api/common.go
View File

@ -21,7 +21,7 @@ import (
// Common constants for daemon and client.
const (
// DefaultVersion of Current REST API
DefaultVersion string = "1.30"
DefaultVersion string = "1.31"
// NoBaseImageSpecifier is the symbol used by the FROM
// command to specify that no base image is to be used.
@ -126,7 +126,7 @@ func MatchesContentType(contentType, expectedType string) bool {
// LoadOrCreateTrustKey attempts to load the libtrust key at the given path,
// otherwise generates a new one
func LoadOrCreateTrustKey(trustKeyPath string) (libtrust.PrivateKey, error) {
err := system.MkdirAll(filepath.Dir(trustKeyPath), 0700)
err := system.MkdirAll(filepath.Dir(trustKeyPath), 0700, "")
if err != nil {
return nil, err
}

29
components/engine/api/server/backend/build/backend.go
View File

@ -4,9 +4,10 @@ import (
"fmt"
"github.com/docker/distribution/reference"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/backend"
"github.com/docker/docker/builder"
"github.com/docker/docker/builder/dockerfile"
"github.com/docker/docker/builder/fscache"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/stringid"
"github.com/pkg/errors"
@ -16,19 +17,24 @@ import (
// ImageComponent provides an interface for working with images
type ImageComponent interface {
SquashImage(from string, to string) (string, error)
TagImageWithReference(image.ID, reference.Named) error
TagImageWithReference(image.ID, string, reference.Named) error
}
// Builder defines interface for running a build
type Builder interface {
Build(context.Context, backend.BuildConfig) (*builder.Result, error)
}
// Backend provides build functionality to the API router
type Backend struct {
manager *dockerfile.BuildManager
builder Builder
fsCache *fscache.FSCache
imageComponent ImageComponent
}
// NewBackend creates a new build backend from components
func NewBackend(components ImageComponent, builderBackend builder.Backend) *Backend {
manager := dockerfile.NewBuildManager(builderBackend)
return &Backend{imageComponent: components, manager: manager}
func NewBackend(components ImageComponent, builder Builder, fsCache *fscache.FSCache) (*Backend, error) {
return &Backend{imageComponent: components, builder: builder, fsCache: fsCache}, nil
}
// Build builds an image from a Source
@ -39,7 +45,7 @@ func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string
return "", err
}
build, err := b.manager.Build(ctx, config)
build, err := b.builder.Build(ctx, config)
if err != nil {
return "", err
}
@ -57,6 +63,15 @@ func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string
return imageID, err
}
// PruneCache removes all cached build sources
func (b *Backend) PruneCache(ctx context.Context) (*types.BuildCachePruneReport, error) {
size, err := b.fsCache.Prune()
if err != nil {
return nil, errors.Wrap(err, "failed to prune build cache")
}
return &types.BuildCachePruneReport{SpaceReclaimed: size}, nil
}
func squashBuild(build *builder.Result, imageComponent ImageComponent) (string, error) {
var fromID string
if build.FromImage != nil {

9
components/engine/api/server/backend/build/tag.go
View File

@ -3,9 +3,11 @@ package build
import (
"fmt"
"io"
"runtime"
"github.com/docker/distribution/reference"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/system"
"github.com/pkg/errors"
)
@ -33,7 +35,12 @@ func NewTagger(backend ImageComponent, stdout io.Writer, names []string) (*Tagge
// TagImages creates image tags for the imageID
func (bt *Tagger) TagImages(imageID image.ID) error {
for _, rt := range bt.repoAndTags {
if err := bt.imageComponent.TagImageWithReference(imageID, rt); err != nil {
// TODO @jhowardmsft LCOW support. Will need revisiting.
platform := runtime.GOOS
if platform == "windows" && system.LCOWSupported() {
platform = "linux"
}
if err := bt.imageComponent.TagImageWithReference(imageID, platform, rt); err != nil {
return err
}
fmt.Fprintf(bt.stdout, "Successfully tagged %s\n", reference.FamiliarString(rt))

41
components/engine/api/server/httputils/errors.go
View File

@ -9,6 +9,7 @@ import (
"github.com/docker/docker/api/types/versions"
"github.com/gorilla/mux"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
)
// httpStatusError is an interface
@ -44,11 +45,17 @@ func GetHTTPErrorStatusCode(err error) int {
case inputValidationError:
statusCode = http.StatusBadRequest
default:
statusCode = statusCodeFromGRPCError(err)
if statusCode != http.StatusInternalServerError {
return statusCode
}
// FIXME: this is brittle and should not be necessary, but we still need to identify if
// there are errors falling back into this logic.
// If we need to differentiate between different possible error types,
// we should create appropriate error types that implement the httpStatusError interface.
errStr := strings.ToLower(errMsg)
for _, status := range []struct {
keyword string
code int
@ -66,6 +73,7 @@ func GetHTTPErrorStatusCode(err error) int {
{"this node", http.StatusServiceUnavailable},
{"needs to be unlocked", http.StatusServiceUnavailable},
{"certificates have expired", http.StatusServiceUnavailable},
{"repository does not exist", http.StatusNotFound},
} {
if strings.Contains(errStr, status.keyword) {
statusCode = status.code
@ -102,3 +110,36 @@ func MakeErrorHandler(err error) http.HandlerFunc {
}
}
}
// statusCodeFromGRPCError returns status code according to gRPC error
func statusCodeFromGRPCError(err error) int {
switch grpc.Code(err) {
case codes.InvalidArgument: // code 3
return http.StatusBadRequest
case codes.NotFound: // code 5
return http.StatusNotFound
case codes.AlreadyExists: // code 6
return http.StatusConflict
case codes.PermissionDenied: // code 7
return http.StatusForbidden
case codes.FailedPrecondition: // code 9
return http.StatusBadRequest
case codes.Unauthenticated: // code 16
return http.StatusUnauthorized
case codes.OutOfRange: // code 11
return http.StatusBadRequest
case codes.Unimplemented: // code 12
return http.StatusNotImplemented
case codes.Unavailable: // code 14
return http.StatusServiceUnavailable
default:
// codes.Canceled(1)
// codes.Unknown(2)
// codes.DeadlineExceeded(4)
// codes.ResourceExhausted(8)
// codes.Aborted(10)
// codes.Internal(13)
// codes.DataLoss(15)
return http.StatusInternalServerError
}
}

2
components/engine/api/server/httputils/httputils_write_json.go
View File

@ -1,5 +1,3 @@
// +build go1.7
package httputils
import (

16
components/engine/api/server/httputils/httputils_write_json_go16.go
View File

@ -1,16 +0,0 @@
// +build go1.6,!go1.7
package httputils
import (
"encoding/json"
"net/http"
)
// WriteJSON writes the value v to the http response stream as json with standard json encoding.
func WriteJSON(w http.ResponseWriter, code int, v interface{}) error {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(code)
enc := json.NewEncoder(w)
return enc.Encode(v)
}

2
components/engine/api/server/middleware/debug.go
View File

@ -64,7 +64,7 @@ func maskSecretKeys(inp interface{}) {
if form, ok := inp.(map[string]interface{}); ok {
loop0:
for k, v := range form {
for _, m := range []string{"password", "secret", "jointoken", "unlockkey"} {
for _, m := range []string{"password", "secret", "jointoken", "unlockkey", "signingcakey"} {
if strings.EqualFold(m, k) {
form[k] = "*****"
continue loop0

4
components/engine/api/server/router/build/backend.go
View File

@ -1,6 +1,7 @@
package build
import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/backend"
"golang.org/x/net/context"
)
@ -10,6 +11,9 @@ type Backend interface {
// Build a Docker image returning the id of the image
// TODO: make this return a reference instead of string
Build(context.Context, backend.BuildConfig) (string, error)
// Prune build cache
PruneCache(context.Context) (*types.BuildCachePruneReport, error)
}
type experimentalProvider interface {

1
components/engine/api/server/router/build/build.go
View File

@ -24,5 +24,6 @@ func (r *buildRouter) Routes() []router.Route {
func (r *buildRouter) initRoutes() {
r.routes = []router.Route{
router.NewPostRoute("/build", r.postBuild, router.WithCancel),
router.NewPostRoute("/build/prune", r.postPrune, router.WithCancel),
}
}

9
components/engine/api/server/router/build/build_routes.go
View File

@ -127,10 +127,19 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui
}
options.CacheFrom = cacheFrom
}
options.SessionID = r.FormValue("session")
return options, nil
}
func (br *buildRouter) postPrune(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
report, err := br.backend.PruneCache(ctx)
if err != nil {
return err
}
return httputils.WriteJSON(w, http.StatusOK, report)
}
func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
var (
notVerboseBuffer = bytes.NewBuffer(nil)

4
components/engine/api/server/router/image/backend.go
View File

@ -35,12 +35,12 @@ type imageBackend interface {
type importExportBackend interface {
LoadImage(inTar io.ReadCloser, outStream io.Writer, quiet bool) error
ImportImage(src string, repository, tag string, msg string, inConfig io.ReadCloser, outStream io.Writer, changes []string) error
ImportImage(src string, repository, platform string, tag string, msg string, inConfig io.ReadCloser, outStream io.Writer, changes []string) error
ExportImage(names []string, outStream io.Writer) error
}
type registryBackend interface {
PullImage(ctx context.Context, image, tag string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error
PullImage(ctx context.Context, image, tag, platform string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error
PushImage(ctx context.Context, image, tag string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error
SearchRegistryForImages(ctx context.Context, filtersArgs string, term string, limit int, authConfig *types.AuthConfig, metaHeaders map[string][]string) (*registry.SearchResults, error)
}

41
components/engine/api/server/router/image/image_routes.go
View File

@ -6,6 +6,7 @@ import (
"fmt"
"io"
"net/http"
"runtime"
"strconv"
"strings"
@ -17,6 +18,7 @@ import (
"github.com/docker/docker/api/types/versions"
"github.com/docker/docker/pkg/ioutils"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/pkg/system"
"github.com/docker/docker/registry"
"golang.org/x/net/context"
)
@ -85,6 +87,41 @@ func (s *imageRouter) postImagesCreate(ctx context.Context, w http.ResponseWrite
)
defer output.Close()
// TODO @jhowardmsft LCOW Support: Eventually we will need an API change
// so that platform comes from (for example) r.Form.Get("platform"). For
// the initial implementation, we assume that the platform is the
// runtime OS of the host. It will also need a validation function such
// as below which should be called after getting it from the API.
//
// Ensures the requested platform is valid and normalized
//func validatePlatform(req string) (string, error) {
// req = strings.ToLower(req)
// if req == "" {
// req = runtime.GOOS // default to host platform
// }
// valid := []string{runtime.GOOS}
//
// if runtime.GOOS == "windows" && system.LCOWSupported() {
// valid = append(valid, "linux")
// }
//
// for _, item := range valid {
// if req == item {
// return req, nil
// }
// }
// return "", fmt.Errorf("invalid platform requested: %s", req)
//}
//
// And in the call-site:
// if platform, err = validatePlatform(platform); err != nil {
// return err
// }
platform := runtime.GOOS
if platform == "windows" && system.LCOWSupported() {
platform = "linux"
}
w.Header().Set("Content-Type", "application/json")
if image != "" { //pull
@ -106,13 +143,13 @@ func (s *imageRouter) postImagesCreate(ctx context.Context, w http.ResponseWrite
}
}
err = s.backend.PullImage(ctx, image, tag, metaHeaders, authConfig, output)
err = s.backend.PullImage(ctx, image, tag, platform, metaHeaders, authConfig, output)
} else { //import
src := r.Form.Get("fromSrc")
// 'err' MUST NOT be defined within this block, we need any error
// generated from the download to be available to the output
// stream processing below
err = s.backend.ImportImage(src, repo, tag, message, r.Body, output, r.Form["changes"])
err = s.backend.ImportImage(src, repo, platform, tag, message, r.Body, output, r.Form["changes"])
}
if err != nil {
if !output.Flushed() {

23
components/engine/api/server/router/network/network_routes.go
View File

@ -98,6 +98,14 @@ func (n *networkRouter) getNetwork(ctx context.Context, w http.ResponseWriter, r
return errors.NewBadRequestError(err)
}
}
scope := r.URL.Query().Get("scope")
isMatchingScope := func(scope, term string) bool {
if term != "" {
return scope == term
}
return true
}
// In case multiple networks have duplicate names, return error.
// TODO (yongtang): should we wrap with version here for backward compatibility?
@ -112,15 +120,15 @@ func (n *networkRouter) getNetwork(ctx context.Context, w http.ResponseWriter, r
nw := n.backend.GetNetworks()
for _, network := range nw {
if network.ID() == term {
if network.ID() == term && isMatchingScope(network.Info().Scope(), scope) {
return httputils.WriteJSON(w, http.StatusOK, *n.buildDetailedNetworkResources(network, verbose))
}
if network.Name() == term {
if network.Name() == term && isMatchingScope(network.Info().Scope(), scope) {
// No need to check the ID collision here as we are still in
// local scope and the network ID is unique in this scope.
listByFullName[network.ID()] = *n.buildDetailedNetworkResources(network, verbose)
}
if strings.HasPrefix(network.ID(), term) {
if strings.HasPrefix(network.ID(), term) && isMatchingScope(network.Info().Scope(), scope) {
// No need to check the ID collision here as we are still in
// local scope and the network ID is unique in this scope.
listByPartialID[network.ID()] = *n.buildDetailedNetworkResources(network, verbose)
@ -129,10 +137,10 @@ func (n *networkRouter) getNetwork(ctx context.Context, w http.ResponseWriter, r
nr, _ := n.cluster.GetNetworks()
for _, network := range nr {
if network.ID == term {
if network.ID == term && isMatchingScope(network.Scope, scope) {
return httputils.WriteJSON(w, http.StatusOK, network)
}
if network.Name == term {
if network.Name == term && isMatchingScope(network.Scope, scope) {
// Check the ID collision as we are in swarm scope here, and
// the map (of the listByFullName) may have already had a
// network with the same ID (from local scope previously)
@ -140,7 +148,7 @@ func (n *networkRouter) getNetwork(ctx context.Context, w http.ResponseWriter, r
listByFullName[network.ID] = network
}
}
if strings.HasPrefix(network.ID, term) {
if strings.HasPrefix(network.ID, term) && isMatchingScope(network.Scope, scope) {
// Check the ID collision as we are in swarm scope here, and
// the map (of the listByPartialID) may have already had a
// network with the same ID (from local scope previously)
@ -410,6 +418,9 @@ func buildIpamResources(r *types.NetworkResource, nwInfo libnetwork.NetworkInfo)
if !hasIpv6Conf {
for _, ip6Info := range ipv6Info {
if ip6Info.IPAMData.Pool == nil {
continue
}
iData := network.IPAMConfig{}
iData.Subnet = ip6Info.IPAMData.Pool.String()
iData.Gateway = ip6Info.IPAMData.Gateway.String()

12
components/engine/api/server/router/session/backend.go Normal file
View File

@ -0,0 +1,12 @@
package session
import (
"net/http"
"golang.org/x/net/context"
)
// Backend abstracts an session receiver from an http request.
type Backend interface {
HandleHTTPRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) error
}

29
components/engine/api/server/router/session/session.go Normal file
View File

@ -0,0 +1,29 @@
package session
import "github.com/docker/docker/api/server/router"
// sessionRouter is a router to talk with the session controller
type sessionRouter struct {
backend Backend
routes []router.Route
}
// NewRouter initializes a new session router
func NewRouter(b Backend) router.Router {
r := &sessionRouter{
backend: b,
}
r.initRoutes()
return r
}
// Routes returns the available routers to the session controller
func (r *sessionRouter) Routes() []router.Route {
return r.routes
}
func (r *sessionRouter) initRoutes() {
r.routes = []router.Route{
router.Experimental(router.NewPostRoute("/session", r.startSession)),
}
}

16
components/engine/api/server/router/session/session_routes.go Normal file
View File

@ -0,0 +1,16 @@
package session
import (
"net/http"
apierrors "github.com/docker/docker/api/errors"
"golang.org/x/net/context"
)
func (sr *sessionRouter) startSession(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
err := sr.backend.HandleHTTPRequest(ctx, w, r)
if err != nil {
return apierrors.NewBadRequestError(err)
}
return nil
}

5
components/engine/api/server/router/system/system.go
View File

@ -2,6 +2,7 @@ package system
import (
"github.com/docker/docker/api/server/router"
"github.com/docker/docker/builder/fscache"
"github.com/docker/docker/daemon/cluster"
)
@ -11,13 +12,15 @@ type systemRouter struct {
backend Backend
cluster *cluster.Cluster
routes []router.Route
builder *fscache.FSCache
}
// NewRouter initializes a new system router
func NewRouter(b Backend, c *cluster.Cluster) router.Router {
func NewRouter(b Backend, c *cluster.Cluster, fscache *fscache.FSCache) router.Router {
r := &systemRouter{
backend: b,
cluster: c,
builder: fscache,
}
r.routes = []router.Route{

6
components/engine/api/server/router/system/system_routes.go
View File

@ -17,6 +17,7 @@ import (
timetypes "github.com/docker/docker/api/types/time"
"github.com/docker/docker/api/types/versions"
"github.com/docker/docker/pkg/ioutils"
pkgerrors "github.com/pkg/errors"
"golang.org/x/net/context"
)
@ -75,6 +76,11 @@ func (s *systemRouter) getDiskUsage(ctx context.Context, w http.ResponseWriter,
if err != nil {
return err
}
builderSize, err := s.builder.DiskUsage()
if err != nil {
return pkgerrors.Wrap(err, "error getting build cache usage")
}
du.BuilderSize = builderSize
return httputils.WriteJSON(w, http.StatusOK, du)
}

80
components/engine/api/swagger.yaml
View File

@ -19,10 +19,10 @@ produces:
consumes:
- "application/json"
- "text/plain"
basePath: "/v1.30"
basePath: "/v1.31"
info:
title: "Docker Engine API"
version: "1.30"
version: "1.31"
x-logo:
url: "https://docs.docker.com/images/logo-docker-main.png"
description: |
@ -52,10 +52,11 @@ info:
The API uses an open schema model, which means server may add extra properties to responses. Likewise, the server will ignore any extra query parameters and request body properties. When you write clients, you need to ignore additional properties in responses to ensure they do not break when talking to newer Docker daemons.
This documentation is for version 1.30 of the API, which was introduced with Docker 17.06. Use this table to find documentation for previous versions of the API:
This documentation is for version 1.31 of the API. Use this table to find documentation for previous versions of the API:
Docker version | API version | Changes
----------------|-------------|---------
17.06.x | [1.30](https://docs.docker.com/engine/api/v1.30/) | [API changes](https://docs.docker.com/engine/api/version-history/#v1-30-api-changes)
17.05.x | [1.29](https://docs.docker.com/engine/api/v1.29/) | [API changes](https://docs.docker.com/engine/api/version-history/#v1-29-api-changes)
17.04.x | [1.28](https://docs.docker.com/engine/api/v1.28/) | [API changes](https://docs.docker.com/engine/api/version-history/#v1-28-api-changes)
17.03.1 | [1.27](https://docs.docker.com/engine/api/v1.27/) | [API changes](https://docs.docker.com/engine/api/version-history/#v1-27-api-changes)
@ -2698,6 +2699,11 @@ paths:
/containers/json:
get:
summary: "List containers"
description: |
Returns a list of containers. For details on the format, see [the inspect endpoint](#operation/ContainerInspect).
Note that it uses a different, smaller representation of a container than inspecting a single container. For example,
the list of linked containers is not propagated .
operationId: "ContainerList"
produces:
- "application/json"
@ -4739,6 +4745,27 @@ paths:
schema:
$ref: "#/definitions/ErrorResponse"
tags: ["Image"]
/build/prune:
post:
summary: "Delete builder cache"
produces:
- "application/json"
operationId: "BuildPrune"
responses:
200:
description: "No error"
schema:
type: "object"
properties:
SpaceReclaimed:
description: "Disk space reclaimed in bytes"
type: "integer"
format: "int64"
500:
description: "Server error"
schema:
$ref: "#/definitions/ErrorResponse"
tags: ["Image"]
/images/create:
post:
summary: "Create an image"
@ -6431,6 +6458,10 @@ paths:
description: "Detailed inspect output for troubleshooting"
type: "boolean"
default: false
- name: "scope"
in: "query"
description: "Filter the network by scope (swarm, global, or local)"
type: "string"
tags: ["Network"]
delete:
@ -8471,3 +8502,46 @@ paths:
type: "string"
required: true
tags: ["Distribution"]
/session:
post:
summary: "Initialize interactive session"
description: |
Start a new interactive session with a server. Session allows server to call back to the client for advanced capabilities.
> **Note**: This endpoint is *experimental* and only available if the daemon is started with experimental
> features enabled. The specifications for this endpoint may still change in a future version of the API.
### Hijacking
This endpoint hijacks the HTTP connection to HTTP2 transport that allows the client to expose gPRC services on that connection.
For example, the client sends this request to upgrade the connection:
```
POST /session HTTP/1.1
Upgrade: h2c
Connection: Upgrade
```
The Docker daemon will respond with a `101 UPGRADED` response follow with the raw stream:
```
HTTP/1.1 101 UPGRADED
Connection: Upgrade
Upgrade: h2c
```
operationId: "Session"
produces:
- "application/vnd.docker.raw-stream"
responses:
101:
description: "no error, hijacking successful"
400:
description: "bad parameter"
schema:
$ref: "#/definitions/ErrorResponse"
500:
description: "server error"
schema:
$ref: "#/definitions/ErrorResponse"
tags: ["Session (experimental)"]

15
components/engine/api/types/backend/build.go
View File

@ -7,6 +7,18 @@ import (
"github.com/docker/docker/pkg/streamformatter"
)
// PullOption defines different modes for accessing images
type PullOption int
const (
// PullOptionNoPull only returns local images
PullOptionNoPull PullOption = iota
// PullOptionForcePull always tries to pull a ref from the registry first
PullOptionForcePull
// PullOptionPreferLocal uses local image if it exists, otherwise pulls
PullOptionPreferLocal
)
// ProgressWriter is a data object to transport progress streams to the client
type ProgressWriter struct {
Output io.Writer
@ -25,7 +37,8 @@ type BuildConfig struct {
// GetImageAndLayerOptions are the options supported by GetImageAndReleasableLayer
type GetImageAndLayerOptions struct {
ForcePull bool
PullOption PullOption
AuthConfig map[string]types.AuthConfig
Output io.Writer
Platform string
}

7
components/engine/api/types/client.go
View File

@ -7,7 +7,7 @@ import (
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/filters"
"github.com/docker/go-units"
units "github.com/docker/go-units"
)
// CheckpointCreateOptions holds parameters to create a checkpoint from a container
@ -178,6 +178,11 @@ type ImageBuildOptions struct {
SecurityOpt []string
ExtraHosts []string // List of extra hosts
Target string
SessionID string
// TODO @jhowardmsft LCOW Support: This will require extending to include
// `Platform string`, but is ommited for now as it's hard-coded temporarily
// to avoid API changes.
}
// ImageBuildResponse holds information

1
components/engine/api/types/configs.go
View File

@ -16,6 +16,7 @@ type ContainerCreateConfig struct {
HostConfig *container.HostConfig
NetworkingConfig *network.NetworkingConfig
AdjustCPUShares bool
Platform string
}
// ContainerRmConfig holds arguments for the container remove

22
components/engine/api/types/types.go
View File

@ -320,6 +320,7 @@ type ContainerJSONBase struct {
Name string
RestartCount int
Driver string
Platform string
MountLabel string
ProcessLabel string
AppArmorProfile string
@ -468,6 +469,12 @@ type NetworkDisconnect struct {
Force bool
}
// NetworkInspectOptions holds parameters to inspect network
type NetworkInspectOptions struct {
Scope string
Verbose bool
}
// Checkpoint represents the details of a checkpoint
type Checkpoint struct {
Name string // Name is the name of the checkpoint
@ -482,10 +489,11 @@ type Runtime struct {
// DiskUsage contains response of Engine API:
// GET "/system/df"
type DiskUsage struct {
LayersSize int64
Images []*ImageSummary
Containers []*Container
Volumes []*Volume
LayersSize int64
Images []*ImageSummary
Containers []*Container
Volumes []*Volume
BuilderSize int64
}
// ContainersPruneReport contains the response for Engine API:
@ -509,6 +517,12 @@ type ImagesPruneReport struct {
SpaceReclaimed uint64
}
// BuildCachePruneReport contains the response for Engine API:
// POST "/build/prune"
type BuildCachePruneReport struct {
SpaceReclaimed uint64
}
// NetworksPruneReport contains the response for Engine API:
// POST "/networks/prune"
type NetworksPruneReport struct {

12
components/engine/builder/builder.go
View File

@ -11,6 +11,7 @@ import (
"github.com/docker/docker/api/types/backend"
"github.com/docker/docker/api/types/container"
containerpkg "github.com/docker/docker/container"
"github.com/docker/docker/layer"
"golang.org/x/net/context"
)
@ -42,11 +43,7 @@ type Backend interface {
// ContainerCreateWorkdir creates the workdir
ContainerCreateWorkdir(containerID string) error
// ContainerCopy copies/extracts a source FileInfo to a destination path inside a container
// specified by a container object.
// TODO: extract in the builder instead of passing `decompress`
// TODO: use containerd/fs.changestream instead as a source
CopyOnBuild(containerID string, destPath string, srcRoot string, srcPath string, decompress bool) error
CreateImage(config []byte, parent string, platform string) (Image, error)
ImageCacheBuilder
}
@ -81,7 +78,7 @@ type Result struct {
// ImageCacheBuilder represents a generator for stateful image cache.
type ImageCacheBuilder interface {
// MakeImageCache creates a stateful image cache.
MakeImageCache(cacheFrom []string) ImageCache
MakeImageCache(cacheFrom []string, platform string) ImageCache
}
// ImageCache abstracts an image cache.
@ -96,10 +93,13 @@ type ImageCache interface {
type Image interface {
ImageID() string
RunConfig() *container.Config
MarshalJSON() ([]byte, error)
}
// ReleaseableLayer is an image layer that can be mounted and released
type ReleaseableLayer interface {
Release() error
Mount() (string, error)
Commit(platform string) (ReleaseableLayer, error)
DiffID() layer.DiffID
}

130
components/engine/builder/dockerfile/builder.go
View File

@ -5,7 +5,9 @@ import (
"fmt"
"io"
"io/ioutil"
"runtime"
"strings"
"time"
"github.com/Sirupsen/logrus"
"github.com/docker/docker/api/types"
@ -14,9 +16,15 @@ import (
"github.com/docker/docker/builder"
"github.com/docker/docker/builder/dockerfile/command"
"github.com/docker/docker/builder/dockerfile/parser"
"github.com/docker/docker/builder/fscache"
"github.com/docker/docker/builder/remotecontext"
"github.com/docker/docker/client/session"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/chrootarchive"
"github.com/docker/docker/pkg/idtools"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/pkg/stringid"
"github.com/docker/docker/pkg/system"
"github.com/pkg/errors"
"golang.org/x/net/context"
"golang.org/x/sync/syncmap"
@ -35,18 +43,33 @@ var validCommitCommands = map[string]bool{
"workdir": true,
}
// SessionGetter is object used to get access to a session by uuid
type SessionGetter interface {
Get(ctx context.Context, uuid string) (session.Caller, error)
}
// BuildManager is shared across all Builder objects
type BuildManager struct {
archiver *archive.Archiver
backend builder.Backend
pathCache pathCache // TODO: make this persistent
sg SessionGetter
fsCache *fscache.FSCache
}
// NewBuildManager creates a BuildManager
func NewBuildManager(b builder.Backend) *BuildManager {
return &BuildManager{
func NewBuildManager(b builder.Backend, sg SessionGetter, fsCache *fscache.FSCache, idMappings *idtools.IDMappings) (*BuildManager, error) {
bm := &BuildManager{
backend: b,
pathCache: &syncmap.Map{},
sg: sg,
archiver: chrootarchive.NewArchiver(idMappings),
fsCache: fsCache,
}
if err := fsCache.RegisterTransport(remotecontext.ClientSessionRemote, NewClientSessionTransport()); err != nil {
return nil, err
}
return bm, nil
}
// Build starts a new build from a BuildConfig
@ -60,12 +83,30 @@ func (bm *BuildManager) Build(ctx context.Context, config backend.BuildConfig) (
if err != nil {
return nil, err
}
if source != nil {
defer func() {
defer func() {
if source != nil {
if err := source.Close(); err != nil {
logrus.Debugf("[BUILDER] failed to remove temporary context: %v", err)
}
}()
}
}()
// TODO @jhowardmsft LCOW support - this will require rework to allow both linux and Windows simultaneously.
// This is an interim solution to hardcode to linux if LCOW is turned on.
if dockerfile.Platform == "" {
dockerfile.Platform = runtime.GOOS
if dockerfile.Platform == "windows" && system.LCOWSupported() {
dockerfile.Platform = "linux"
}
}
ctx, cancel := context.WithCancel(ctx)
defer cancel()
if src, err := bm.initializeClientSession(ctx, cancel, config.Options); err != nil {
return nil, err
} else if src != nil {
source = src
}
builderOptions := builderOptions{
@ -73,16 +114,55 @@ func (bm *BuildManager) Build(ctx context.Context, config backend.BuildConfig) (
ProgressWriter: config.ProgressWriter,
Backend: bm.backend,
PathCache: bm.pathCache,
Archiver: bm.archiver,
Platform: dockerfile.Platform,
}
return newBuilder(ctx, builderOptions).build(source, dockerfile)
}
func (bm *BuildManager) initializeClientSession(ctx context.Context, cancel func(), options *types.ImageBuildOptions) (builder.Source, error) {
if options.SessionID == "" || bm.sg == nil {
return nil, nil
}
logrus.Debug("client is session enabled")
ctx, cancelCtx := context.WithTimeout(ctx, sessionConnectTimeout)
defer cancelCtx()
c, err := bm.sg.Get(ctx, options.SessionID)
if err != nil {
return nil, err
}
go func() {
<-c.Context().Done()
cancel()
}()
if options.RemoteContext == remotecontext.ClientSessionRemote {
st := time.Now()
csi, err := NewClientSessionSourceIdentifier(ctx, bm.sg,
options.SessionID, []string{"/"})
if err != nil {
return nil, err
}
src, err := bm.fsCache.SyncFrom(ctx, csi)
if err != nil {
return nil, err
}
logrus.Debugf("sync-time: %v", time.Since(st))
return src, nil
}
return nil, nil
}
// builderOptions are the dependencies required by the builder
type builderOptions struct {
Options *types.ImageBuildOptions
Backend builder.Backend
ProgressWriter backend.ProgressWriter
PathCache pathCache
Archiver *archive.Archiver
Platform string
}
// Builder is a Dockerfile builder
@ -98,6 +178,7 @@ type Builder struct {
docker builder.Backend
clientCtx context.Context
archiver *archive.Archiver
buildStages *buildStages
disableCommit bool
buildArgs *buildArgs
@ -105,14 +186,32 @@ type Builder struct {
pathCache pathCache
containerManager *containerManager
imageProber ImageProber
// TODO @jhowardmft LCOW Support. This will be moved to options at a later
// stage, however that cannot be done now as it affects the public API
// if it were.
platform string
}
// newBuilder creates a new Dockerfile builder from an optional dockerfile and a Options.
// TODO @jhowardmsft LCOW support: Eventually platform can be moved into the builder
// options, however, that would be an API change as it shares types.ImageBuildOptions.
func newBuilder(clientCtx context.Context, options builderOptions) *Builder {
config := options.Options
if config == nil {
config = new(types.ImageBuildOptions)
}
// @jhowardmsft LCOW Support. For the time being, this is interim. Eventually
// will be moved to types.ImageBuildOptions, but it can't for now as that would
// be an API change.
if options.Platform == "" {
options.Platform = runtime.GOOS
}
if options.Platform == "windows" && system.LCOWSupported() {
options.Platform = "linux"
}
b := &Builder{
clientCtx: clientCtx,
options: config,
@ -121,13 +220,16 @@ func newBuilder(clientCtx context.Context, options builderOptions) *Builder {
Aux: options.ProgressWriter.AuxFormatter,
Output: options.ProgressWriter.Output,
docker: options.Backend,
archiver: options.Archiver,
buildArgs: newBuildArgs(config.BuildArgs),
buildStages: newBuildStages(),
imageSources: newImageSources(clientCtx, options),
pathCache: options.PathCache,
imageProber: newImageProber(options.Backend, config.CacheFrom, config.NoCache),
imageProber: newImageProber(options.Backend, config.CacheFrom, options.Platform, config.NoCache),
containerManager: newContainerManager(options.Backend),
platform: options.Platform,
}
return b
}
@ -258,6 +360,17 @@ func BuildFromConfig(config *container.Config, changes []string) (*container.Con
return nil, err
}
// TODO @jhowardmsft LCOW support. For now, if LCOW enabled, switch to linux.
// Also explicitly set the platform. Ultimately this will be in the builder
// options, but we can't do that yet as it would change the API.
if dockerfile.Platform == "" {
dockerfile.Platform = runtime.GOOS
}
if dockerfile.Platform == "windows" && system.LCOWSupported() {
dockerfile.Platform = "linux"
}
b.platform = dockerfile.Platform
// ensure that the commands are valid
for _, n := range dockerfile.AST.Children {
if !validCommitCommands[n.Value] {
@ -274,7 +387,7 @@ func BuildFromConfig(config *container.Config, changes []string) (*container.Con
}
dispatchState := newDispatchState()
dispatchState.runConfig = config
return dispatchFromDockerfile(b, dockerfile, dispatchState)
return dispatchFromDockerfile(b, dockerfile, dispatchState, nil)
}
func checkDispatchDockerfile(dockerfile *parser.Node) error {
@ -286,7 +399,7 @@ func checkDispatchDockerfile(dockerfile *parser.Node) error {
return nil
}
func dispatchFromDockerfile(b *Builder, result *parser.Result, dispatchState *dispatchState) (*container.Config, error) {
func dispatchFromDockerfile(b *Builder, result *parser.Result, dispatchState *dispatchState, source builder.Source) (*container.Config, error) {
shlex := NewShellLex(result.EscapeToken)
ast := result.AST
total := len(ast.Children)
@ -297,6 +410,7 @@ func dispatchFromDockerfile(b *Builder, result *parser.Result, dispatchState *di
stepMsg: formatStep(i, total),
node: n,
shlex: shlex,
source: source,
}
if _, err := b.dispatch(opts); err != nil {
return nil, err

4
components/engine/builder/dockerfile/builder_unix.go
View File

@ -2,4 +2,6 @@
package dockerfile
var defaultShell = []string{"/bin/sh", "-c"}
func defaultShellForPlatform(platform string) []string {
return []string{"/bin/sh", "-c"}
}

7
components/engine/builder/dockerfile/builder_windows.go
View File

@ -1,3 +1,8 @@
package dockerfile
var defaultShell = []string{"cmd", "/S", "/C"}
func defaultShellForPlatform(platform string) []string {
if platform == "linux" {
return []string{"/bin/sh", "-c"}
}
return []string{"cmd", "/S", "/C"}
}

78
components/engine/builder/dockerfile/clientsession.go Normal file
View File

@ -0,0 +1,78 @@
package dockerfile
import (
"time"
"github.com/docker/docker/builder/fscache"
"github.com/docker/docker/builder/remotecontext"
"github.com/docker/docker/client/session"
"github.com/docker/docker/client/session/filesync"
"github.com/pkg/errors"
"golang.org/x/net/context"
)
const sessionConnectTimeout = 5 * time.Second
// ClientSessionTransport is a transport for copying files from docker client
// to the daemon.
type ClientSessionTransport struct{}
// NewClientSessionTransport returns new ClientSessionTransport instance
func NewClientSessionTransport() *ClientSessionTransport {
return &ClientSessionTransport{}
}
// Copy data from a remote to a destination directory.
func (cst *ClientSessionTransport) Copy(ctx context.Context, id fscache.RemoteIdentifier, dest string, cu filesync.CacheUpdater) error {
csi, ok := id.(*ClientSessionSourceIdentifier)
if !ok {
return errors.New("invalid identifier for client session")
}
return filesync.FSSync(ctx, csi.caller, filesync.FSSendRequestOpt{
SrcPaths: csi.srcPaths,
DestDir: dest,
CacheUpdater: cu,
})
}
// ClientSessionSourceIdentifier is an identifier that can be used for requesting
// files from remote client
type ClientSessionSourceIdentifier struct {
srcPaths []string
caller session.Caller
sharedKey string
uuid string
}
// NewClientSessionSourceIdentifier returns new ClientSessionSourceIdentifier instance
func NewClientSessionSourceIdentifier(ctx context.Context, sg SessionGetter, uuid string, sources []string) (*ClientSessionSourceIdentifier, error) {
csi := &ClientSessionSourceIdentifier{
uuid: uuid,
srcPaths: sources,
}
caller, err := sg.Get(ctx, uuid)
if err != nil {
return nil, errors.Wrapf(err, "failed to get session for %s", uuid)
}
csi.caller = caller
return csi, nil
}
// Transport returns transport identifier for remote identifier
func (csi *ClientSessionSourceIdentifier) Transport() string {
return remotecontext.ClientSessionRemote
}
// SharedKey returns shared key for remote identifier. Shared key is used
// for finding the base for a repeated transfer.
func (csi *ClientSessionSourceIdentifier) SharedKey() string {
return csi.caller.SharedKey()
}
// Key returns unique key for remote identifier. Requests with same key return
// same data.
func (csi *ClientSessionSourceIdentifier) Key() string {
return csi.uuid
}

3
components/engine/builder/dockerfile/containerbackend.go
View File

@ -28,10 +28,11 @@ func newContainerManager(docker builder.ExecBackend) *containerManager {
}
// Create a container
func (c *containerManager) Create(runConfig *container.Config, hostConfig *container.HostConfig) (container.ContainerCreateCreatedBody, error) {
func (c *containerManager) Create(runConfig *container.Config, hostConfig *container.HostConfig, platform string) (container.ContainerCreateCreatedBody, error) {
container, err := c.backend.ContainerCreate(types.ContainerCreateConfig{
Config: runConfig,
HostConfig: hostConfig,
Platform: platform,
})
if err != nil {
return container, err

93
components/engine/builder/dockerfile/copy.go
View File

@ -13,10 +13,12 @@ import (
"github.com/docker/docker/builder"
"github.com/docker/docker/builder/remotecontext"
"github.com/docker/docker/pkg/httputils"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/idtools"
"github.com/docker/docker/pkg/ioutils"
"github.com/docker/docker/pkg/progress"
"github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/pkg/symlink"
"github.com/docker/docker/pkg/system"
"github.com/docker/docker/pkg/urlutil"
"github.com/pkg/errors"
@ -35,6 +37,10 @@ type copyInfo struct {
hash string
}
func (c copyInfo) fullPath() (string, error) {
return symlink.FollowSymlinkInScope(filepath.Join(c.root, c.path), c.root)
}
func newCopyInfoFromSource(source builder.Source, path string, hash string) copyInfo {
return copyInfo{root: source.Root(), path: path, hash: hash}
}
@ -149,7 +155,7 @@ func (o *copier) calcCopyInfo(origPath string, allowWildcards bool) ([]copyInfo,
var err error
o.source, err = imageSource.Source()
if err != nil {
return nil, errors.Wrapf(err, "failed to copy")
return nil, errors.Wrapf(err, "failed to copy from %s", imageSource.ImageID())
}
}
@ -292,7 +298,6 @@ func errOnSourceDownload(_ string) (builder.Source, string, error) {
}
func downloadSource(output io.Writer, stdout io.Writer, srcURL string) (remote builder.Source, p string, err error) {
// get filename from URL
u, err := url.Parse(srcURL)
if err != nil {
return
@ -303,8 +308,7 @@ func downloadSource(output io.Writer, stdout io.Writer, srcURL string) (remote b
return
}
// Initiate the download
resp, err := httputils.Download(srcURL)
resp, err := remotecontext.GetWithStatusError(srcURL)
if err != nil {
return
}
@ -355,6 +359,83 @@ func downloadSource(output io.Writer, stdout io.Writer, srcURL string) (remote b
return
}
lc, err := remotecontext.NewLazyContext(tmpDir)
lc, err := remotecontext.NewLazySource(tmpDir)
return lc, filename, err
}
type copyFileOptions struct {
decompress bool
archiver *archive.Archiver
}
func performCopyForInfo(dest copyInfo, source copyInfo, options copyFileOptions) error {
srcPath, err := source.fullPath()
if err != nil {
return err
}
destPath, err := dest.fullPath()
if err != nil {
return err
}
archiver := options.archiver
src, err := os.Stat(srcPath)
if err != nil {
return errors.Wrapf(err, "source path not found")
}
if src.IsDir() {
return copyDirectory(archiver, srcPath, destPath)
}
if options.decompress && archive.IsArchivePath(srcPath) {
return archiver.UntarPath(srcPath, destPath)
}
destExistsAsDir, err := isExistingDirectory(destPath)
if err != nil {
return err
}
// dest.path must be used because destPath has already been cleaned of any
// trailing slash
if endsInSlash(dest.path) || destExistsAsDir {
// source.path must be used to get the correct filename when the source
// is a symlink
destPath = filepath.Join(destPath, filepath.Base(source.path))
}
return copyFile(archiver, srcPath, destPath)
}
func copyDirectory(archiver *archive.Archiver, source, dest string) error {
if err := archiver.CopyWithTar(source, dest); err != nil {
return errors.Wrapf(err, "failed to copy directory")
}
return fixPermissions(source, dest, archiver.IDMappings.RootPair())
}
func copyFile(archiver *archive.Archiver, source, dest string) error {
rootIDs := archiver.IDMappings.RootPair()
if err := idtools.MkdirAllAndChownNew(filepath.Dir(dest), 0755, rootIDs); err != nil {
return errors.Wrapf(err, "failed to create new directory")
}
if err := archiver.CopyFileWithTar(source, dest); err != nil {
return errors.Wrapf(err, "failed to copy file")
}
return fixPermissions(source, dest, rootIDs)
}
func endsInSlash(path string) bool {
return strings.HasSuffix(path, string(os.PathSeparator))
}
// isExistingDirectory returns true if the path exists and is a directory
func isExistingDirectory(path string) (bool, error) {
destStat, err := os.Stat(path)
switch {
case os.IsNotExist(err):
return false, nil
case err != nil:
return false, err
}
return destStat.IsDir(), nil
}

45
components/engine/builder/dockerfile/copy_test.go Normal file
View File

@ -0,0 +1,45 @@
package dockerfile
import (
"testing"
"github.com/docker/docker/pkg/testutil/tempfile"
"github.com/stretchr/testify/assert"
)
func TestIsExistingDirectory(t *testing.T) {
tmpfile := tempfile.NewTempFile(t, "file-exists-test", "something")
defer tmpfile.Remove()
tmpdir := tempfile.NewTempDir(t, "dir-exists-test")
defer tmpdir.Remove()
var testcases = []struct {
doc string
path string
expected bool
}{
{
doc: "directory exists",
path: tmpdir.Path,
expected: true,
},
{
doc: "path doesn't exist",
path: "/bogus/path/does/not/exist",
expected: false,
},
{
doc: "file exists",
path: tmpfile.Name(),
expected: false,
},
}
for _, testcase := range testcases {
result, err := isExistingDirectory(testcase.path)
if !assert.NoError(t, err) {
continue
}
assert.Equal(t, testcase.expected, result, testcase.doc)
}
}

36
components/engine/builder/dockerfile/copy_unix.go Normal file
View File

@ -0,0 +1,36 @@
// +build !windows
package dockerfile
import (
"os"
"path/filepath"
"github.com/docker/docker/pkg/idtools"
)
func fixPermissions(source, destination string, rootIDs idtools.IDPair) error {
skipChownRoot, err := isExistingDirectory(destination)
if err != nil {
return err
}
// We Walk on the source rather than on the destination because we don't
// want to change permissions on things we haven't created or modified.
return filepath.Walk(source, func(fullpath string, info os.FileInfo, err error) error {
// Do not alter the walk root iff. it existed before, as it doesn't fall under
// the domain of "things we should chown".
if skipChownRoot && source == fullpath {
return nil
}
// Path is prefixed by source: substitute with destination instead.
cleaned, err := filepath.Rel(source, fullpath)
if err != nil {
return err
}
fullpath = filepath.Join(destination, cleaned)
return os.Lchown(fullpath, rootIDs.UID, rootIDs.GID)
})
}

8
components/engine/builder/dockerfile/copy_windows.go Normal file
View File

@ -0,0 +1,8 @@
package dockerfile
import "github.com/docker/docker/pkg/idtools"
func fixPermissions(source, destination string, rootIDs idtools.IDPair) error {
// chown is not supported on Windows
return nil
}

38
components/engine/builder/dockerfile/dispatchers.go
View File

@ -23,8 +23,10 @@ import (
"github.com/docker/docker/api/types/strslice"
"github.com/docker/docker/builder"
"github.com/docker/docker/builder/dockerfile/parser"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/jsonmessage"
"github.com/docker/docker/pkg/signal"
"github.com/docker/docker/pkg/system"
"github.com/docker/go-connections/nat"
"github.com/pkg/errors"
)
@ -195,6 +197,7 @@ func (b *Builder) getImageMount(fromFlag *Flag) (*imageMount, error) {
return nil, nil
}
var localOnly bool
imageRefOrID := fromFlag.Value
stage, err := b.buildStages.get(fromFlag.Value)
if err != nil {
@ -202,8 +205,9 @@ func (b *Builder) getImageMount(fromFlag *Flag) (*imageMount, error) {
}
if stage != nil {
imageRefOrID = stage.ImageID()
localOnly = true
}
return b.imageSources.Get(imageRefOrID)
return b.imageSources.Get(imageRefOrID, localOnly)
}
// FROM imagename[:tag | @digest] [AS build-stage-name]
@ -251,10 +255,8 @@ func parseBuildStageName(args []string) (string, error) {
return stageName, nil
}
// scratchImage is used as a token for the empty base image. It uses buildStage
// as a convenient implementation of builder.Image, but is not actually a
// buildStage.
var scratchImage builder.Image = &buildStage{}
// scratchImage is used as a token for the empty base image.
var scratchImage builder.Image = &image.Image{}
func (b *Builder) getFromImage(shlex *ShellLex, name string) (builder.Image, error) {
substitutionArgs := []string{}
@ -267,18 +269,22 @@ func (b *Builder) getFromImage(shlex *ShellLex, name string) (builder.Image, err
return nil, err
}
if im, ok := b.buildStages.getByName(name); ok {
return im, nil
var localOnly bool
if stage, ok := b.buildStages.getByName(name); ok {
name = stage.ImageID()
localOnly = true
}
// Windows cannot support a container with no base image.
// Windows cannot support a container with no base image unless it is LCOW.
if name == api.NoBaseImageSpecifier {
if runtime.GOOS == "windows" {
return nil, errors.New("Windows does not support FROM scratch")
if b.platform == "windows" || (b.platform != "windows" && !system.LCOWSupported()) {
return nil, errors.New("Windows does not support FROM scratch")
}
}
return scratchImage, nil
}
imageMount, err := b.imageSources.Get(name)
imageMount, err := b.imageSources.Get(name, localOnly)
if err != nil {
return nil, err
}
@ -325,7 +331,7 @@ func processOnBuild(req dispatchRequest) error {
}
}
if _, err := dispatchFromDockerfile(req.builder, dockerfile, dispatchState); err != nil {
if _, err := dispatchFromDockerfile(req.builder, dockerfile, dispatchState, req.source); err != nil {
return err
}
}
@ -397,7 +403,7 @@ func workdir(req dispatchRequest) error {
}
comment := "WORKDIR " + runConfig.WorkingDir
runConfigWithCommentCmd := copyRunConfig(runConfig, withCmdCommentString(comment))
runConfigWithCommentCmd := copyRunConfig(runConfig, withCmdCommentString(comment, req.builder.platform))
containerID, err := req.builder.probeAndCreate(req.state, runConfigWithCommentCmd)
if err != nil || containerID == "" {
return err
@ -415,7 +421,7 @@ func workdir(req dispatchRequest) error {
// the current SHELL which defaults to 'sh -c' under linux or 'cmd /S /C' under
// Windows, in the event there is only one argument The difference in processing:
//
// RUN echo hi # sh -c echo hi (Linux)
// RUN echo hi # sh -c echo hi (Linux and LCOW)
// RUN echo hi # cmd /S /C echo hi (Windows)
// RUN [ "echo", "hi" ] # echo hi
//
@ -431,7 +437,7 @@ func run(req dispatchRequest) error {
stateRunConfig := req.state.runConfig
args := handleJSONArgs(req.args, req.attributes)
if !req.attributes["json"] {
args = append(getShell(stateRunConfig), args...)
args = append(getShell(stateRunConfig, req.builder.platform), args...)
}
cmdFromArgs := strslice.StrSlice(args)
buildArgs := req.builder.buildArgs.FilterAllowed(stateRunConfig.Env)
@ -516,7 +522,7 @@ func cmd(req dispatchRequest) error {
runConfig := req.state.runConfig
cmdSlice := handleJSONArgs(req.args, req.attributes)
if !req.attributes["json"] {
cmdSlice = append(getShell(runConfig), cmdSlice...)
cmdSlice = append(getShell(runConfig, req.builder.platform), cmdSlice...)
}
runConfig.Cmd = strslice.StrSlice(cmdSlice)
@ -668,7 +674,7 @@ func entrypoint(req dispatchRequest) error {
runConfig.Entrypoint = nil
default:
// ENTRYPOINT echo hi
runConfig.Entrypoint = strslice.StrSlice(append(getShell(runConfig), parsed[0]))
runConfig.Entrypoint = strslice.StrSlice(append(getShell(runConfig, req.builder.platform), parsed[0]))
}
// when setting the entrypoint if a CMD was not explicitly set then

17
components/engine/builder/dockerfile/dispatchers_test.go
View File

@ -63,7 +63,7 @@ func newBuilderWithMockBackend() *Builder {
Backend: mockBackend,
}),
buildStages: newBuildStages(),
imageProber: newImageProber(mockBackend, nil, false),
imageProber: newImageProber(mockBackend, nil, runtime.GOOS, false),
containerManager: newContainerManager(mockBackend),
}
return b
@ -194,7 +194,7 @@ func TestFromScratch(t *testing.T) {
req := defaultDispatchReq(b, "scratch")
err := from(req)
if runtime.GOOS == "windows" {
if runtime.GOOS == "windows" && !system.LCOWSupported() {
assert.EqualError(t, err, "Windows does not support FROM scratch")
return
}
@ -202,7 +202,12 @@ func TestFromScratch(t *testing.T) {
require.NoError(t, err)
assert.True(t, req.state.hasFromImage())
assert.Equal(t, "", req.state.imageID)
assert.Equal(t, []string{"PATH=" + system.DefaultPathEnv}, req.state.runConfig.Env)
// Windows does not set the default path. TODO @jhowardmsft LCOW support. This will need revisiting as we get further into the implementation
expected := "PATH=" + system.DefaultPathEnv(runtime.GOOS)
if runtime.GOOS == "windows" {
expected = ""
}
assert.Equal(t, []string{expected}, req.state.runConfig.Env)
}
func TestFromWithArg(t *testing.T) {
@ -469,7 +474,7 @@ func TestRunWithBuildArgs(t *testing.T) {
runConfig := &container.Config{}
origCmd := strslice.StrSlice([]string{"cmd", "in", "from", "image"})
cmdWithShell := strslice.StrSlice(append(getShell(runConfig), "echo foo"))
cmdWithShell := strslice.StrSlice(append(getShell(runConfig, runtime.GOOS), "echo foo"))
envVars := []string{"|1", "one=two"}
cachedCmd := strslice.StrSlice(append(envVars, cmdWithShell...))
@ -483,10 +488,10 @@ func TestRunWithBuildArgs(t *testing.T) {
}
mockBackend := b.docker.(*MockBackend)
mockBackend.makeImageCacheFunc = func(_ []string) builder.ImageCache {
mockBackend.makeImageCacheFunc = func(_ []string, _ string) builder.ImageCache {
return imageCache
}
b.imageProber = newImageProber(mockBackend, nil, false)
b.imageProber = newImageProber(mockBackend, nil, runtime.GOOS, false)
mockBackend.getImageFunc = func(_ string) (builder.Image, builder.ReleaseableLayer, error) {
return &mockImage{
id: "abcdef",

18
components/engine/builder/dockerfile/evaluator.go
View File

@ -22,6 +22,7 @@ package dockerfile
import (
"bytes"
"fmt"
"runtime"
"strings"
"github.com/docker/docker/api/types/container"
@ -171,11 +172,9 @@ func (b *Builder) dispatch(options dispatchOptions) (*dispatchState, error) {
buildsFailed.WithValues(metricsUnknownInstructionError).Inc()
return nil, fmt.Errorf("unknown instruction: %s", upperCasedCmd)
}
if err := f(newDispatchRequestFromOptions(options, b, args)); err != nil {
return nil, err
}
options.state.updateRunConfig()
return options.state, nil
err = f(newDispatchRequestFromOptions(options, b, args))
return options.state, err
}
type dispatchOptions struct {
@ -230,14 +229,19 @@ func (s *dispatchState) beginStage(stageName string, image builder.Image) {
}
// Add the default PATH to runConfig.ENV if one exists for the platform and there
// is no PATH set. Note that windows won't have one as it's set by HCS
// is no PATH set. Note that Windows containers on Windows won't have one as it's set by HCS
func (s *dispatchState) setDefaultPath() {
if system.DefaultPathEnv == "" {
// TODO @jhowardmsft LCOW Support - This will need revisiting later
platform := runtime.GOOS
if platform == "windows" && system.LCOWSupported() {
platform = "linux"
}
if system.DefaultPathEnv(platform) == "" {
return
}
envMap := opts.ConvertKVStringsToMap(s.runConfig.Env)
if _, ok := envMap["PATH"]; !ok {
s.runConfig.Env = append(s.runConfig.Env, "PATH="+system.DefaultPathEnv)
s.runConfig.Env = append(s.runConfig.Env, "PATH="+system.DefaultPathEnv(platform))
}
}

2
components/engine/builder/dockerfile/evaluator_test.go
View File

@ -158,7 +158,7 @@ func executeTestCase(t *testing.T, testCase dispatchTestCase) {
}
}()
context, err := remotecontext.MakeTarSumContext(tarStream)
context, err := remotecontext.FromArchive(tarStream)
if err != nil {
t.Fatalf("Error when creating tar context: %s", err)

71
components/engine/builder/dockerfile/imagecontext.go
View File

@ -6,37 +6,29 @@ import (
"github.com/Sirupsen/logrus"
"github.com/docker/docker/api/types/backend"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/builder"
"github.com/docker/docker/builder/remotecontext"
dockerimage "github.com/docker/docker/image"
"github.com/pkg/errors"
"golang.org/x/net/context"
)
type buildStage struct {
id string
config *container.Config
id string
}
func newBuildStageFromImage(image builder.Image) *buildStage {
return &buildStage{id: image.ImageID(), config: image.RunConfig()}
func newBuildStage(imageID string) *buildStage {
return &buildStage{id: imageID}
}
func (b *buildStage) ImageID() string {
return b.id
}
func (b *buildStage) RunConfig() *container.Config {
return b.config
}
func (b *buildStage) update(imageID string, runConfig *container.Config) {
func (b *buildStage) update(imageID string) {
b.id = imageID
b.config = runConfig
}
var _ builder.Image = &buildStage{}
// buildStages tracks each stage of a build so they can be retrieved by index
// or by name.
type buildStages struct {
@ -48,12 +40,12 @@ func newBuildStages() *buildStages {
return &buildStages{byName: make(map[string]*buildStage)}
}
func (s *buildStages) getByName(name string) (builder.Image, bool) {
func (s *buildStages) getByName(name string) (*buildStage, bool) {
stage, ok := s.byName[strings.ToLower(name)]
return stage, ok
}
func (s *buildStages) get(indexOrName string) (builder.Image, error) {
func (s *buildStages) get(indexOrName string) (*buildStage, error) {
index, err := strconv.Atoi(indexOrName)
if err == nil {
if err := s.validateIndex(index); err != nil {
@ -78,7 +70,7 @@ func (s *buildStages) validateIndex(i int) error {
}
func (s *buildStages) add(name string, image builder.Image) error {
stage := newBuildStageFromImage(image)
stage := newBuildStage(image.ImageID())
name = strings.ToLower(name)
if len(name) > 0 {
if _, ok := s.byName[name]; ok {
@ -90,26 +82,38 @@ func (s *buildStages) add(name string, image builder.Image) error {
return nil
}
func (s *buildStages) update(imageID string, runConfig *container.Config) {
s.sequence[len(s.sequence)-1].update(imageID, runConfig)
func (s *buildStages) update(imageID string) {
s.sequence[len(s.sequence)-1].update(imageID)
}
type getAndMountFunc func(string) (builder.Image, builder.ReleaseableLayer, error)
type getAndMountFunc func(string, bool) (builder.Image, builder.ReleaseableLayer, error)
// imageSources mounts images and provides a cache for mounted images. It tracks
// all images so they can be unmounted at the end of the build.
type imageSources struct {
byImageID map[string]*imageMount
mounts []*imageMount
getImage getAndMountFunc
cache pathCache // TODO: remove
}
// TODO @jhowardmsft LCOW Support: Eventually, platform can be moved to options.Options.Platform,
// and removed from builderOptions, but that can't be done yet as it would affect the API.
func newImageSources(ctx context.Context, options builderOptions) *imageSources {
getAndMount := func(idOrRef string) (builder.Image, builder.ReleaseableLayer, error) {
getAndMount := func(idOrRef string, localOnly bool) (builder.Image, builder.ReleaseableLayer, error) {
pullOption := backend.PullOptionNoPull
if !localOnly {
if options.Options.PullParent {
pullOption = backend.PullOptionForcePull
} else {
pullOption = backend.PullOptionPreferLocal
}
}
return options.Backend.GetImageAndReleasableLayer(ctx, idOrRef, backend.GetImageAndLayerOptions{
ForcePull: options.Options.PullParent,
PullOption: pullOption,
AuthConfig: options.Options.AuthConfigs,
Output: options.ProgressWriter.Output,
Platform: options.Platform,
})
}
@ -119,22 +123,22 @@ func newImageSources(ctx context.Context, options builderOptions) *imageSources
}
}
func (m *imageSources) Get(idOrRef string) (*imageMount, error) {
func (m *imageSources) Get(idOrRef string, localOnly bool) (*imageMount, error) {
if im, ok := m.byImageID[idOrRef]; ok {
return im, nil
}
image, layer, err := m.getImage(idOrRef)
image, layer, err := m.getImage(idOrRef, localOnly)
if err != nil {
return nil, err
}
im := newImageMount(image, layer)
m.byImageID[image.ImageID()] = im
m.Add(im)
return im, nil
}
func (m *imageSources) Unmount() (retErr error) {
for _, im := range m.byImageID {
for _, im := range m.mounts {
if err := im.unmount(); err != nil {
logrus.Error(err)
retErr = err
@ -143,6 +147,16 @@ func (m *imageSources) Unmount() (retErr error) {
return
}
func (m *imageSources) Add(im *imageMount) {
switch im.image {
case nil:
im.image = &dockerimage.Image{}
default:
m.byImageID[im.image.ImageID()] = im
}
m.mounts = append(m.mounts, im)
}
// imageMount is a reference to an image that can be used as a builder.Source
type imageMount struct {
image builder.Image
@ -164,7 +178,7 @@ func (im *imageMount) Source() (builder.Source, error) {
if err != nil {
return nil, errors.Wrapf(err, "failed to mount %s", im.image.ImageID())
}
source, err := remotecontext.NewLazyContext(mountPath)
source, err := remotecontext.NewLazySource(mountPath)
if err != nil {
return nil, errors.Wrapf(err, "failed to create lazycontext for %s", mountPath)
}
@ -180,6 +194,7 @@ func (im *imageMount) unmount() error {
if err := im.layer.Release(); err != nil {
return errors.Wrapf(err, "failed to unmount previous build image %s", im.image.ImageID())
}
im.layer = nil
return nil
}
@ -187,6 +202,10 @@ func (im *imageMount) Image() builder.Image {
return im.image
}
func (im *imageMount) Layer() builder.ReleaseableLayer {
return im.layer
}
func (im *imageMount) ImageID() string {
return im.image.ImageID()
}

4
components/engine/builder/dockerfile/imageprobe.go
View File

@ -19,13 +19,13 @@ type imageProber struct {
cacheBusted bool
}
func newImageProber(cacheBuilder builder.ImageCacheBuilder, cacheFrom []string, noCache bool) ImageProber {
func newImageProber(cacheBuilder builder.ImageCacheBuilder, cacheFrom []string, platform string, noCache bool) ImageProber {
if noCache {
return &nopProber{}
}
reset := func() builder.ImageCache {
return cacheBuilder.MakeImageCache(cacheFrom)
return cacheBuilder.MakeImageCache(cacheFrom, platform)
}
return &imageProber{cache: reset(), reset: reset}
}

104
components/engine/builder/dockerfile/internals.go
View File

@ -12,6 +12,7 @@ import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/backend"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/image"
"github.com/docker/docker/pkg/stringid"
"github.com/pkg/errors"
)
@ -24,7 +25,7 @@ func (b *Builder) commit(dispatchState *dispatchState, comment string) error {
return errors.New("Please provide a source image with `from` prior to commit")
}
runConfigWithCommentCmd := copyRunConfig(dispatchState.runConfig, withCmdComment(comment))
runConfigWithCommentCmd := copyRunConfig(dispatchState.runConfig, withCmdComment(comment, b.platform))
hit, err := b.probeCache(dispatchState, runConfigWithCommentCmd)
if err != nil || hit {
return err
@ -37,7 +38,6 @@ func (b *Builder) commit(dispatchState *dispatchState, comment string) error {
return b.commitContainer(dispatchState, id, runConfigWithCommentCmd)
}
// TODO: see if any args can be dropped
func (b *Builder) commitContainer(dispatchState *dispatchState, id string, containerConfig *container.Config) error {
if b.disableCommit {
return nil
@ -60,7 +60,47 @@ func (b *Builder) commitContainer(dispatchState *dispatchState, id string, conta
}
dispatchState.imageID = imageID
b.buildStages.update(imageID, dispatchState.runConfig)
b.buildStages.update(imageID)
return nil
}
func (b *Builder) exportImage(state *dispatchState, imageMount *imageMount, runConfig *container.Config) error {
newLayer, err := imageMount.Layer().Commit(b.platform)
if err != nil {
return err
}
// add an image mount without an image so the layer is properly unmounted
// if there is an error before we can add the full mount with image
b.imageSources.Add(newImageMount(nil, newLayer))
parentImage, ok := imageMount.Image().(*image.Image)
if !ok {
return errors.Errorf("unexpected image type")
}
newImage := image.NewChildImage(parentImage, image.ChildConfig{
Author: state.maintainer,
ContainerConfig: runConfig,
DiffID: newLayer.DiffID(),
Config: copyRunConfig(state.runConfig),
}, parentImage.OS)
// TODO: it seems strange to marshal this here instead of just passing in the
// image struct
config, err := newImage.MarshalJSON()
if err != nil {
return errors.Wrap(err, "failed to encode image config")
}
exportedImage, err := b.docker.CreateImage(config, state.imageID, parentImage.OS)
if err != nil {
return errors.Wrapf(err, "failed to export image")
}
state.imageID = exportedImage.ImageID()
b.imageSources.Add(newImageMount(exportedImage, newLayer))
b.buildStages.update(state.imageID)
return nil
}
@ -70,25 +110,47 @@ func (b *Builder) performCopy(state *dispatchState, inst copyInstruction) error
// TODO: should this have been using origPaths instead of srcHash in the comment?
runConfigWithCommentCmd := copyRunConfig(
state.runConfig,
withCmdCommentString(fmt.Sprintf("%s %s in %s ", inst.cmdName, srcHash, inst.dest)))
containerID, err := b.probeAndCreate(state, runConfigWithCommentCmd)
if err != nil || containerID == "" {
withCmdCommentString(fmt.Sprintf("%s %s in %s ", inst.cmdName, srcHash, inst.dest), b.platform))
hit, err := b.probeCache(state, runConfigWithCommentCmd)
if err != nil || hit {
return err
}
// Twiddle the destination when it's a relative path - meaning, make it
// relative to the WORKINGDIR
dest, err := normaliseDest(inst.cmdName, state.runConfig.WorkingDir, inst.dest)
imageMount, err := b.imageSources.Get(state.imageID, true)
if err != nil {
return errors.Wrapf(err, "failed to get destination image %q", state.imageID)
}
destInfo, err := createDestInfo(state.runConfig.WorkingDir, inst, imageMount)
if err != nil {
return err
}
opts := copyFileOptions{
decompress: inst.allowLocalDecompression,
archiver: b.archiver,
}
for _, info := range inst.infos {
if err := b.docker.CopyOnBuild(containerID, dest, info.root, info.path, inst.allowLocalDecompression); err != nil {
return err
if err := performCopyForInfo(destInfo, info, opts); err != nil {
return errors.Wrapf(err, "failed to copy files")
}
}
return b.commitContainer(state, containerID, runConfigWithCommentCmd)
return b.exportImage(state, imageMount, runConfigWithCommentCmd)
}
func createDestInfo(workingDir string, inst copyInstruction, imageMount *imageMount) (copyInfo, error) {
// Twiddle the destination when it's a relative path - meaning, make it
// relative to the WORKINGDIR
dest, err := normaliseDest(workingDir, inst.dest)
if err != nil {
return copyInfo{}, errors.Wrapf(err, "invalid %s", inst.cmdName)
}
destMount, err := imageMount.Source()
if err != nil {
return copyInfo{}, errors.Wrapf(err, "failed to mount copy source")
}
return newCopyInfoFromSource(destMount, dest, ""), nil
}
// For backwards compat, if there's just one info then use it as the
@ -128,9 +190,9 @@ func withCmd(cmd []string) runConfigModifier {
// withCmdComment sets Cmd to a nop comment string. See withCmdCommentString for
// why there are two almost identical versions of this.
func withCmdComment(comment string) runConfigModifier {
func withCmdComment(comment string, platform string) runConfigModifier {
return func(runConfig *container.Config) {
runConfig.Cmd = append(getShell(runConfig), "#(nop) ", comment)
runConfig.Cmd = append(getShell(runConfig, platform), "#(nop) ", comment)
}
}
@ -138,9 +200,9 @@ func withCmdComment(comment string) runConfigModifier {
// A few instructions (workdir, copy, add) used a nop comment that is a single arg
// where as all the other instructions used a two arg comment string. This
// function implements the single arg version.
func withCmdCommentString(comment string) runConfigModifier {
func withCmdCommentString(comment string, platform string) runConfigModifier {
return func(runConfig *container.Config) {
runConfig.Cmd = append(getShell(runConfig), "#(nop) "+comment)
runConfig.Cmd = append(getShell(runConfig, platform), "#(nop) "+comment)
}
}
@ -167,9 +229,9 @@ func withEntrypointOverride(cmd []string, entrypoint []string) runConfigModifier
// getShell is a helper function which gets the right shell for prefixing the
// shell-form of RUN, ENTRYPOINT and CMD instructions
func getShell(c *container.Config) []string {
func getShell(c *container.Config, platform string) []string {
if 0 == len(c.Shell) {
return append([]string{}, defaultShell[:]...)
return append([]string{}, defaultShellForPlatform(platform)[:]...)
}
return append([]string{}, c.Shell[:]...)
}
@ -182,7 +244,7 @@ func (b *Builder) probeCache(dispatchState *dispatchState, runConfig *container.
fmt.Fprint(b.Stdout, " ---> Using cache\n")
dispatchState.imageID = string(cachedID)
b.buildStages.update(dispatchState.imageID, runConfig)
b.buildStages.update(dispatchState.imageID)
return true, nil
}
@ -194,13 +256,13 @@ func (b *Builder) probeAndCreate(dispatchState *dispatchState, runConfig *contai
}
// Set a log config to override any default value set on the daemon
hostConfig := &container.HostConfig{LogConfig: defaultLogConfig}
container, err := b.containerManager.Create(runConfig, hostConfig)
container, err := b.containerManager.Create(runConfig, hostConfig, b.platform)
return container.ID, err
}
func (b *Builder) create(runConfig *container.Config) (string, error) {
hostConfig := hostConfigFromOptions(b.options)
container, err := b.containerManager.Create(runConfig, hostConfig)
container, err := b.containerManager.Create(runConfig, hostConfig, b.platform)
if err != nil {
return "", err
}

5
components/engine/builder/dockerfile/internals_test.go
View File

@ -2,6 +2,7 @@ package dockerfile
import (
"fmt"
"runtime"
"testing"
"github.com/docker/docker/api/types"
@ -97,9 +98,9 @@ func TestCopyRunConfig(t *testing.T) {
},
{
doc: "Set the command to a comment",
modifiers: []runConfigModifier{withCmdComment("comment")},
modifiers: []runConfigModifier{withCmdComment("comment", runtime.GOOS)},
expected: &container.Config{
Cmd: append(defaultShell, "#(nop) ", "comment"),
Cmd: append(defaultShellForPlatform(runtime.GOOS), "#(nop) ", "comment"),
Env: defaultEnv,
},
},

2
components/engine/builder/dockerfile/internals_unix.go
View File

@ -12,7 +12,7 @@ import (
// normaliseDest normalises the destination of a COPY/ADD command in a
// platform semantically consistent way.
func normaliseDest(cmdName, workingDir, requested string) (string, error) {
func normaliseDest(workingDir, requested string) (string, error) {
dest := filepath.FromSlash(requested)
endsInSlash := strings.HasSuffix(requested, string(os.PathSeparator))
if !system.IsAbs(requested) {

6
components/engine/builder/dockerfile/internals_windows.go
View File

@ -12,7 +12,7 @@ import (
// normaliseDest normalises the destination of a COPY/ADD command in a
// platform semantically consistent way.
func normaliseDest(cmdName, workingDir, requested string) (string, error) {
func normaliseDest(workingDir, requested string) (string, error) {
dest := filepath.FromSlash(requested)
endsInSlash := strings.HasSuffix(dest, string(os.PathSeparator))
@ -32,7 +32,7 @@ func normaliseDest(cmdName, workingDir, requested string) (string, error) {
// we only want to validate where the DriveColon part has been supplied.
if filepath.IsAbs(dest) {
if strings.ToUpper(string(dest[0])) != "C" {
return "", fmt.Errorf("Windows does not support %s with a destinations not on the system drive (C:)", cmdName)
return "", fmt.Errorf("Windows does not support destinations not on the system drive (C:)")
}
dest = dest[2:] // Strip the drive letter
}
@ -44,7 +44,7 @@ func normaliseDest(cmdName, workingDir, requested string) (string, error) {
}
if !system.IsAbs(dest) {
if string(workingDir[0]) != "C" {
return "", fmt.Errorf("Windows does not support %s with relative paths when WORKDIR is not the system drive", cmdName)
return "", fmt.Errorf("Windows does not support relative paths when WORKDIR is not the system drive")
}
dest = filepath.Join(string(os.PathSeparator), workingDir[2:], dest)
// Make sure we preserve any trailing slash

36
components/engine/builder/dockerfile/internals_windows_test.go
View File

@ -2,16 +2,22 @@
package dockerfile
import "testing"
import (
"fmt"
"testing"
"github.com/docker/docker/pkg/testutil"
"github.com/stretchr/testify/assert"
)
func TestNormaliseDest(t *testing.T) {
tests := []struct{ current, requested, expected, etext string }{
{``, `D:\`, ``, `Windows does not support TEST with a destinations not on the system drive (C:)`},
{``, `e:/`, ``, `Windows does not support TEST with a destinations not on the system drive (C:)`},
{``, `D:\`, ``, `Windows does not support destinations not on the system drive (C:)`},
{``, `e:/`, ``, `Windows does not support destinations not on the system drive (C:)`},
{`invalid`, `./c1`, ``, `Current WorkingDir invalid is not platform consistent`},
{`C:`, ``, ``, `Current WorkingDir C: is not platform consistent`},
{`C`, ``, ``, `Current WorkingDir C is not platform consistent`},
{`D:\`, `.`, ``, "Windows does not support TEST with relative paths when WORKDIR is not the system drive"},
{`D:\`, `.`, ``, "Windows does not support relative paths when WORKDIR is not the system drive"},
{``, `D`, `D`, ``},
{``, `./a1`, `.\a1`, ``},
{``, `.\b1`, `.\b1`, ``},
@ -32,20 +38,16 @@ func TestNormaliseDest(t *testing.T) {
{`C:\wdm`, `foo/bar/`, `\wdm\foo\bar\`, ``},
{`C:\wdn`, `foo\bar/`, `\wdn\foo\bar\`, ``},
}
for _, i := range tests {
got, err := normaliseDest("TEST", i.current, i.requested)
if err != nil && i.etext == "" {
t.Fatalf("TestNormaliseDest Got unexpected error %q for %s %s. ", err.Error(), i.current, i.requested)
}
if i.etext != "" && ((err == nil) || (err != nil && err.Error() != i.etext)) {
if err == nil {
t.Fatalf("TestNormaliseDest Expected an error for %s %s but didn't get one", i.current, i.requested)
} else {
t.Fatalf("TestNormaliseDest Wrong error text for %s %s - %s", i.current, i.requested, err.Error())
for _, testcase := range tests {
msg := fmt.Sprintf("Input: %s, %s", testcase.current, testcase.requested)
actual, err := normaliseDest(testcase.current, testcase.requested)
if testcase.etext == "" {
if !assert.NoError(t, err, msg) {
continue
}
}
if i.etext == "" && got != i.expected {
t.Fatalf("TestNormaliseDest Expected %q for %q and %q. Got %q", i.expected, i.current, i.requested, got)
assert.Equal(t, testcase.expected, actual, msg)
} else {
testutil.ErrorContains(t, err, testcase.etext)
}
}
}

25
components/engine/builder/dockerfile/mockbackend_test.go
View File

@ -1,6 +1,7 @@
package dockerfile
import (
"encoding/json"
"io"
"github.com/docker/docker/api/types"
@ -8,6 +9,7 @@ import (
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/builder"
containerpkg "github.com/docker/docker/container"
"github.com/docker/docker/layer"
"golang.org/x/net/context"
)
@ -16,7 +18,7 @@ type MockBackend struct {
containerCreateFunc func(config types.ContainerCreateConfig) (container.ContainerCreateCreatedBody, error)
commitFunc func(string, *backend.ContainerCommitConfig) (string, error)
getImageFunc func(string) (builder.Image, builder.ReleaseableLayer, error)
makeImageCacheFunc func(cacheFrom []string) builder.ImageCache
makeImageCacheFunc func(cacheFrom []string, platform string) builder.ImageCache
}
func (m *MockBackend) ContainerAttachRaw(cID string, stdin io.ReadCloser, stdout, stderr io.Writer, stream bool, attached chan struct{}) error {
@ -69,13 +71,17 @@ func (m *MockBackend) GetImageAndReleasableLayer(ctx context.Context, refOrID st
return &mockImage{id: "theid"}, &mockLayer{}, nil
}
func (m *MockBackend) MakeImageCache(cacheFrom []string) builder.ImageCache {
func (m *MockBackend) MakeImageCache(cacheFrom []string, platform string) builder.ImageCache {
if m.makeImageCacheFunc != nil {
return m.makeImageCacheFunc(cacheFrom)
return m.makeImageCacheFunc(cacheFrom, platform)
}
return nil
}
func (m *MockBackend) CreateImage(config []byte, parent string, platform string) (builder.Image, error) {
return nil, nil
}
type mockImage struct {
id string
config *container.Config
@ -89,6 +95,11 @@ func (i *mockImage) RunConfig() *container.Config {
return i.config
}
func (i *mockImage) MarshalJSON() ([]byte, error) {
type rawImage mockImage
return json.Marshal(rawImage(*i))
}
type mockImageCache struct {
getCacheFunc func(parentID string, cfg *container.Config) (string, error)
}
@ -109,3 +120,11 @@ func (l *mockLayer) Release() error {
func (l *mockLayer) Mount() (string, error) {
return "mountPath", nil
}
func (l *mockLayer) Commit(string) (builder.ReleaseableLayer, error) {
return nil, nil
}
func (l *mockLayer) DiffID() layer.DiffID {
return layer.DiffID("abcdef")
}

8
components/engine/builder/dockerfile/parser/line_parsers_test.go
View File

@ -64,3 +64,11 @@ func TestNodeFromLabels(t *testing.T) {
assert.Equal(t, expected, node)
}
func TestParseNameValWithoutVal(t *testing.T) {
directive := Directive{}
// In Config.Env, a variable without `=` is removed from the environment. (#31634)
// However, in Dockerfile, we don't allow "unsetting" an environment variable. (#11922)
_, err := parseNameVal("foo", "ENV", &directive)
assert.Error(t, err, "ENV must have two arguments")
}

123
components/engine/builder/dockerfile/parser/parser.go
View File

@ -7,11 +7,13 @@ import (
"fmt"
"io"
"regexp"
"runtime"
"strconv"
"strings"
"unicode"
"github.com/docker/docker/builder/dockerfile/command"
"github.com/docker/docker/pkg/system"
"github.com/pkg/errors"
)
@ -79,22 +81,28 @@ func (node *Node) AddChild(child *Node, startLine, endLine int) {
}
var (
dispatch map[string]func(string, *Directive) (*Node, map[string]bool, error)
tokenWhitespace = regexp.MustCompile(`[\t\v\f\r ]+`)
tokenEscapeCommand = regexp.MustCompile(`^#[ \t]*escape[ \t]*=[ \t]*(?P<escapechar>.).*$`)
tokenComment = regexp.MustCompile(`^#.*$`)
dispatch map[string]func(string, *Directive) (*Node, map[string]bool, error)
tokenWhitespace = regexp.MustCompile(`[\t\v\f\r ]+`)
tokenEscapeCommand = regexp.MustCompile(`^#[ \t]*escape[ \t]*=[ \t]*(?P<escapechar>.).*$`)
tokenPlatformCommand = regexp.MustCompile(`^#[ \t]*platform[ \t]*=[ \t]*(?P<platform>.*)$`)
tokenComment = regexp.MustCompile(`^#.*$`)
)
// DefaultEscapeToken is the default escape token
const DefaultEscapeToken = '\\'
// DefaultPlatformToken is the platform assumed for the build if not explicitly provided
var DefaultPlatformToken = runtime.GOOS
// Directive is the structure used during a build run to hold the state of
// parsing directives.
type Directive struct {
escapeToken rune // Current escape token
platformToken string // Current platform token
lineContinuationRegex *regexp.Regexp // Current line continuation regex
processingComplete bool // Whether we are done looking for directives
escapeSeen bool // Whether the escape directive has been seen
platformSeen bool // Whether the platform directive has been seen
}
// setEscapeToken sets the default token for escaping characters in a Dockerfile.
@ -107,29 +115,61 @@ func (d *Directive) setEscapeToken(s string) error {
return nil
}
// processLine looks for a parser directive '# escapeToken=<char>. Parser
// directives must precede any builder instruction or other comments, and cannot
// be repeated.
func (d *Directive) processLine(line string) error {
// setPlatformToken sets the default platform for pulling images in a Dockerfile.
func (d *Directive) setPlatformToken(s string) error {
s = strings.ToLower(s)
valid := []string{runtime.GOOS}
if runtime.GOOS == "windows" && system.LCOWSupported() {
valid = append(valid, "linux")
}
for _, item := range valid {
if s == item {
d.platformToken = s
return nil
}
}
return fmt.Errorf("invalid PLATFORM '%s'. Must be one of %v", s, valid)
}
// possibleParserDirective looks for one or more parser directives '# escapeToken=<char>' and
// '# platform=<string>'. Parser directives must precede any builder instruction
// or other comments, and cannot be repeated.
func (d *Directive) possibleParserDirective(line string) error {
if d.processingComplete {
return nil
}
// Processing is finished after the first call
defer func() { d.processingComplete = true }()
tecMatch := tokenEscapeCommand.FindStringSubmatch(strings.ToLower(line))
if len(tecMatch) == 0 {
return nil
}
if d.escapeSeen == true {
return errors.New("only one escape parser directive can be used")
}
for i, n := range tokenEscapeCommand.SubexpNames() {
if n == "escapechar" {
d.escapeSeen = true
return d.setEscapeToken(tecMatch[i])
if len(tecMatch) != 0 {
for i, n := range tokenEscapeCommand.SubexpNames() {
if n == "escapechar" {
if d.escapeSeen == true {
return errors.New("only one escape parser directive can be used")
}
d.escapeSeen = true
return d.setEscapeToken(tecMatch[i])
}
}
}
// TODO @jhowardmsft LCOW Support: Eventually this check can be removed,
// but only recognise a platform token if running in LCOW mode.
if runtime.GOOS == "windows" && system.LCOWSupported() {
tpcMatch := tokenPlatformCommand.FindStringSubmatch(strings.ToLower(line))
if len(tpcMatch) != 0 {
for i, n := range tokenPlatformCommand.SubexpNames() {
if n == "platform" {
if d.platformSeen == true {
return errors.New("only one platform parser directive can be used")
}
d.platformSeen = true
return d.setPlatformToken(tpcMatch[i])
}
}
}
}
d.processingComplete = true
return nil
}
@ -137,6 +177,7 @@ func (d *Directive) processLine(line string) error {
func NewDefaultDirective() *Directive {
directive := Directive{}
directive.setEscapeToken(string(DefaultEscapeToken))
directive.setPlatformToken(runtime.GOOS)
return &directive
}
@ -201,6 +242,7 @@ func newNodeFromLine(line string, directive *Directive) (*Node, error) {
type Result struct {
AST *Node
EscapeToken rune
Platform string
}
// Parse reads lines from a Reader, parses the lines into an AST and returns
@ -213,34 +255,36 @@ func Parse(rwc io.Reader) (*Result, error) {
var err error
for scanner.Scan() {
bytes := scanner.Bytes()
switch currentLine {
case 0:
bytes, err = processFirstLine(d, bytes)
if err != nil {
return nil, err
}
default:
bytes = processLine(bytes, true)
bytesRead := scanner.Bytes()
if currentLine == 0 {
// First line, strip the byte-order-marker if present
bytesRead = bytes.TrimPrefix(bytesRead, utf8bom)
}
bytesRead, err = processLine(d, bytesRead, true)
if err != nil {
return nil, err
}
currentLine++
startLine := currentLine
line, isEndOfLine := trimContinuationCharacter(string(bytes), d)
line, isEndOfLine := trimContinuationCharacter(string(bytesRead), d)
if isEndOfLine && line == "" {
continue
}
for !isEndOfLine && scanner.Scan() {
bytes := processLine(scanner.Bytes(), false)
bytesRead, err := processLine(d, scanner.Bytes(), false)
if err != nil {
return nil, err
}
currentLine++
// TODO: warn this is being deprecated/removed
if isEmptyContinuationLine(bytes) {
if isEmptyContinuationLine(bytesRead) {
continue
}
continuationLine := string(bytes)
continuationLine := string(bytesRead)
continuationLine, isEndOfLine = trimContinuationCharacter(continuationLine, d)
line += continuationLine
}
@ -251,8 +295,7 @@ func Parse(rwc io.Reader) (*Result, error) {
}
root.AddChild(child, startLine, currentLine)
}
return &Result{AST: root, EscapeToken: d.escapeToken}, nil
return &Result{AST: root, EscapeToken: d.escapeToken, Platform: d.platformToken}, nil
}
func trimComments(src []byte) []byte {
@ -279,16 +322,10 @@ func trimContinuationCharacter(line string, d *Directive) (string, bool) {
// TODO: remove stripLeftWhitespace after deprecation period. It seems silly
// to preserve whitespace on continuation lines. Why is that done?
func processLine(token []byte, stripLeftWhitespace bool) []byte {
func processLine(d *Directive, token []byte, stripLeftWhitespace bool) ([]byte, error) {
if stripLeftWhitespace {
token = trimWhitespace(token)
}
return trimComments(token)
}
func processFirstLine(d *Directive, token []byte) ([]byte, error) {
token = bytes.TrimPrefix(token, utf8bom)
token = trimWhitespace(token)
err := d.processLine(string(token))
err := d.possibleParserDirective(string(token))
return trimComments(token), err
}

19
components/engine/builder/dockerignore/dockerignore.go
View File

@ -38,8 +38,23 @@ func ReadAll(reader io.Reader) ([]string, error) {
if pattern == "" {
continue
}
pattern = filepath.Clean(pattern)
pattern = filepath.ToSlash(pattern)
// normalize absolute paths to paths relative to the context
// (taking care of '!' prefix)
invert := pattern[0] == '!'
if invert {
pattern = strings.TrimSpace(pattern[1:])
}
if len(pattern) > 0 {
pattern = filepath.Clean(pattern)
pattern = filepath.ToSlash(pattern)
if len(pattern) > 1 && pattern[0] == '/' {
pattern = pattern[1:]
}
}
if invert {
pattern = "!" + pattern
}
excludes = append(excludes, pattern)
}
if err := scanner.Err(); err != nil {

22
components/engine/builder/dockerignore/dockerignore_test.go
View File

@ -25,7 +25,7 @@ func TestReadAll(t *testing.T) {
}
diName := filepath.Join(tmpDir, ".dockerignore")
content := fmt.Sprintf("test1\n/test2\n/a/file/here\n\nlastfile")
content := fmt.Sprintf("test1\n/test2\n/a/file/here\n\nlastfile\n# this is a comment\n! /inverted/abs/path\n!\n! \n")
err = ioutil.WriteFile(diName, []byte(content), 0777)
if err != nil {
t.Fatal(err)
@ -42,16 +42,28 @@ func TestReadAll(t *testing.T) {
t.Fatal(err)
}
if len(di) != 7 {
t.Fatalf("Expected 5 entries, got %v", len(di))
}
if di[0] != "test1" {
t.Fatal("First element is not test1")
}
if di[1] != "/test2" {
t.Fatal("Second element is not /test2")
if di[1] != "test2" { // according to https://docs.docker.com/engine/reference/builder/#dockerignore-file, /foo/bar should be treated as foo/bar
t.Fatal("Second element is not test2")
}
if di[2] != "/a/file/here" {
t.Fatal("Third element is not /a/file/here")
if di[2] != "a/file/here" { // according to https://docs.docker.com/engine/reference/builder/#dockerignore-file, /foo/bar should be treated as foo/bar
t.Fatal("Third element is not a/file/here")
}
if di[3] != "lastfile" {
t.Fatal("Fourth element is not lastfile")
}
if di[4] != "!inverted/abs/path" {
t.Fatal("Fifth element is not !inverted/abs/path")
}
if di[5] != "!" {
t.Fatalf("Sixth element is not !, but %s", di[5])
}
if di[6] != "!" {
t.Fatalf("Sixth element is not !, but %s", di[6])
}
}

602
components/engine/builder/fscache/fscache.go Normal file
View File

@ -0,0 +1,602 @@
package fscache
import (
"encoding/json"
"os"
"path/filepath"
"sort"
"sync"
"time"
"github.com/Sirupsen/logrus"
"github.com/boltdb/bolt"
"github.com/docker/docker/builder"
"github.com/docker/docker/builder/remotecontext"
"github.com/docker/docker/client/session/filesync"
"github.com/docker/docker/pkg/directory"
"github.com/docker/docker/pkg/stringid"
"github.com/pkg/errors"
"github.com/tonistiigi/fsutil"
"golang.org/x/net/context"
"golang.org/x/sync/singleflight"
)
const dbFile = "fscache.db"
const cacheKey = "cache"
const metaKey = "meta"
// Backend is a backing implementation for FSCache
type Backend interface {
Get(id string) (string, error)
Remove(id string) error
}
// FSCache allows syncing remote resources to cached snapshots
type FSCache struct {
opt Opt
transports map[string]Transport
mu sync.Mutex
g singleflight.Group
store *fsCacheStore
}
// Opt defines options for initializing FSCache
type Opt struct {
Backend Backend
Root string // for storing local metadata
GCPolicy GCPolicy
}
// GCPolicy defines policy for garbage collection
type GCPolicy struct {
MaxSize uint64
MaxKeepDuration time.Duration
}
// NewFSCache returns new FSCache object
func NewFSCache(opt Opt) (*FSCache, error) {
store, err := newFSCacheStore(opt)
if err != nil {
return nil, err
}
return &FSCache{
store: store,
opt: opt,
transports: make(map[string]Transport),
}, nil
}
// Transport defines a method for syncing remote data to FSCache
type Transport interface {
Copy(ctx context.Context, id RemoteIdentifier, dest string, cs filesync.CacheUpdater) error
}
// RemoteIdentifier identifies a transfer request
type RemoteIdentifier interface {
Key() string
SharedKey() string
Transport() string
}
// RegisterTransport registers a new transport method
func (fsc *FSCache) RegisterTransport(id string, transport Transport) error {
fsc.mu.Lock()
defer fsc.mu.Unlock()
if _, ok := fsc.transports[id]; ok {
return errors.Errorf("transport %v already exists", id)
}
fsc.transports[id] = transport
return nil
}
// SyncFrom returns a source based on a remote identifier
func (fsc *FSCache) SyncFrom(ctx context.Context, id RemoteIdentifier) (builder.Source, error) { // cacheOpt
trasportID := id.Transport()
fsc.mu.Lock()
transport, ok := fsc.transports[id.Transport()]
if !ok {
fsc.mu.Unlock()
return nil, errors.Errorf("invalid transport %s", trasportID)
}
logrus.Debugf("SyncFrom %s %s", id.Key(), id.SharedKey())
fsc.mu.Unlock()
sourceRef, err, _ := fsc.g.Do(id.Key(), func() (interface{}, error) {
var sourceRef *cachedSourceRef
sourceRef, err := fsc.store.Get(id.Key())
if err == nil {
return sourceRef, nil
}
// check for unused shared cache
sharedKey := id.SharedKey()
if sharedKey != "" {
r, err := fsc.store.Rebase(sharedKey, id.Key())
if err == nil {
sourceRef = r
}
}
if sourceRef == nil {
var err error
sourceRef, err = fsc.store.New(id.Key(), sharedKey)
if err != nil {
return nil, errors.Wrap(err, "failed to create remote context")
}
}
if err := syncFrom(ctx, sourceRef, transport, id); err != nil {
sourceRef.Release()
return nil, err
}
if err := sourceRef.resetSize(-1); err != nil {
return nil, err
}
return sourceRef, nil
})
if err != nil {
return nil, err
}
ref := sourceRef.(*cachedSourceRef)
if ref.src == nil { // failsafe
return nil, errors.Errorf("invalid empty pull")
}
wc := &wrappedContext{Source: ref.src, closer: func() error {
ref.Release()
return nil
}}
return wc, nil
}
// DiskUsage reports how much data is allocated by the cache
func (fsc *FSCache) DiskUsage() (int64, error) {
return fsc.store.DiskUsage()
}
// Prune allows manually cleaning up the cache
func (fsc *FSCache) Prune() (uint64, error) {
return fsc.store.Prune()
}
// Close stops the gc and closes the persistent db
func (fsc *FSCache) Close() error {
return fsc.store.Close()
}
func syncFrom(ctx context.Context, cs *cachedSourceRef, transport Transport, id RemoteIdentifier) (retErr error) {
src := cs.src
if src == nil {
src = remotecontext.NewCachableSource(cs.Dir())
}
if !cs.cached {
if err := cs.storage.db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(id.Key()))
dt := b.Get([]byte(cacheKey))
if dt != nil {
if err := src.UnmarshalBinary(dt); err != nil {
return err
}
} else {
return errors.Wrap(src.Scan(), "failed to scan cache records")
}
return nil
}); err != nil {
return err
}
}
dc := &detectChanges{f: src.HandleChange}
// todo: probably send a bucket to `Copy` and let it return source
// but need to make sure that tx is safe
if err := transport.Copy(ctx, id, cs.Dir(), dc); err != nil {
return errors.Wrapf(err, "failed to copy to %s", cs.Dir())
}
if !dc.supported {
if err := src.Scan(); err != nil {
return errors.Wrap(err, "failed to scan cache records after transfer")
}
}
cs.cached = true
cs.src = src
return cs.storage.db.Update(func(tx *bolt.Tx) error {
dt, err := src.MarshalBinary()
if err != nil {
return err
}
b := tx.Bucket([]byte(id.Key()))
return b.Put([]byte(cacheKey), dt)
})
}
type fsCacheStore struct {
root string
mu sync.Mutex
sources map[string]*cachedSource
db *bolt.DB
fs Backend
gcTimer *time.Timer
gcPolicy GCPolicy
}
// CachePolicy defines policy for keeping a resource in cache
type CachePolicy struct {
Priority int
LastUsed time.Time
}
func defaultCachePolicy() CachePolicy {
return CachePolicy{Priority: 10, LastUsed: time.Now()}
}
func newFSCacheStore(opt Opt) (*fsCacheStore, error) {
if err := os.MkdirAll(opt.Root, 0700); err != nil {
return nil, err
}
p := filepath.Join(opt.Root, dbFile)
db, err := bolt.Open(p, 0600, nil)
if err != nil {
return nil, errors.Wrap(err, "failed to open database file %s")
}
s := &fsCacheStore{db: db, sources: make(map[string]*cachedSource), fs: opt.Backend, gcPolicy: opt.GCPolicy}
db.View(func(tx *bolt.Tx) error {
return tx.ForEach(func(name []byte, b *bolt.Bucket) error {
dt := b.Get([]byte(metaKey))
if dt == nil {
return nil
}
var sm sourceMeta
if err := json.Unmarshal(dt, &sm); err != nil {
return err
}
dir, err := s.fs.Get(sm.BackendID)
if err != nil {
return err // TODO: handle gracefully
}
source := &cachedSource{
refs: make(map[*cachedSourceRef]struct{}),
id: string(name),
dir: dir,
sourceMeta: sm,
storage: s,
}
s.sources[string(name)] = source
return nil
})
})
s.gcTimer = s.startPeriodicGC(5 * time.Minute)
return s, nil
}
func (s *fsCacheStore) startPeriodicGC(interval time.Duration) *time.Timer {
var t *time.Timer
t = time.AfterFunc(interval, func() {
if err := s.GC(); err != nil {
logrus.Errorf("build gc error: %v", err)
}
t.Reset(interval)
})
return t
}
func (s *fsCacheStore) Close() error {
s.gcTimer.Stop()
return s.db.Close()
}
func (s *fsCacheStore) New(id, sharedKey string) (*cachedSourceRef, error) {
s.mu.Lock()
defer s.mu.Unlock()
var ret *cachedSource
if err := s.db.Update(func(tx *bolt.Tx) error {
b, err := tx.CreateBucket([]byte(id))
if err != nil {
return err
}
backendID := stringid.GenerateRandomID()
dir, err := s.fs.Get(backendID)
if err != nil {
return err
}
source := &cachedSource{
refs: make(map[*cachedSourceRef]struct{}),
id: id,
dir: dir,
sourceMeta: sourceMeta{
BackendID: backendID,
SharedKey: sharedKey,
CachePolicy: defaultCachePolicy(),
},
storage: s,
}
dt, err := json.Marshal(source.sourceMeta)
if err != nil {
return err
}
if err := b.Put([]byte(metaKey), dt); err != nil {
return err
}
s.sources[id] = source
ret = source
return nil
}); err != nil {
return nil, err
}
return ret.getRef(), nil
}
func (s *fsCacheStore) Rebase(sharedKey, newid string) (*cachedSourceRef, error) {
s.mu.Lock()
defer s.mu.Unlock()
var ret *cachedSource
for id, snap := range s.sources {
if snap.SharedKey == sharedKey && len(snap.refs) == 0 {
if err := s.db.Update(func(tx *bolt.Tx) error {
if err := tx.DeleteBucket([]byte(id)); err != nil {
return err
}
b, err := tx.CreateBucket([]byte(newid))
if err != nil {
return err
}
snap.id = newid
snap.CachePolicy = defaultCachePolicy()
dt, err := json.Marshal(snap.sourceMeta)
if err != nil {
return err
}
if err := b.Put([]byte(metaKey), dt); err != nil {
return err
}
delete(s.sources, id)
s.sources[newid] = snap
return nil
}); err != nil {
return nil, err
}
ret = snap
break
}
}
if ret == nil {
return nil, errors.Errorf("no candidate for rebase")
}
return ret.getRef(), nil
}
func (s *fsCacheStore) Get(id string) (*cachedSourceRef, error) {
s.mu.Lock()
defer s.mu.Unlock()
src, ok := s.sources[id]
if !ok {
return nil, errors.Errorf("not found")
}
return src.getRef(), nil
}
// DiskUsage reports how much data is allocated by the cache
func (s *fsCacheStore) DiskUsage() (int64, error) {
s.mu.Lock()
defer s.mu.Unlock()
var size int64
for _, snap := range s.sources {
if len(snap.refs) == 0 {
ss, err := snap.getSize()
if err != nil {
return 0, err
}
size += ss
}
}
return size, nil
}
// Prune allows manually cleaning up the cache
func (s *fsCacheStore) Prune() (uint64, error) {
s.mu.Lock()
defer s.mu.Unlock()
var size uint64
for id, snap := range s.sources {
if len(snap.refs) == 0 {
ss, err := snap.getSize()
if err != nil {
return size, err
}
if err := s.delete(id); err != nil {
return size, errors.Wrapf(err, "failed to delete %s", id)
}
size += uint64(ss)
}
}
return size, nil
}
// GC runs a garbage collector on FSCache
func (s *fsCacheStore) GC() error {
s.mu.Lock()
defer s.mu.Unlock()
var size uint64
cutoff := time.Now().Add(-s.gcPolicy.MaxKeepDuration)
var blacklist []*cachedSource
for id, snap := range s.sources {
if len(snap.refs) == 0 {
if cutoff.After(snap.CachePolicy.LastUsed) {
if err := s.delete(id); err != nil {
return errors.Wrapf(err, "failed to delete %s", id)
}
} else {
ss, err := snap.getSize()
if err != nil {
return err
}
size += uint64(ss)
blacklist = append(blacklist, snap)
}
}
}
sort.Sort(sortableCacheSources(blacklist))
for _, snap := range blacklist {
if size <= s.gcPolicy.MaxSize {
break
}
ss, err := snap.getSize()
if err != nil {
return err
}
if err := s.delete(snap.id); err != nil {
return errors.Wrapf(err, "failed to delete %s", snap.id)
}
size -= uint64(ss)
}
return nil
}
// keep mu while calling this
func (s *fsCacheStore) delete(id string) error {
src, ok := s.sources[id]
if !ok {
return nil
}
if len(src.refs) > 0 {
return errors.Errorf("can't delete %s because it has active references", id)
}
delete(s.sources, id)
if err := s.db.Update(func(tx *bolt.Tx) error {
return tx.DeleteBucket([]byte(id))
}); err != nil {
return err
}
if err := s.fs.Remove(src.BackendID); err != nil {
return err
}
return nil
}
type sourceMeta struct {
SharedKey string
BackendID string
CachePolicy CachePolicy
Size int64
}
type cachedSource struct {
sourceMeta
refs map[*cachedSourceRef]struct{}
id string
dir string
src *remotecontext.CachableSource
storage *fsCacheStore
cached bool // keep track if cache is up to date
}
type cachedSourceRef struct {
*cachedSource
}
func (cs *cachedSource) Dir() string {
return cs.dir
}
// hold storage lock before calling
func (cs *cachedSource) getRef() *cachedSourceRef {
ref := &cachedSourceRef{cachedSource: cs}
cs.refs[ref] = struct{}{}
return ref
}
// hold storage lock before calling
func (cs *cachedSource) getSize() (int64, error) {
if cs.sourceMeta.Size < 0 {
ss, err := directory.Size(cs.dir)
if err != nil {
return 0, err
}
if err := cs.resetSize(ss); err != nil {
return 0, err
}
return ss, nil
}
return cs.sourceMeta.Size, nil
}
func (cs *cachedSource) resetSize(val int64) error {
cs.sourceMeta.Size = val
return cs.saveMeta()
}
func (cs *cachedSource) saveMeta() error {
return cs.storage.db.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(cs.id))
dt, err := json.Marshal(cs.sourceMeta)
if err != nil {
return err
}
return b.Put([]byte(metaKey), dt)
})
}
func (csr *cachedSourceRef) Release() error {
csr.cachedSource.storage.mu.Lock()
defer csr.cachedSource.storage.mu.Unlock()
delete(csr.cachedSource.refs, csr)
if len(csr.cachedSource.refs) == 0 {
go csr.cachedSource.storage.GC()
}
return nil
}
type detectChanges struct {
f fsutil.ChangeFunc
supported bool
}
func (dc *detectChanges) HandleChange(kind fsutil.ChangeKind, path string, fi os.FileInfo, err error) error {
if dc == nil {
return nil
}
return dc.f(kind, path, fi, err)
}
func (dc *detectChanges) MarkSupported(v bool) {
if dc == nil {
return
}
dc.supported = v
}
type wrappedContext struct {
builder.Source
closer func() error
}
func (wc *wrappedContext) Close() error {
if err := wc.Source.Close(); err != nil {
return err
}
return wc.closer()
}
type sortableCacheSources []*cachedSource
// Len is the number of elements in the collection.
func (s sortableCacheSources) Len() int {
return len(s)
}
// Less reports whether the element with
// index i should sort before the element with index j.
func (s sortableCacheSources) Less(i, j int) bool {
return s[i].CachePolicy.LastUsed.Before(s[j].CachePolicy.LastUsed)
}
// Swap swaps the elements with indexes i and j.
func (s sortableCacheSources) Swap(i, j int) {
s[i], s[j] = s[j], s[i]
}

131
components/engine/builder/fscache/fscache_test.go Normal file
View File

@ -0,0 +1,131 @@
package fscache
import (
"io/ioutil"
"os"
"path/filepath"
"testing"
"time"
"github.com/docker/docker/client/session/filesync"
"github.com/stretchr/testify/assert"
"golang.org/x/net/context"
)
func TestFSCache(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "fscache")
assert.Nil(t, err)
defer os.RemoveAll(tmpDir)
backend := NewNaiveCacheBackend(filepath.Join(tmpDir, "backend"))
opt := Opt{
Root: tmpDir,
Backend: backend,
GCPolicy: GCPolicy{MaxSize: 15, MaxKeepDuration: time.Hour},
}
fscache, err := NewFSCache(opt)
assert.Nil(t, err)
defer fscache.Close()
err = fscache.RegisterTransport("test", &testTransport{})
assert.Nil(t, err)
src1, err := fscache.SyncFrom(context.TODO(), &testIdentifier{"foo", "data", "bar"})
assert.Nil(t, err)
dt, err := ioutil.ReadFile(filepath.Join(src1.Root(), "foo"))
assert.Nil(t, err)
assert.Equal(t, string(dt), "data")
// same id doesn't recalculate anything
src2, err := fscache.SyncFrom(context.TODO(), &testIdentifier{"foo", "data2", "bar"})
assert.Nil(t, err)
assert.Equal(t, src1.Root(), src2.Root())
dt, err = ioutil.ReadFile(filepath.Join(src1.Root(), "foo"))
assert.Nil(t, err)
assert.Equal(t, string(dt), "data")
assert.Nil(t, src2.Close())
src3, err := fscache.SyncFrom(context.TODO(), &testIdentifier{"foo2", "data2", "bar"})
assert.Nil(t, err)
assert.NotEqual(t, src1.Root(), src3.Root())
dt, err = ioutil.ReadFile(filepath.Join(src3.Root(), "foo2"))
assert.Nil(t, err)
assert.Equal(t, string(dt), "data2")
s, err := fscache.DiskUsage()
assert.Nil(t, err)
assert.Equal(t, s, int64(0))
assert.Nil(t, src3.Close())
s, err = fscache.DiskUsage()
assert.Nil(t, err)
assert.Equal(t, s, int64(5))
// new upload with the same shared key shoutl overwrite
src4, err := fscache.SyncFrom(context.TODO(), &testIdentifier{"foo3", "data3", "bar"})
assert.Nil(t, err)
assert.NotEqual(t, src1.Root(), src3.Root())
dt, err = ioutil.ReadFile(filepath.Join(src3.Root(), "foo3"))
assert.Nil(t, err)
assert.Equal(t, string(dt), "data3")
assert.Equal(t, src4.Root(), src3.Root())
assert.Nil(t, src4.Close())
s, err = fscache.DiskUsage()
assert.Nil(t, err)
assert.Equal(t, s, int64(10))
// this one goes over the GC limit
src5, err := fscache.SyncFrom(context.TODO(), &testIdentifier{"foo4", "datadata", "baz"})
assert.Nil(t, err)
assert.Nil(t, src5.Close())
// GC happens async
time.Sleep(100 * time.Millisecond)
// only last insertion after GC
s, err = fscache.DiskUsage()
assert.Nil(t, err)
assert.Equal(t, s, int64(8))
// prune deletes everything
released, err := fscache.Prune()
assert.Nil(t, err)
assert.Equal(t, released, uint64(8))
s, err = fscache.DiskUsage()
assert.Nil(t, err)
assert.Equal(t, s, int64(0))
}
type testTransport struct {
}
func (t *testTransport) Copy(ctx context.Context, id RemoteIdentifier, dest string, cs filesync.CacheUpdater) error {
testid := id.(*testIdentifier)
return ioutil.WriteFile(filepath.Join(dest, testid.filename), []byte(testid.data), 0600)
}
type testIdentifier struct {
filename string
data string
sharedKey string
}
func (t *testIdentifier) Key() string {
return t.filename
}
func (t *testIdentifier) SharedKey() string {
return t.sharedKey
}
func (t *testIdentifier) Transport() string {
return "test"
}

28
components/engine/builder/fscache/naivedriver.go Normal file
View File

@ -0,0 +1,28 @@
package fscache
import (
"os"
"path/filepath"
"github.com/pkg/errors"
)
// NewNaiveCacheBackend is a basic backend implementation for fscache
func NewNaiveCacheBackend(root string) Backend {
return &naiveCacheBackend{root: root}
}
type naiveCacheBackend struct {
root string
}
func (tcb *naiveCacheBackend) Get(id string) (string, error) {
d := filepath.Join(tcb.root, id)
if err := os.MkdirAll(d, 0700); err != nil {
return "", errors.Wrapf(err, "failed to create tmp dir for %s", d)
}
return d, nil
}
func (tcb *naiveCacheBackend) Remove(id string) error {
return errors.WithStack(os.RemoveAll(filepath.Join(tcb.root, id)))
}

128
components/engine/builder/remotecontext/archive.go Normal file
View File

@ -0,0 +1,128 @@
package remotecontext
import (
"io"
"os"
"path/filepath"
"github.com/docker/docker/builder"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/chrootarchive"
"github.com/docker/docker/pkg/ioutils"
"github.com/docker/docker/pkg/symlink"
"github.com/docker/docker/pkg/tarsum"
"github.com/pkg/errors"
)
type archiveContext struct {
root string
sums tarsum.FileInfoSums
}
func (c *archiveContext) Close() error {
return os.RemoveAll(c.root)
}
func convertPathError(err error, cleanpath string) error {
if err, ok := err.(*os.PathError); ok {
err.Path = cleanpath
return err
}
return err
}
type modifiableContext interface {
builder.Source
// Remove deletes the entry specified by `path`.
// It is usual for directory entries to delete all its subentries.
Remove(path string) error
}
// FromArchive returns a build source from a tar stream.
//
// It extracts the tar stream to a temporary folder that is deleted as soon as
// the Context is closed.
// As the extraction happens, a tarsum is calculated for every file, and the set of
// all those sums then becomes the source of truth for all operations on this Context.
//
// Closing tarStream has to be done by the caller.
func FromArchive(tarStream io.Reader) (builder.Source, error) {
root, err := ioutils.TempDir("", "docker-builder")
if err != nil {
return nil, err
}
tsc := &archiveContext{root: root}
// Make sure we clean-up upon error. In the happy case the caller
// is expected to manage the clean-up
defer func() {
if err != nil {
tsc.Close()
}
}()
decompressedStream, err := archive.DecompressStream(tarStream)
if err != nil {
return nil, err
}
sum, err := tarsum.NewTarSum(decompressedStream, true, tarsum.Version1)
if err != nil {
return nil, err
}
err = chrootarchive.Untar(sum, root, nil)
if err != nil {
return nil, err
}
tsc.sums = sum.GetSums()
return tsc, nil
}
func (c *archiveContext) Root() string {
return c.root
}
func (c *archiveContext) Remove(path string) error {
_, fullpath, err := normalize(path, c.root)
if err != nil {
return err
}
return os.RemoveAll(fullpath)
}
func (c *archiveContext) Hash(path string) (string, error) {
cleanpath, fullpath, err := normalize(path, c.root)
if err != nil {
return "", err
}
rel, err := filepath.Rel(c.root, fullpath)
if err != nil {
return "", convertPathError(err, cleanpath)
}
// Use the checksum of the followed path(not the possible symlink) because
// this is the file that is actually copied.
if tsInfo := c.sums.GetFile(filepath.ToSlash(rel)); tsInfo != nil {
return tsInfo.Sum(), nil
}
// We set sum to path by default for the case where GetFile returns nil.
// The usual case is if relative path is empty.
return path, nil // backwards compat TODO: see if really needed
}
func normalize(path, root string) (cleanPath, fullPath string, err error) {
cleanPath = filepath.Clean(string(os.PathSeparator) + path)[1:]
fullPath, err = symlink.FollowSymlinkInScope(filepath.Join(root, path), root)
if err != nil {
return "", "", errors.Wrapf(err, "forbidden path outside the build context: %s (%s)", path, cleanPath)
}
if _, err := os.Lstat(fullPath); err != nil {
return "", "", errors.WithStack(convertPathError(err, path))
}
return
}

16
components/engine/builder/remotecontext/detect.go
View File

@ -14,12 +14,14 @@ import (
"github.com/docker/docker/builder/dockerfile/parser"
"github.com/docker/docker/builder/dockerignore"
"github.com/docker/docker/pkg/fileutils"
"github.com/docker/docker/pkg/httputils"
"github.com/docker/docker/pkg/symlink"
"github.com/docker/docker/pkg/urlutil"
"github.com/pkg/errors"
)
// ClientSessionRemote is identifier for client-session context transport
const ClientSessionRemote = "client-session"
// Detect returns a context and dockerfile from remote location or local
// archive. progressReader is only used if remoteURL is actually a URL
// (not empty, and not a Git endpoint).
@ -30,6 +32,12 @@ func Detect(config backend.BuildConfig) (remote builder.Source, dockerfile *pars
switch {
case remoteURL == "":
remote, dockerfile, err = newArchiveRemote(config.Source, dockerfilePath)
case remoteURL == ClientSessionRemote:
res, err := parser.Parse(config.Source)
if err != nil {
return nil, nil, err
}
return nil, res, nil
case urlutil.IsGitURL(remoteURL):
remote, dockerfile, err = newGitRemote(remoteURL, dockerfilePath)
case urlutil.IsURL(remoteURL):
@ -42,7 +50,7 @@ func Detect(config backend.BuildConfig) (remote builder.Source, dockerfile *pars
func newArchiveRemote(rc io.ReadCloser, dockerfilePath string) (builder.Source, *parser.Result, error) {
defer rc.Close()
c, err := MakeTarSumContext(rc)
c, err := FromArchive(rc)
if err != nil {
return nil, nil, err
}
@ -82,7 +90,7 @@ func withDockerfileFromContext(c modifiableContext, dockerfilePath string) (buil
}
func newGitRemote(gitURL string, dockerfilePath string) (builder.Source, *parser.Result, error) {
c, err := MakeGitContext(gitURL) // TODO: change this to NewLazyContext
c, err := MakeGitContext(gitURL) // TODO: change this to NewLazySource
if err != nil {
return nil, nil, err
}
@ -93,7 +101,7 @@ func newURLRemote(url string, dockerfilePath string, progressReader func(in io.R
var dockerfile io.ReadCloser
dockerfileFoundErr := errors.New("found-dockerfile")
c, err := MakeRemoteContext(url, map[string]func(io.ReadCloser) (io.ReadCloser, error){
httputils.MimeTypes.TextPlain: func(rc io.ReadCloser) (io.ReadCloser, error) {
mimeTypes.TextPlain: func(rc io.ReadCloser) (io.ReadCloser, error) {
dockerfile = rc
return nil, dockerfileFoundErr
},

13
components/engine/builder/remotecontext/filehash.go
View File

@ -12,10 +12,21 @@ import (
// NewFileHash returns new hash that is used for the builder cache keys
func NewFileHash(path, name string, fi os.FileInfo) (hash.Hash, error) {
hdr, err := archive.FileInfoHeader(path, name, fi)
var link string
if fi.Mode()&os.ModeSymlink != 0 {
var err error
link, err = os.Readlink(path)
if err != nil {
return nil, err
}
}
hdr, err := archive.FileInfoHeader(name, fi, link)
if err != nil {
return nil, err
}
if err := archive.ReadSecurityXattrToTarHeader(path, hdr); err != nil {
return nil, err
}
tsh := &tarsumHash{hdr: hdr, Hash: sha256.New()}
tsh.Reset() // initialize header
return tsh, nil

3
components/engine/builder/remotecontext/generate.go Normal file
View File

@ -0,0 +1,3 @@
package remotecontext
//go:generate protoc --gogoslick_out=. tarsum.proto

6
components/engine/builder/remotecontext/git.go
View File

@ -4,13 +4,13 @@ import (
"os"
"github.com/docker/docker/builder"
"github.com/docker/docker/builder/remotecontext/git"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/gitutils"
)
// MakeGitContext returns a Context from gitURL that is cloned in a temporary directory.
func MakeGitContext(gitURL string) (builder.Source, error) {
root, err := gitutils.Clone(gitURL)
root, err := git.Clone(gitURL)
if err != nil {
return nil, err
}
@ -25,5 +25,5 @@ func MakeGitContext(gitURL string) (builder.Source, error) {
c.Close()
os.RemoveAll(root)
}()
return MakeTarSumContext(c)
return FromArchive(c)
}

2
components/engine/pkg/gitutils/gitutils.go → components/engine/builder/remotecontext/git/gitutils.go
View File

@ -1,4 +1,4 @@
package gitutils
package git
import (
"fmt"

104
components/engine/pkg/gitutils/gitutils_test.go → components/engine/builder/remotecontext/git/gitutils_test.go
View File

@ -1,4 +1,4 @@
package gitutils
package git
import (
"fmt"
@ -8,10 +8,12 @@ import (
"net/url"
"os"
"path/filepath"
"reflect"
"runtime"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestCloneArgsSmartHttp(t *testing.T) {
@ -28,9 +30,7 @@ func TestCloneArgsSmartHttp(t *testing.T) {
args := fetchArgs(serverURL, "master")
exp := []string{"fetch", "--recurse-submodules=yes", "--depth", "1", "origin", "master"}
if !reflect.DeepEqual(args, exp) {
t.Fatalf("Expected %v, got %v", exp, args)
}
assert.Equal(t, exp, args)
}
func TestCloneArgsDumbHttp(t *testing.T) {
@ -46,18 +46,14 @@ func TestCloneArgsDumbHttp(t *testing.T) {
args := fetchArgs(serverURL, "master")
exp := []string{"fetch", "--recurse-submodules=yes", "origin", "master"}
if !reflect.DeepEqual(args, exp) {
t.Fatalf("Expected %v, got %v", exp, args)
}
assert.Equal(t, exp, args)
}
func TestCloneArgsGit(t *testing.T) {
u, _ := url.Parse("git://github.com/docker/docker")
args := fetchArgs(u, "master")
exp := []string{"fetch", "--recurse-submodules=yes", "--depth", "1", "origin", "master"}
if !reflect.DeepEqual(args, exp) {
t.Fatalf("Expected %v, got %v", exp, args)
}
assert.Equal(t, exp, args)
}
func gitGetConfig(name string) string {
@ -72,9 +68,7 @@ func gitGetConfig(name string) string {
func TestCheckoutGit(t *testing.T) {
root, err := ioutil.TempDir("", "docker-build-git-checkout")
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
defer os.RemoveAll(root)
autocrlf := gitGetConfig("core.autocrlf")
@ -89,30 +83,22 @@ func TestCheckoutGit(t *testing.T) {
gitDir := filepath.Join(root, "repo")
_, err = git("init", gitDir)
if err != nil {
t.Fatal(err)
}
require.NoError(t, err)
if _, err = gitWithinDir(gitDir, "config", "user.email", "test@docker.com"); err != nil {
t.Fatal(err)
}
_, err = gitWithinDir(gitDir, "config", "user.email", "test@docker.com")
require.NoError(t, err)
if _, err = gitWithinDir(gitDir, "config", "user.name", "Docker test"); err != nil {
t.Fatal(err)
}
_, err = gitWithinDir(gitDir, "config", "user.name", "Docker test")
require.NoError(t, err)
if err = ioutil.WriteFile(filepath.Join(gitDir, "Dockerfile"), []byte("FROM scratch"), 0644); err != nil {
t.Fatal(err)
}
err = ioutil.WriteFile(filepath.Join(gitDir, "Dockerfile"), []byte("FROM scratch"), 0644)
require.NoError(t, err)
subDir := filepath.Join(gitDir, "subdir")
if err = os.Mkdir(subDir, 0755); err != nil {
t.Fatal(err)
}
require.NoError(t, os.Mkdir(subDir, 0755))
if err = ioutil.WriteFile(filepath.Join(subDir, "Dockerfile"), []byte("FROM scratch\nEXPOSE 5000"), 0644); err != nil {
t.Fatal(err)
}
err = ioutil.WriteFile(filepath.Join(subDir, "Dockerfile"), []byte("FROM scratch\nEXPOSE 5000"), 0644)
require.NoError(t, err)
if runtime.GOOS != "windows" {
if err = os.Symlink("../subdir", filepath.Join(gitDir, "parentlink")); err != nil {
@ -124,37 +110,29 @@ func TestCheckoutGit(t *testing.T) {
}
}
if _, err = gitWithinDir(gitDir, "add", "-A"); err != nil {
t.Fatal(err)
}
_, err = gitWithinDir(gitDir, "add", "-A")
require.NoError(t, err)
if _, err = gitWithinDir(gitDir, "commit", "-am", "First commit"); err != nil {
t.Fatal(err)
}
_, err = gitWithinDir(gitDir, "commit", "-am", "First commit")
require.NoError(t, err)
if _, err = gitWithinDir(gitDir, "checkout", "-b", "test"); err != nil {
t.Fatal(err)
}
_, err = gitWithinDir(gitDir, "checkout", "-b", "test")
require.NoError(t, err)
if err = ioutil.WriteFile(filepath.Join(gitDir, "Dockerfile"), []byte("FROM scratch\nEXPOSE 3000"), 0644); err != nil {
t.Fatal(err)
}
err = ioutil.WriteFile(filepath.Join(gitDir, "Dockerfile"), []byte("FROM scratch\nEXPOSE 3000"), 0644)
require.NoError(t, err)
if err = ioutil.WriteFile(filepath.Join(subDir, "Dockerfile"), []byte("FROM busybox\nEXPOSE 5000"), 0644); err != nil {
t.Fatal(err)
}
err = ioutil.WriteFile(filepath.Join(subDir, "Dockerfile"), []byte("FROM busybox\nEXPOSE 5000"), 0644)
require.NoError(t, err)
if _, err = gitWithinDir(gitDir, "add", "-A"); err != nil {
t.Fatal(err)
}
_, err = gitWithinDir(gitDir, "add", "-A")
require.NoError(t, err)
if _, err = gitWithinDir(gitDir, "commit", "-am", "Branch commit"); err != nil {
t.Fatal(err)
}
_, err = gitWithinDir(gitDir, "commit", "-am", "Branch commit")
require.NoError(t, err)
if _, err = gitWithinDir(gitDir, "checkout", "master"); err != nil {
t.Fatal(err)
}
_, err = gitWithinDir(gitDir, "checkout", "master")
require.NoError(t, err)
type singleCase struct {
frag string
@ -190,21 +168,13 @@ func TestCheckoutGit(t *testing.T) {
ref, subdir := getRefAndSubdir(c.frag)
r, err := checkoutGit(gitDir, ref, subdir)
fail := err != nil
if fail != c.fail {
t.Fatalf("Expected %v failure, error was %v\n", c.fail, err)
}
if c.fail {
assert.Error(t, err)
continue
}
b, err := ioutil.ReadFile(filepath.Join(r, "Dockerfile"))
if err != nil {
t.Fatal(err)
}
if string(b) != c.exp {
t.Fatalf("Expected %v, was %v\n", c.exp, string(b))
}
require.NoError(t, err)
assert.Equal(t, c.exp, string(b))
}
}

18
components/engine/builder/remotecontext/lazycontext.go
View File

@ -12,30 +12,30 @@ import (
"github.com/pkg/errors"
)
// NewLazyContext creates a new LazyContext. LazyContext defines a hashed build
// NewLazySource creates a new LazyContext. LazyContext defines a hashed build
// context based on a root directory. Individual files are hashed first time
// they are asked. It is not safe to call methods of LazyContext concurrently.
func NewLazyContext(root string) (builder.Source, error) {
return &lazyContext{
func NewLazySource(root string) (builder.Source, error) {
return &lazySource{
root: root,
sums: make(map[string]string),
}, nil
}
type lazyContext struct {
type lazySource struct {
root string
sums map[string]string
}
func (c *lazyContext) Root() string {
func (c *lazySource) Root() string {
return c.root
}
func (c *lazyContext) Close() error {
func (c *lazySource) Close() error {
return nil
}
func (c *lazyContext) Hash(path string) (string, error) {
func (c *lazySource) Hash(path string) (string, error) {
cleanPath, fullPath, err := normalize(path, c.root)
if err != nil {
return "", err
@ -43,7 +43,7 @@ func (c *lazyContext) Hash(path string) (string, error) {
fi, err := os.Lstat(fullPath)
if err != nil {
return "", err
return "", errors.WithStack(err)
}
relPath, err := Rel(c.root, fullPath)
@ -62,7 +62,7 @@ func (c *lazyContext) Hash(path string) (string, error) {
return sum, nil
}
func (c *lazyContext) prepareHash(relPath string, fi os.FileInfo) (string, error) {
func (c *lazySource) prepareHash(relPath string, fi os.FileInfo) (string, error) {
p := filepath.Join(c.root, relPath)
h, err := NewFileHash(p, relPath, fi)
if err != nil {

12
components/engine/pkg/httputils/mimetype.go → components/engine/builder/remotecontext/mimetype.go
View File

@ -1,29 +1,27 @@
package httputils
package remotecontext
import (
"mime"
"net/http"
)
// MimeTypes stores the MIME content type.
var MimeTypes = struct {
// mimeTypes stores the MIME content type.
var mimeTypes = struct {
TextPlain string
OctetStream string
}{"text/plain", "application/octet-stream"}
// DetectContentType returns a best guess representation of the MIME
// detectContentType returns a best guess representation of the MIME
// content type for the bytes at c. The value detected by
// http.DetectContentType is guaranteed not be nil, defaulting to
// application/octet-stream when a better guess cannot be made. The
// result of this detection is then run through mime.ParseMediaType()
// which separates the actual MIME string from any parameters.
func DetectContentType(c []byte) (string, map[string]string, error) {
func detectContentType(c []byte) (string, map[string]string, error) {
ct := http.DetectContentType(c)
contentType, args, err := mime.ParseMediaType(ct)
if err != nil {
return "", nil, err
}
return contentType, args, nil
}

16
components/engine/builder/remotecontext/mimetype_test.go Normal file
View File

@ -0,0 +1,16 @@
package remotecontext
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestDetectContentType(t *testing.T) {
input := []byte("That is just a plain text")
contentType, _, err := detectContentType(input)
require.NoError(t, err)
assert.Equal(t, "text/plain", contentType)
}

32
components/engine/builder/remotecontext/remote.go
View File

@ -2,14 +2,14 @@ package remotecontext
import (
"bytes"
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"regexp"
"github.com/docker/docker/builder"
"github.com/docker/docker/pkg/httputils"
"github.com/pkg/errors"
)
// When downloading remote contexts, limit the amount (in bytes)
@ -28,9 +28,9 @@ var mimeRe = regexp.MustCompile(acceptableRemoteMIME)
//
// If a match is found, then the body is sent to the contentType handler and a (potentially compressed) tar stream is expected
// to be returned. If no match is found, it is assumed the body is a tar stream (compressed or not).
// In either case, an (assumed) tar stream is passed to MakeTarSumContext whose result is returned.
// In either case, an (assumed) tar stream is passed to FromArchive whose result is returned.
func MakeRemoteContext(remoteURL string, contentTypeHandlers map[string]func(io.ReadCloser) (io.ReadCloser, error)) (builder.Source, error) {
f, err := httputils.Download(remoteURL)
f, err := GetWithStatusError(remoteURL)
if err != nil {
return nil, fmt.Errorf("error downloading remote context %s: %v", remoteURL, err)
}
@ -63,7 +63,25 @@ func MakeRemoteContext(remoteURL string, contentTypeHandlers map[string]func(io.
// Pass through - this is a pre-packaged context, presumably
// with a Dockerfile with the right name inside it.
return MakeTarSumContext(contextReader)
return FromArchive(contextReader)
}
// GetWithStatusError does an http.Get() and returns an error if the
// status code is 4xx or 5xx.
func GetWithStatusError(url string) (resp *http.Response, err error) {
if resp, err = http.Get(url); err != nil {
return nil, err
}
if resp.StatusCode < 400 {
return resp, nil
}
msg := fmt.Sprintf("failed to GET %s with status %s", url, resp.Status)
body, err := ioutil.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
return nil, errors.Wrapf(err, msg+": error reading body")
}
return nil, errors.Errorf(msg+": %s", bytes.TrimSpace(body))
}
// inspectResponse looks into the http response data at r to determine whether its
@ -94,8 +112,8 @@ func inspectResponse(ct string, r io.ReadCloser, clen int64) (string, io.ReadClo
// content type for files without an extension (e.g. 'Dockerfile')
// so if we receive this value we better check for text content
contentType := ct
if len(ct) == 0 || ct == httputils.MimeTypes.OctetStream {
contentType, _, err = httputils.DetectContentType(preamble)
if len(ct) == 0 || ct == mimeTypes.OctetStream {
contentType, _, err = detectContentType(preamble)
if err != nil {
return contentType, bodyReader, err
}

69
components/engine/builder/remotecontext/remote_test.go
View File

@ -11,7 +11,9 @@ import (
"github.com/docker/docker/builder"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/httputils"
"github.com/docker/docker/pkg/testutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
var binaryContext = []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00} //xz magic
@ -188,7 +190,7 @@ func TestMakeRemoteContext(t *testing.T) {
mux.Handle("/", http.FileServer(http.Dir(contextDir)))
remoteContext, err := MakeRemoteContext(remoteURL, map[string]func(io.ReadCloser) (io.ReadCloser, error){
httputils.MimeTypes.TextPlain: func(rc io.ReadCloser) (io.ReadCloser, error) {
mimeTypes.TextPlain: func(rc io.ReadCloser) (io.ReadCloser, error) {
dockerfile, err := ioutil.ReadAll(rc)
if err != nil {
return nil, err
@ -210,25 +212,52 @@ func TestMakeRemoteContext(t *testing.T) {
t.Fatal("Remote context should not be nil")
}
tarSumCtx, ok := remoteContext.(*tarSumContext)
if !ok {
t.Fatal("Cast error, remote context should be casted to tarSumContext")
h, err := remoteContext.Hash(builder.DefaultDockerfileName)
if err != nil {
t.Fatalf("failed to compute hash %s", err)
}
fileInfoSums := tarSumCtx.sums
if fileInfoSums.Len() != 1 {
t.Fatalf("Size of file info sums should be 1, got: %d", fileInfoSums.Len())
}
fileInfo := fileInfoSums.GetFile(builder.DefaultDockerfileName)
if fileInfo == nil {
t.Fatalf("There should be file named %s in fileInfoSums", builder.DefaultDockerfileName)
}
if fileInfo.Pos() != 0 {
t.Fatalf("File %s should have position 0, got %d", builder.DefaultDockerfileName, fileInfo.Pos())
if expected, actual := "7b6b6b66bee9e2102fbdc2228be6c980a2a23adf371962a37286a49f7de0f7cc", h; expected != actual {
t.Fatalf("There should be file named %s %s in fileInfoSums", expected, actual)
}
}
func TestGetWithStatusError(t *testing.T) {
var testcases = []struct {
err error
statusCode int
expectedErr string
expectedBody string
}{
{
statusCode: 200,
expectedBody: "THE BODY",
},
{
statusCode: 400,
expectedErr: "with status 400 Bad Request: broke",
expectedBody: "broke",
},
}
for _, testcase := range testcases {
ts := httptest.NewServer(
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
buffer := bytes.NewBufferString(testcase.expectedBody)
w.WriteHeader(testcase.statusCode)
w.Write(buffer.Bytes())
}),
)
defer ts.Close()
response, err := GetWithStatusError(ts.URL)
if testcase.expectedErr == "" {
require.NoError(t, err)
body, err := testutil.ReadBody(response.Body)
require.NoError(t, err)
assert.Contains(t, string(body), testcase.expectedBody)
} else {
testutil.ErrorContains(t, err, testcase.expectedErr)
}
}
}

226
components/engine/builder/remotecontext/tarsum.go
View File

@ -1,128 +1,174 @@
package remotecontext
import (
"io"
"fmt"
"os"
"path/filepath"
"sync"
"github.com/docker/docker/builder"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/chrootarchive"
"github.com/docker/docker/pkg/ioutils"
"github.com/docker/docker/pkg/symlink"
"github.com/docker/docker/pkg/tarsum"
iradix "github.com/hashicorp/go-immutable-radix"
"github.com/pkg/errors"
"github.com/tonistiigi/fsutil"
)
type tarSumContext struct {
type hashed interface {
Hash() string
}
// CachableSource is a source that contains cache records for its contents
type CachableSource struct {
mu sync.Mutex
root string
sums tarsum.FileInfoSums
tree *iradix.Tree
txn *iradix.Txn
}
func (c *tarSumContext) Close() error {
return os.RemoveAll(c.root)
// NewCachableSource creates new CachableSource
func NewCachableSource(root string) *CachableSource {
ts := &CachableSource{
tree: iradix.New(),
root: root,
}
return ts
}
func convertPathError(err error, cleanpath string) error {
if err, ok := err.(*os.PathError); ok {
err.Path = cleanpath
// MarshalBinary marshals current cache information to a byte array
func (cs *CachableSource) MarshalBinary() ([]byte, error) {
b := TarsumBackup{Hashes: make(map[string]string)}
root := cs.getRoot()
root.Walk(func(k []byte, v interface{}) bool {
b.Hashes[string(k)] = v.(*fileInfo).sum
return false
})
return b.Marshal()
}
// UnmarshalBinary decodes cache information for presented byte array
func (cs *CachableSource) UnmarshalBinary(data []byte) error {
var b TarsumBackup
if err := b.Unmarshal(data); err != nil {
return err
}
return err
}
type modifiableContext interface {
builder.Source
// Remove deletes the entry specified by `path`.
// It is usual for directory entries to delete all its subentries.
Remove(path string) error
}
// MakeTarSumContext returns a build Context from a tar stream.
//
// It extracts the tar stream to a temporary folder that is deleted as soon as
// the Context is closed.
// As the extraction happens, a tarsum is calculated for every file, and the set of
// all those sums then becomes the source of truth for all operations on this Context.
//
// Closing tarStream has to be done by the caller.
func MakeTarSumContext(tarStream io.Reader) (builder.Source, error) {
root, err := ioutils.TempDir("", "docker-builder")
if err != nil {
return nil, err
txn := iradix.New().Txn()
for p, v := range b.Hashes {
txn.Insert([]byte(p), &fileInfo{sum: v})
}
cs.mu.Lock()
defer cs.mu.Unlock()
cs.tree = txn.Commit()
return nil
}
tsc := &tarSumContext{root: root}
// Make sure we clean-up upon error. In the happy case the caller
// is expected to manage the clean-up
defer func() {
// Scan rescans the cache information from the file system
func (cs *CachableSource) Scan() error {
lc, err := NewLazySource(cs.root)
if err != nil {
return err
}
txn := iradix.New().Txn()
err = filepath.Walk(cs.root, func(path string, info os.FileInfo, err error) error {
if err != nil {
tsc.Close()
return errors.Wrapf(err, "failed to walk %s", path)
}
}()
decompressedStream, err := archive.DecompressStream(tarStream)
if err != nil {
return nil, err
}
sum, err := tarsum.NewTarSum(decompressedStream, true, tarsum.Version1)
if err != nil {
return nil, err
}
err = chrootarchive.Untar(sum, root, nil)
if err != nil {
return nil, err
}
tsc.sums = sum.GetSums()
return tsc, nil
}
func (c *tarSumContext) Root() string {
return c.root
}
func (c *tarSumContext) Remove(path string) error {
_, fullpath, err := normalize(path, c.root)
rel, err := Rel(cs.root, path)
if err != nil {
return err
}
h, err := lc.Hash(rel)
if err != nil {
return err
}
txn.Insert([]byte(rel), &fileInfo{sum: h})
return nil
})
if err != nil {
return err
}
return os.RemoveAll(fullpath)
cs.mu.Lock()
defer cs.mu.Unlock()
cs.tree = txn.Commit()
return nil
}
func (c *tarSumContext) Hash(path string) (string, error) {
cleanpath, fullpath, err := normalize(path, c.root)
if err != nil {
return "", err
// HandleChange notifies the source about a modification operation
func (cs *CachableSource) HandleChange(kind fsutil.ChangeKind, p string, fi os.FileInfo, err error) (retErr error) {
cs.mu.Lock()
if cs.txn == nil {
cs.txn = cs.tree.Txn()
}
if kind == fsutil.ChangeKindDelete {
cs.txn.Delete([]byte(p))
cs.mu.Unlock()
return
}
rel, err := filepath.Rel(c.root, fullpath)
if err != nil {
return "", convertPathError(err, cleanpath)
h, ok := fi.(hashed)
if !ok {
cs.mu.Unlock()
return errors.Errorf("invalid fileinfo: %s", p)
}
// Use the checksum of the followed path(not the possible symlink) because
// this is the file that is actually copied.
if tsInfo := c.sums.GetFile(filepath.ToSlash(rel)); tsInfo != nil {
return tsInfo.Sum(), nil
hfi := &fileInfo{
sum: h.Hash(),
}
// We set sum to path by default for the case where GetFile returns nil.
// The usual case is if relative path is empty.
return path, nil // backwards compat TODO: see if really needed
cs.txn.Insert([]byte(p), hfi)
cs.mu.Unlock()
return nil
}
func normalize(path, root string) (cleanPath, fullPath string, err error) {
cleanPath = filepath.Clean(string(os.PathSeparator) + path)[1:]
fullPath, err = symlink.FollowSymlinkInScope(filepath.Join(root, path), root)
if err != nil {
return "", "", errors.Wrapf(err, "forbidden path outside the build context: %s (%s)", path, cleanPath)
func (cs *CachableSource) getRoot() *iradix.Node {
cs.mu.Lock()
if cs.txn != nil {
cs.tree = cs.txn.Commit()
cs.txn = nil
}
if _, err := os.Lstat(fullPath); err != nil {
t := cs.tree
cs.mu.Unlock()
return t.Root()
}
// Close closes the source
func (cs *CachableSource) Close() error {
return nil
}
func (cs *CachableSource) normalize(path string) (cleanpath, fullpath string, err error) {
cleanpath = filepath.Clean(string(os.PathSeparator) + path)[1:]
fullpath, err = symlink.FollowSymlinkInScope(filepath.Join(cs.root, path), cs.root)
if err != nil {
return "", "", fmt.Errorf("Forbidden path outside the context: %s (%s)", path, fullpath)
}
_, err = os.Lstat(fullpath)
if err != nil {
return "", "", convertPathError(err, path)
}
return
}
// Hash returns a hash for a single file in the source
func (cs *CachableSource) Hash(path string) (string, error) {
n := cs.getRoot()
sum := ""
// TODO: check this for symlinks
v, ok := n.Get([]byte(path))
if !ok {
sum = path
} else {
sum = v.(*fileInfo).sum
}
return sum, nil
}
// Root returns a root directory for the source
func (cs *CachableSource) Root() string {
return cs.root
}
type fileInfo struct {
sum string
}
func (fi *fileInfo) Hash() string {
return fi.sum
}

525
components/engine/builder/remotecontext/tarsum.pb.go Normal file
View File

@ -0,0 +1,525 @@
// Code generated by protoc-gen-gogo.
// source: tarsum.proto
// DO NOT EDIT!
/*
Package remotecontext is a generated protocol buffer package.
It is generated from these files:
tarsum.proto
It has these top-level messages:
TarsumBackup
*/
package remotecontext
import proto "github.com/gogo/protobuf/proto"
import fmt "fmt"
import math "math"
import strings "strings"
import reflect "reflect"
import github_com_gogo_protobuf_sortkeys "github.com/gogo/protobuf/sortkeys"
import io "io"
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf
// This is a compile-time assertion to ensure that this generated file
// is compatible with the proto package it is being compiled against.
// A compilation error at this line likely means your copy of the
// proto package needs to be updated.
const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
type TarsumBackup struct {
Hashes map[string]string `protobuf:"bytes,1,rep,name=Hashes" json:"Hashes,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
}
func (m *TarsumBackup) Reset() { *m = TarsumBackup{} }
func (*TarsumBackup) ProtoMessage() {}
func (*TarsumBackup) Descriptor() ([]byte, []int) { return fileDescriptorTarsum, []int{0} }
func (m *TarsumBackup) GetHashes() map[string]string {
if m != nil {
return m.Hashes
}
return nil
}
func init() {
proto.RegisterType((*TarsumBackup)(nil), "remotecontext.TarsumBackup")
}
func (this *TarsumBackup) Equal(that interface{}) bool {
if that == nil {
if this == nil {
return true
}
return false
}
that1, ok := that.(*TarsumBackup)
if !ok {
that2, ok := that.(TarsumBackup)
if ok {
that1 = &that2
} else {
return false
}
}
if that1 == nil {
if this == nil {
return true
}
return false
} else if this == nil {
return false
}
if len(this.Hashes) != len(that1.Hashes) {
return false
}
for i := range this.Hashes {
if this.Hashes[i] != that1.Hashes[i] {
return false
}
}
return true
}
func (this *TarsumBackup) GoString() string {
if this == nil {
return "nil"
}
s := make([]string, 0, 5)
s = append(s, "&remotecontext.TarsumBackup{")
keysForHashes := make([]string, 0, len(this.Hashes))
for k, _ := range this.Hashes {
keysForHashes = append(keysForHashes, k)
}
github_com_gogo_protobuf_sortkeys.Strings(keysForHashes)
mapStringForHashes := "map[string]string{"
for _, k := range keysForHashes {
mapStringForHashes += fmt.Sprintf("%#v: %#v,", k, this.Hashes[k])
}
mapStringForHashes += "}"
if this.Hashes != nil {
s = append(s, "Hashes: "+mapStringForHashes+",\n")
}
s = append(s, "}")
return strings.Join(s, "")
}
func valueToGoStringTarsum(v interface{}, typ string) string {
rv := reflect.ValueOf(v)
if rv.IsNil() {
return "nil"
}
pv := reflect.Indirect(rv).Interface()
return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv)
}
func (m *TarsumBackup) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *TarsumBackup) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if len(m.Hashes) > 0 {
for k, _ := range m.Hashes {
dAtA[i] = 0xa
i++
v := m.Hashes[k]
mapSize := 1 + len(k) + sovTarsum(uint64(len(k))) + 1 + len(v) + sovTarsum(uint64(len(v)))
i = encodeVarintTarsum(dAtA, i, uint64(mapSize))
dAtA[i] = 0xa
i++
i = encodeVarintTarsum(dAtA, i, uint64(len(k)))
i += copy(dAtA[i:], k)
dAtA[i] = 0x12
i++
i = encodeVarintTarsum(dAtA, i, uint64(len(v)))
i += copy(dAtA[i:], v)
}
}
return i, nil
}
func encodeFixed64Tarsum(dAtA []byte, offset int, v uint64) int {
dAtA[offset] = uint8(v)
dAtA[offset+1] = uint8(v >> 8)
dAtA[offset+2] = uint8(v >> 16)
dAtA[offset+3] = uint8(v >> 24)
dAtA[offset+4] = uint8(v >> 32)
dAtA[offset+5] = uint8(v >> 40)
dAtA[offset+6] = uint8(v >> 48)
dAtA[offset+7] = uint8(v >> 56)
return offset + 8
}
func encodeFixed32Tarsum(dAtA []byte, offset int, v uint32) int {
dAtA[offset] = uint8(v)
dAtA[offset+1] = uint8(v >> 8)
dAtA[offset+2] = uint8(v >> 16)
dAtA[offset+3] = uint8(v >> 24)
return offset + 4
}
func encodeVarintTarsum(dAtA []byte, offset int, v uint64) int {
for v >= 1<<7 {
dAtA[offset] = uint8(v&0x7f | 0x80)
v >>= 7
offset++
}
dAtA[offset] = uint8(v)
return offset + 1
}
func (m *TarsumBackup) Size() (n int) {
var l int
_ = l
if len(m.Hashes) > 0 {
for k, v := range m.Hashes {
_ = k
_ = v
mapEntrySize := 1 + len(k) + sovTarsum(uint64(len(k))) + 1 + len(v) + sovTarsum(uint64(len(v)))
n += mapEntrySize + 1 + sovTarsum(uint64(mapEntrySize))
}
}
return n
}
func sovTarsum(x uint64) (n int) {
for {
n++
x >>= 7
if x == 0 {
break
}
}
return n
}
func sozTarsum(x uint64) (n int) {
return sovTarsum(uint64((x << 1) ^ uint64((int64(x) >> 63))))
}
func (this *TarsumBackup) String() string {
if this == nil {
return "nil"
}
keysForHashes := make([]string, 0, len(this.Hashes))
for k, _ := range this.Hashes {
keysForHashes = append(keysForHashes, k)
}
github_com_gogo_protobuf_sortkeys.Strings(keysForHashes)
mapStringForHashes := "map[string]string{"
for _, k := range keysForHashes {
mapStringForHashes += fmt.Sprintf("%v: %v,", k, this.Hashes[k])
}
mapStringForHashes += "}"
s := strings.Join([]string{`&TarsumBackup{`,
`Hashes:` + mapStringForHashes + `,`,
`}`,
}, "")
return s
}
func valueToStringTarsum(v interface{}) string {
rv := reflect.ValueOf(v)
if rv.IsNil() {
return "nil"
}
pv := reflect.Indirect(rv).Interface()
return fmt.Sprintf("*%v", pv)
}
func (m *TarsumBackup) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTarsum
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: TarsumBackup: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: TarsumBackup: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Hashes", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTarsum
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthTarsum
}
postIndex := iNdEx + msglen
if postIndex > l {
return io.ErrUnexpectedEOF
}
var keykey uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTarsum
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
keykey |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
var stringLenmapkey uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTarsum
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLenmapkey |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLenmapkey := int(stringLenmapkey)
if intStringLenmapkey < 0 {
return ErrInvalidLengthTarsum
}
postStringIndexmapkey := iNdEx + intStringLenmapkey
if postStringIndexmapkey > l {
return io.ErrUnexpectedEOF
}
mapkey := string(dAtA[iNdEx:postStringIndexmapkey])
iNdEx = postStringIndexmapkey
if m.Hashes == nil {
m.Hashes = make(map[string]string)
}
if iNdEx < postIndex {
var valuekey uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTarsum
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
valuekey |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
var stringLenmapvalue uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTarsum
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLenmapvalue |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLenmapvalue := int(stringLenmapvalue)
if intStringLenmapvalue < 0 {
return ErrInvalidLengthTarsum
}
postStringIndexmapvalue := iNdEx + intStringLenmapvalue
if postStringIndexmapvalue > l {
return io.ErrUnexpectedEOF
}
mapvalue := string(dAtA[iNdEx:postStringIndexmapvalue])
iNdEx = postStringIndexmapvalue
m.Hashes[mapkey] = mapvalue
} else {
var mapvalue string
m.Hashes[mapkey] = mapvalue
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipTarsum(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthTarsum
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func skipTarsum(dAtA []byte) (n int, err error) {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowTarsum
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
wireType := int(wire & 0x7)
switch wireType {
case 0:
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowTarsum
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
iNdEx++
if dAtA[iNdEx-1] < 0x80 {
break
}
}
return iNdEx, nil
case 1:
iNdEx += 8
return iNdEx, nil
case 2:
var length int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowTarsum
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
length |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
iNdEx += length
if length < 0 {
return 0, ErrInvalidLengthTarsum
}
return iNdEx, nil
case 3:
for {
var innerWire uint64
var start int = iNdEx
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowTarsum
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
innerWire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
innerWireType := int(innerWire & 0x7)
if innerWireType == 4 {
break
}
next, err := skipTarsum(dAtA[start:])
if err != nil {
return 0, err
}
iNdEx = start + next
}
return iNdEx, nil
case 4:
return iNdEx, nil
case 5:
iNdEx += 4
return iNdEx, nil
default:
return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
}
}
panic("unreachable")
}
var (
ErrInvalidLengthTarsum = fmt.Errorf("proto: negative length found during unmarshaling")
ErrIntOverflowTarsum = fmt.Errorf("proto: integer overflow")
)
func init() { proto.RegisterFile("tarsum.proto", fileDescriptorTarsum) }
var fileDescriptorTarsum = []byte{
// 196 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0x29, 0x49, 0x2c, 0x2a,
0x2e, 0xcd, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x2d, 0x4a, 0xcd, 0xcd, 0x2f, 0x49,
0x4d, 0xce, 0xcf, 0x2b, 0x49, 0xad, 0x28, 0x51, 0xea, 0x62, 0xe4, 0xe2, 0x09, 0x01, 0xcb, 0x3b,
0x25, 0x26, 0x67, 0x97, 0x16, 0x08, 0xd9, 0x73, 0xb1, 0x79, 0x24, 0x16, 0x67, 0xa4, 0x16, 0x4b,
0x30, 0x2a, 0x30, 0x6b, 0x70, 0x1b, 0xa9, 0xeb, 0xa1, 0x68, 0xd0, 0x43, 0x56, 0xac, 0x07, 0x51,
0xe9, 0x9a, 0x57, 0x52, 0x54, 0x19, 0x04, 0xd5, 0x26, 0x65, 0xc9, 0xc5, 0x8d, 0x24, 0x2c, 0x24,
0xc0, 0xc5, 0x9c, 0x9d, 0x5a, 0x29, 0xc1, 0xa8, 0xc0, 0xa8, 0xc1, 0x19, 0x04, 0x62, 0x0a, 0x89,
0x70, 0xb1, 0x96, 0x25, 0xe6, 0x94, 0xa6, 0x4a, 0x30, 0x81, 0xc5, 0x20, 0x1c, 0x2b, 0x26, 0x0b,
0x46, 0x27, 0x9d, 0x0b, 0x0f, 0xe5, 0x18, 0x6e, 0x3c, 0x94, 0x63, 0xf8, 0xf0, 0x50, 0x8e, 0xb1,
0xe1, 0x91, 0x1c, 0xe3, 0x8a, 0x47, 0x72, 0x8c, 0x27, 0x1e, 0xc9, 0x31, 0x5e, 0x78, 0x24, 0xc7,
0xf8, 0xe0, 0x91, 0x1c, 0xe3, 0x8b, 0x47, 0x72, 0x0c, 0x1f, 0x1e, 0xc9, 0x31, 0x4e, 0x78, 0x2c,
0xc7, 0x90, 0xc4, 0x06, 0xf6, 0x90, 0x31, 0x20, 0x00, 0x00, 0xff, 0xff, 0x89, 0x57, 0x7d, 0x3f,
0xe0, 0x00, 0x00, 0x00,
}

7
components/engine/builder/remotecontext/tarsum.proto Normal file
View File

@ -0,0 +1,7 @@
syntax = "proto3";
package remotecontext; // no namespace because only used internally
message TarsumBackup {
map<string, string> Hashes = 1;
}

38
components/engine/builder/remotecontext/tarsum_test.go
View File

@ -9,6 +9,7 @@ import (
"github.com/docker/docker/builder"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/reexec"
"github.com/pkg/errors"
)
const (
@ -22,24 +23,22 @@ func init() {
func TestCloseRootDirectory(t *testing.T) {
contextDir, err := ioutil.TempDir("", "builder-tarsum-test")
defer os.RemoveAll(contextDir)
if err != nil {
t.Fatalf("Error with creating temporary directory: %s", err)
}
tarsum := &tarSumContext{root: contextDir}
err = tarsum.Close()
src := makeTestArchiveContext(t, contextDir)
err = src.Close()
if err != nil {
t.Fatalf("Error while executing Close: %s", err)
}
_, err = os.Stat(contextDir)
_, err = os.Stat(src.Root())
if !os.IsNotExist(err) {
t.Fatal("Directory should not exist at this point")
defer os.RemoveAll(contextDir)
}
}
@ -49,7 +48,7 @@ func TestHashFile(t *testing.T) {
createTestTempFile(t, contextDir, filename, contents, 0755)
tarSum := makeTestTarsumContext(t, contextDir)
tarSum := makeTestArchiveContext(t, contextDir)
sum, err := tarSum.Hash(filename)
@ -80,7 +79,7 @@ func TestHashSubdir(t *testing.T) {
testFilename := createTestTempFile(t, contextSubdir, filename, contents, 0755)
tarSum := makeTestTarsumContext(t, contextDir)
tarSum := makeTestArchiveContext(t, contextDir)
relativePath, err := filepath.Rel(contextDir, testFilename)
@ -109,11 +108,9 @@ func TestStatNotExisting(t *testing.T) {
contextDir, cleanup := createTestTempDir(t, "", "builder-tarsum-test")
defer cleanup()
tarSum := &tarSumContext{root: contextDir}
_, err := tarSum.Hash("not-existing")
if !os.IsNotExist(err) {
src := makeTestArchiveContext(t, contextDir)
_, err := src.Hash("not-existing")
if !os.IsNotExist(errors.Cause(err)) {
t.Fatalf("This file should not exist: %s", err)
}
}
@ -130,30 +127,31 @@ func TestRemoveDirectory(t *testing.T) {
t.Fatalf("Error when getting relative path: %s", err)
}
tarSum := &tarSumContext{root: contextDir}
src := makeTestArchiveContext(t, contextDir)
tarSum := src.(modifiableContext)
err = tarSum.Remove(relativePath)
if err != nil {
t.Fatalf("Error when executing Remove: %s", err)
}
_, err = os.Stat(contextSubdir)
_, err = src.Hash(contextSubdir)
if !os.IsNotExist(err) {
if !os.IsNotExist(errors.Cause(err)) {
t.Fatal("Directory should not exist at this point")
}
}
func makeTestTarsumContext(t *testing.T, dir string) builder.Source {
func makeTestArchiveContext(t *testing.T, dir string) builder.Source {
tarStream, err := archive.Tar(dir, archive.Uncompressed)
if err != nil {
t.Fatalf("error: %s", err)
}
defer tarStream.Close()
tarSum, err := MakeTarSumContext(tarStream)
tarSum, err := FromArchive(tarStream)
if err != nil {
t.Fatalf("Error when executing MakeTarSumContext: %s", err)
t.Fatalf("Error when executing FromArchive: %s", err)
}
return tarSum
}

6
components/engine/cli/flags/configdir.go → components/engine/cli/config/configdir.go
View File

@ -1,4 +1,4 @@
package flags
package config
import (
"os"
@ -12,9 +12,9 @@ var (
configFileDir = ".docker"
)
// ConfigurationDir returns the path to the configuration directory as specified by the DOCKER_CONFIG environment variable.
// Dir returns the path to the configuration directory as specified by the DOCKER_CONFIG environment variable.
// TODO: this was copied from cli/config/configfile and should be removed once cmd/dockerd moves
func ConfigurationDir() string {
func Dir() string {
return configDir
}

13
components/engine/cli/flags/client.go
View File

@ -1,13 +0,0 @@
package flags
// ClientOptions are the options used to configure the client cli
type ClientOptions struct {
Common *CommonOptions
ConfigDir string
Version bool
}
// NewClientOptions returns a new ClientOptions
func NewClientOptions() *ClientOptions {
return &ClientOptions{Common: NewCommonOptions()}
}

69
components/engine/cli/required.go
View File

@ -25,72 +25,3 @@ func NoArgs(cmd *cobra.Command, args []string) error {
cmd.Short,
)
}
// RequiresMinArgs returns an error if there is not at least min args
func RequiresMinArgs(min int) cobra.PositionalArgs {
return func(cmd *cobra.Command, args []string) error {
if len(args) >= min {
return nil
}
return errors.Errorf(
"\"%s\" requires at least %d argument(s).\nSee '%s --help'.\n\nUsage: %s\n\n%s",
cmd.CommandPath(),
min,
cmd.CommandPath(),
cmd.UseLine(),
cmd.Short,
)
}
}
// RequiresMaxArgs returns an error if there is not at most max args
func RequiresMaxArgs(max int) cobra.PositionalArgs {
return func(cmd *cobra.Command, args []string) error {
if len(args) <= max {
return nil
}
return errors.Errorf(
"\"%s\" requires at most %d argument(s).\nSee '%s --help'.\n\nUsage: %s\n\n%s",
cmd.CommandPath(),
max,
cmd.CommandPath(),
cmd.UseLine(),
cmd.Short,
)
}
}
// RequiresRangeArgs returns an error if there is not at least min args and at most max args
func RequiresRangeArgs(min int, max int) cobra.PositionalArgs {
return func(cmd *cobra.Command, args []string) error {
if len(args) >= min && len(args) <= max {
return nil
}
return errors.Errorf(
"\"%s\" requires at least %d and at most %d argument(s).\nSee '%s --help'.\n\nUsage: %s\n\n%s",
cmd.CommandPath(),
min,
max,
cmd.CommandPath(),
cmd.UseLine(),
cmd.Short,
)
}
}
// ExactArgs returns an error if there is not the exact number of args
func ExactArgs(number int) cobra.PositionalArgs {
return func(cmd *cobra.Command, args []string) error {
if len(args) == number {
return nil
}
return errors.Errorf(
"\"%s\" requires exactly %d argument(s).\nSee '%s --help'.\n\nUsage: %s\n\n%s",
cmd.CommandPath(),
number,
cmd.CommandPath(),
cmd.UseLine(),
cmd.Short,
)
}
}

30
components/engine/client/build_prune.go Normal file
View File

@ -0,0 +1,30 @@
package client
import (
"encoding/json"
"fmt"
"github.com/docker/docker/api/types"
"golang.org/x/net/context"
)
// BuildCachePrune requests the daemon to delete unused cache data
func (cli *Client) BuildCachePrune(ctx context.Context) (*types.BuildCachePruneReport, error) {
if err := cli.NewVersionError("1.31", "build prune"); err != nil {
return nil, err
}
report := types.BuildCachePruneReport{}
serverResp, err := cli.post(ctx, "/build/prune", nil, nil, nil)
if err != nil {
return nil, err
}
defer ensureReaderClosed(serverResp)
if err := json.NewDecoder(serverResp.body).Decode(&report); err != nil {
return nil, fmt.Errorf("Error retrieving disk usage: %v", err)
}
return &report, nil
}

33
components/engine/client/client.go
View File

@ -55,8 +55,11 @@ import (
"strings"
"github.com/docker/docker/api"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/versions"
"github.com/docker/go-connections/sockets"
"github.com/docker/go-connections/tlsconfig"
"golang.org/x/net/context"
)
// ErrRedirect is the error returned by checkRedirect when the request is non-GET.
@ -216,9 +219,9 @@ func (cli *Client) getAPIPath(p string, query url.Values) string {
var apiPath string
if cli.version != "" {
v := strings.TrimPrefix(cli.version, "v")
apiPath = fmt.Sprintf("%s/v%s%s", cli.basePath, v, p)
apiPath = cli.basePath + "/v" + v + p
} else {
apiPath = fmt.Sprintf("%s%s", cli.basePath, p)
apiPath = cli.basePath + p
}
u := &url.URL{
@ -238,13 +241,29 @@ func (cli *Client) ClientVersion() string {
return cli.version
}
// UpdateClientVersion updates the version string associated with this
// instance of the Client. This operation doesn't acquire a mutex.
func (cli *Client) UpdateClientVersion(v string) {
if !cli.manualOverride {
cli.version = v
// NegotiateAPIVersion updates the version string associated with this
// instance of the Client to match the latest version the server supports
func (cli *Client) NegotiateAPIVersion(ctx context.Context) {
ping, _ := cli.Ping(ctx)
cli.NegotiateAPIVersionPing(ping)
}
// NegotiateAPIVersionPing updates the version string associated with this
// instance of the Client to match the latest version the server supports
func (cli *Client) NegotiateAPIVersionPing(p types.Ping) {
if cli.manualOverride {
return
}
// try the latest version before versioning headers existed
if p.APIVersion == "" {
p.APIVersion = "1.24"
}
// if server version is lower than the current cli, downgrade
if versions.LessThan(p.APIVersion, cli.ClientVersion()) {
cli.version = p.APIVersion
}
}
// DaemonHost returns the host associated with this instance of the Client.

254
components/engine/client/client_test.go
View File

@ -2,8 +2,6 @@ package client
import (
"bytes"
"encoding/json"
"io/ioutil"
"net/http"
"net/url"
"os"
@ -14,7 +12,6 @@ import (
"github.com/docker/docker/api"
"github.com/docker/docker/api/types"
"github.com/stretchr/testify/assert"
"golang.org/x/net/context"
)
func TestNewEnvClient(t *testing.T) {
@ -81,57 +78,27 @@ func TestNewEnvClient(t *testing.T) {
expectedVersion: "1.22",
},
}
env := envToMap()
defer mapToEnv(env)
for _, c := range cases {
recoverEnvs := setupEnvs(t, c.envs)
mapToEnv(env)
mapToEnv(c.envs)
apiclient, err := NewEnvClient()
if c.expectedError != "" {
if err == nil {
t.Errorf("expected an error for %v", c)
} else if err.Error() != c.expectedError {
t.Errorf("expected an error %s, got %s, for %v", c.expectedError, err.Error(), c)
}
assert.Error(t, err)
assert.Equal(t, c.expectedError, err.Error())
} else {
if err != nil {
t.Error(err)
}
assert.NoError(t, err)
version := apiclient.ClientVersion()
if version != c.expectedVersion {
t.Errorf("expected %s, got %s, for %v", c.expectedVersion, version, c)
}
assert.Equal(t, c.expectedVersion, version)
}
if c.envs["DOCKER_TLS_VERIFY"] != "" {
// pedantic checking that this is handled correctly
tr := apiclient.client.Transport.(*http.Transport)
if tr.TLSClientConfig == nil {
t.Error("no TLS config found when DOCKER_TLS_VERIFY enabled")
}
if tr.TLSClientConfig.InsecureSkipVerify {
t.Error("TLS verification should be enabled")
}
}
recoverEnvs(t)
}
}
func setupEnvs(t *testing.T, envs map[string]string) func(*testing.T) {
oldEnvs := map[string]string{}
for key, value := range envs {
oldEnv := os.Getenv(key)
oldEnvs[key] = oldEnv
err := os.Setenv(key, value)
if err != nil {
t.Error(err)
}
}
return func(t *testing.T) {
for key, value := range oldEnvs {
err := os.Setenv(key, value)
if err != nil {
t.Error(err)
}
assert.NotNil(t, tr.TLSClientConfig)
assert.Equal(t, tr.TLSClientConfig.InsecureSkipVerify, false)
}
}
}
@ -161,14 +128,10 @@ func TestGetAPIPath(t *testing.T) {
t.Fatal(err)
}
g := c.getAPIPath(cs.p, cs.q)
if g != cs.e {
t.Fatalf("Expected %s, got %s", cs.e, g)
}
assert.Equal(t, g, cs.e)
err = c.Close()
if nil != err {
t.Fatalf("close client failed, error message: %s", err)
}
assert.NoError(t, err)
}
}
@ -189,84 +152,33 @@ func TestParseHost(t *testing.T) {
for _, cs := range cases {
p, a, b, e := ParseHost(cs.host)
if cs.err && e == nil {
t.Fatalf("expected error, got nil")
}
if !cs.err && e != nil {
t.Fatal(e)
}
if cs.proto != p {
t.Fatalf("expected proto %s, got %s", cs.proto, p)
}
if cs.addr != a {
t.Fatalf("expected addr %s, got %s", cs.addr, a)
}
if cs.base != b {
t.Fatalf("expected base %s, got %s", cs.base, b)
}
}
}
func TestUpdateClientVersion(t *testing.T) {
client := &Client{
client: newMockClient(func(req *http.Request) (*http.Response, error) {
splitQuery := strings.Split(req.URL.Path, "/")
queryVersion := splitQuery[1]
b, err := json.Marshal(types.Version{
APIVersion: queryVersion,
})
if err != nil {
return nil, err
}
return &http.Response{
StatusCode: http.StatusOK,
Body: ioutil.NopCloser(bytes.NewReader(b)),
}, nil
}),
}
cases := []struct {
v string
}{
{"1.20"},
{"v1.21"},
{"1.22"},
{"v1.22"},
}
for _, cs := range cases {
client.UpdateClientVersion(cs.v)
r, err := client.ServerVersion(context.Background())
if err != nil {
t.Fatal(err)
}
if strings.TrimPrefix(r.APIVersion, "v") != strings.TrimPrefix(cs.v, "v") {
t.Fatalf("Expected %s, got %s", cs.v, r.APIVersion)
// if we expected an error to be returned...
if cs.err {
assert.Error(t, e)
}
assert.Equal(t, cs.proto, p)
assert.Equal(t, cs.addr, a)
assert.Equal(t, cs.base, b)
}
}
func TestNewEnvClientSetsDefaultVersion(t *testing.T) {
// Unset environment variables
envVarKeys := []string{
"DOCKER_HOST",
"DOCKER_API_VERSION",
"DOCKER_TLS_VERIFY",
"DOCKER_CERT_PATH",
}
envVarValues := make(map[string]string)
for _, key := range envVarKeys {
envVarValues[key] = os.Getenv(key)
os.Setenv(key, "")
env := envToMap()
defer mapToEnv(env)
envMap := map[string]string{
"DOCKER_HOST": "",
"DOCKER_API_VERSION": "",
"DOCKER_TLS_VERIFY": "",
"DOCKER_CERT_PATH": "",
}
mapToEnv(envMap)
client, err := NewEnvClient()
if err != nil {
t.Fatal(err)
}
if client.version != api.DefaultVersion {
t.Fatalf("Expected %s, got %s", api.DefaultVersion, client.version)
}
assert.Equal(t, client.version, api.DefaultVersion)
expected := "1.22"
os.Setenv("DOCKER_API_VERSION", expected)
@ -274,14 +186,112 @@ func TestNewEnvClientSetsDefaultVersion(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if client.version != expected {
t.Fatalf("Expected %s, got %s", expected, client.version)
assert.Equal(t, expected, client.version)
}
// TestNegotiateAPIVersionEmpty asserts that client.Client can
// negotiate a compatible APIVersion when omitted
func TestNegotiateAPIVersionEmpty(t *testing.T) {
env := envToMap()
defer mapToEnv(env)
envMap := map[string]string{
"DOCKER_API_VERSION": "",
}
mapToEnv(envMap)
client, err := NewEnvClient()
if err != nil {
t.Fatal(err)
}
// Restore environment variables
for _, key := range envVarKeys {
os.Setenv(key, envVarValues[key])
ping := types.Ping{
APIVersion: "",
OSType: "linux",
Experimental: false,
}
// set our version to something new
client.version = "1.25"
// if no version from server, expect the earliest
// version before APIVersion was implemented
expected := "1.24"
// test downgrade
client.NegotiateAPIVersionPing(ping)
assert.Equal(t, expected, client.version)
}
// TestNegotiateAPIVersion asserts that client.Client can
// negotiate a compatible APIVersion with the server
func TestNegotiateAPIVersion(t *testing.T) {
client, err := NewEnvClient()
if err != nil {
t.Fatal(err)
}
expected := "1.21"
ping := types.Ping{
APIVersion: expected,
OSType: "linux",
Experimental: false,
}
// set our version to something new
client.version = "1.22"
// test downgrade
client.NegotiateAPIVersionPing(ping)
assert.Equal(t, expected, client.version)
}
// TestNegotiateAPIVersionOverride asserts that we honor
// the environment variable DOCKER_API_VERSION when negotianing versions
func TestNegotiateAPVersionOverride(t *testing.T) {
env := envToMap()
defer mapToEnv(env)
envMap := map[string]string{
"DOCKER_API_VERSION": "9.99",
}
mapToEnv(envMap)
client, err := NewEnvClient()
if err != nil {
t.Fatal(err)
}
ping := types.Ping{
APIVersion: "1.24",
OSType: "linux",
Experimental: false,
}
expected := envMap["DOCKER_API_VERSION"]
// test that we honored the env var
client.NegotiateAPIVersionPing(ping)
assert.Equal(t, expected, client.version)
}
// mapToEnv takes a map of environment variables and sets them
func mapToEnv(env map[string]string) {
for k, v := range env {
os.Setenv(k, v)
}
}
// envToMap returns a map of environment variables
func envToMap() map[string]string {
env := make(map[string]string)
for _, e := range os.Environ() {
kv := strings.SplitAfterN(e, "=", 2)
env[kv[0]] = kv[1]
}
return env
}
type roundTripFunc func(*http.Request) (*http.Response, error)

3
components/engine/client/config_create.go
View File

@ -11,6 +11,9 @@ import (
// ConfigCreate creates a new Config.
func (cli *Client) ConfigCreate(ctx context.Context, config swarm.ConfigSpec) (types.ConfigCreateResponse, error) {
var response types.ConfigCreateResponse
if err := cli.NewVersionError("1.30", "config create"); err != nil {
return response, err
}
resp, err := cli.post(ctx, "/configs/create", nil, config, nil)
if err != nil {
return response, err

16
components/engine/client/config_create_test.go
View File

@ -11,12 +11,23 @@ import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/swarm"
"github.com/stretchr/testify/assert"
"golang.org/x/net/context"
)
func TestConfigCreateUnsupported(t *testing.T) {
client := &Client{
version: "1.29",
client: &http.Client{},
}
_, err := client.ConfigCreate(context.Background(), swarm.ConfigSpec{})
assert.EqualError(t, err, `"config create" requires API version 1.30, but the Docker daemon API version is 1.29`)
}
func TestConfigCreateError(t *testing.T) {
client := &Client{
client: newMockClient(errorMock(http.StatusInternalServerError, "Server error")),
version: "1.30",
client: newMockClient(errorMock(http.StatusInternalServerError, "Server error")),
}
_, err := client.ConfigCreate(context.Background(), swarm.ConfigSpec{})
if err == nil || err.Error() != "Error response from daemon: Server error" {
@ -25,8 +36,9 @@ func TestConfigCreateError(t *testing.T) {
}
func TestConfigCreate(t *testing.T) {
expectedURL := "/configs/create"
expectedURL := "/v1.30/configs/create"
client := &Client{
version: "1.30",
client: newMockClient(func(req *http.Request) (*http.Response, error) {
if !strings.HasPrefix(req.URL.Path, expectedURL) {
return nil, fmt.Errorf("Expected URL '%s', got '%s'", expectedURL, req.URL)

3
components/engine/client/config_inspect.go
View File

@ -12,6 +12,9 @@ import (
// ConfigInspectWithRaw returns the config information with raw data
func (cli *Client) ConfigInspectWithRaw(ctx context.Context, id string) (swarm.Config, []byte, error) {
if err := cli.NewVersionError("1.30", "config inspect"); err != nil {
return swarm.Config{}, nil, err
}
resp, err := cli.get(ctx, "/configs/"+id, nil, nil)
if err != nil {
if resp.statusCode == http.StatusNotFound {

19
components/engine/client/config_inspect_test.go
View File

@ -10,12 +10,23 @@ import (
"testing"
"github.com/docker/docker/api/types/swarm"
"github.com/stretchr/testify/assert"
"golang.org/x/net/context"
)
func TestConfigInspectUnsupported(t *testing.T) {
client := &Client{
version: "1.29",
client: &http.Client{},
}
_, _, err := client.ConfigInspectWithRaw(context.Background(), "nothing")
assert.EqualError(t, err, `"config inspect" requires API version 1.30, but the Docker daemon API version is 1.29`)
}
func TestConfigInspectError(t *testing.T) {
client := &Client{
client: newMockClient(errorMock(http.StatusInternalServerError, "Server error")),
version: "1.30",
client: newMockClient(errorMock(http.StatusInternalServerError, "Server error")),
}
_, _, err := client.ConfigInspectWithRaw(context.Background(), "nothing")
@ -26,7 +37,8 @@ func TestConfigInspectError(t *testing.T) {
func TestConfigInspectConfigNotFound(t *testing.T) {
client := &Client{
client: newMockClient(errorMock(http.StatusNotFound, "Server error")),
version: "1.30",
client: newMockClient(errorMock(http.StatusNotFound, "Server error")),
}
_, _, err := client.ConfigInspectWithRaw(context.Background(), "unknown")
@ -36,8 +48,9 @@ func TestConfigInspectConfigNotFound(t *testing.T) {
}
func TestConfigInspect(t *testing.T) {
expectedURL := "/configs/config_id"
expectedURL := "/v1.30/configs/config_id"
client := &Client{
version: "1.30",
client: newMockClient(func(req *http.Request) (*http.Response, error) {
if !strings.HasPrefix(req.URL.Path, expectedURL) {
return nil, fmt.Errorf("Expected URL '%s', got '%s'", expectedURL, req.URL)

3
components/engine/client/config_list.go
View File

@ -12,6 +12,9 @@ import (
// ConfigList returns the list of configs.
func (cli *Client) ConfigList(ctx context.Context, options types.ConfigListOptions) ([]swarm.Config, error) {
if err := cli.NewVersionError("1.30", "config list"); err != nil {
return nil, err
}
query := url.Values{}
if options.Filters.Len() > 0 {

16
components/engine/client/config_list_test.go
View File

@ -12,12 +12,23 @@ import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/api/types/swarm"
"github.com/stretchr/testify/assert"
"golang.org/x/net/context"
)
func TestConfigListUnsupported(t *testing.T) {
client := &Client{
version: "1.29",
client: &http.Client{},
}
_, err := client.ConfigList(context.Background(), types.ConfigListOptions{})
assert.EqualError(t, err, `"config list" requires API version 1.30, but the Docker daemon API version is 1.29`)
}
func TestConfigListError(t *testing.T) {
client := &Client{
client: newMockClient(errorMock(http.StatusInternalServerError, "Server error")),
version: "1.30",
client: newMockClient(errorMock(http.StatusInternalServerError, "Server error")),
}
_, err := client.ConfigList(context.Background(), types.ConfigListOptions{})
@ -27,7 +38,7 @@ func TestConfigListError(t *testing.T) {
}
func TestConfigList(t *testing.T) {
expectedURL := "/configs"
expectedURL := "/v1.30/configs"
filters := filters.NewArgs()
filters.Add("label", "label1")
@ -54,6 +65,7 @@ func TestConfigList(t *testing.T) {
}
for _, listCase := range listCases {
client := &Client{
version: "1.30",
client: newMockClient(func(req *http.Request) (*http.Response, error) {
if !strings.HasPrefix(req.URL.Path, expectedURL) {
return nil, fmt.Errorf("Expected URL '%s', got '%s'", expectedURL, req.URL)

3
components/engine/client/config_remove.go
View File

@ -4,6 +4,9 @@ import "golang.org/x/net/context"
// ConfigRemove removes a Config.
func (cli *Client) ConfigRemove(ctx context.Context, id string) error {
if err := cli.NewVersionError("1.30", "config remove"); err != nil {
return err
}
resp, err := cli.delete(ctx, "/configs/"+id, nil, nil)
ensureReaderClosed(resp)
return err

16
components/engine/client/config_remove_test.go
View File

@ -8,12 +8,23 @@ import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
"golang.org/x/net/context"
)
func TestConfigRemoveUnsupported(t *testing.T) {
client := &Client{
version: "1.29",
client: &http.Client{},
}
err := client.ConfigRemove(context.Background(), "config_id")
assert.EqualError(t, err, `"config remove" requires API version 1.30, but the Docker daemon API version is 1.29`)
}
func TestConfigRemoveError(t *testing.T) {
client := &Client{
client: newMockClient(errorMock(http.StatusInternalServerError, "Server error")),
version: "1.30",
client: newMockClient(errorMock(http.StatusInternalServerError, "Server error")),
}
err := client.ConfigRemove(context.Background(), "config_id")
@ -23,9 +34,10 @@ func TestConfigRemoveError(t *testing.T) {
}
func TestConfigRemove(t *testing.T) {
expectedURL := "/configs/config_id"
expectedURL := "/v1.30/configs/config_id"
client := &Client{
version: "1.30",
client: newMockClient(func(req *http.Request) (*http.Response, error) {
if !strings.HasPrefix(req.URL.Path, expectedURL) {
return nil, fmt.Errorf("Expected URL '%s', got '%s'", expectedURL, req.URL)

3
components/engine/client/config_update.go
View File

@ -10,6 +10,9 @@ import (
// ConfigUpdate attempts to update a Config
func (cli *Client) ConfigUpdate(ctx context.Context, id string, version swarm.Version, config swarm.ConfigSpec) error {
if err := cli.NewVersionError("1.30", "config update"); err != nil {
return err
}
query := url.Values{}
query.Set("version", strconv.FormatUint(version.Index, 10))
resp, err := cli.post(ctx, "/configs/"+id+"/update", query, config, nil)

19
components/engine/client/config_update_test.go
View File

@ -8,14 +8,24 @@ import (
"strings"
"testing"
"golang.org/x/net/context"
"github.com/docker/docker/api/types/swarm"
"github.com/stretchr/testify/assert"
"golang.org/x/net/context"
)
func TestConfigUpdateUnsupported(t *testing.T) {
client := &Client{
version: "1.29",
client: &http.Client{},
}
err := client.ConfigUpdate(context.Background(), "config_id", swarm.Version{}, swarm.ConfigSpec{})
assert.EqualError(t, err, `"config update" requires API version 1.30, but the Docker daemon API version is 1.29`)
}
func TestConfigUpdateError(t *testing.T) {
client := &Client{
client: newMockClient(errorMock(http.StatusInternalServerError, "Server error")),
version: "1.30",
client: newMockClient(errorMock(http.StatusInternalServerError, "Server error")),
}
err := client.ConfigUpdate(context.Background(), "config_id", swarm.Version{}, swarm.ConfigSpec{})
@ -25,9 +35,10 @@ func TestConfigUpdateError(t *testing.T) {
}
func TestConfigUpdate(t *testing.T) {
expectedURL := "/configs/config_id/update"
expectedURL := "/v1.30/configs/config_id/update"
client := &Client{
version: "1.30",
client: newMockClient(func(req *http.Request) (*http.Response, error) {
if !strings.HasPrefix(req.URL.Path, expectedURL) {
return nil, fmt.Errorf("Expected URL '%s', got '%s'", expectedURL, req.URL)

6
components/engine/client/container_copy.go
View File

@ -20,7 +20,7 @@ func (cli *Client) ContainerStatPath(ctx context.Context, containerID, path stri
query := url.Values{}
query.Set("path", filepath.ToSlash(path)) // Normalize the paths used in the API.
urlStr := fmt.Sprintf("/containers/%s/archive", containerID)
urlStr := "/containers/" + containerID + "/archive"
response, err := cli.head(ctx, urlStr, query, nil)
if err != nil {
return types.ContainerPathStat{}, err
@ -42,7 +42,7 @@ func (cli *Client) CopyToContainer(ctx context.Context, container, path string,
query.Set("copyUIDGID", "true")
}
apiPath := fmt.Sprintf("/containers/%s/archive", container)
apiPath := "/containers/" + container + "/archive"
response, err := cli.putRaw(ctx, apiPath, query, content, nil)
if err != nil {
@ -63,7 +63,7 @@ func (cli *Client) CopyFromContainer(ctx context.Context, container, srcPath str
query := make(url.Values, 1)
query.Set("path", filepath.ToSlash(srcPath)) // Normalize the paths used in the API.
apiPath := fmt.Sprintf("/containers/%s/archive", container)
apiPath := "/containers/" + container + "/archive"
response, err := cli.get(ctx, apiPath, query, nil)
if err != nil {
return nil, types.ContainerPathStat{}, err

8
components/engine/client/distribution_inspect.go
View File

@ -10,6 +10,12 @@ import (
// DistributionInspect returns the image digest with full Manifest
func (cli *Client) DistributionInspect(ctx context.Context, image, encodedRegistryAuth string) (registrytypes.DistributionInspect, error) {
// Contact the registry to retrieve digest and platform information
var distributionInspect registrytypes.DistributionInspect
if err := cli.NewVersionError("1.30", "distribution inspect"); err != nil {
return distributionInspect, err
}
var headers map[string][]string
if encodedRegistryAuth != "" {
@ -18,8 +24,6 @@ func (cli *Client) DistributionInspect(ctx context.Context, image, encodedRegist
}
}
// Contact the registry to retrieve digest and platform information
var distributionInspect registrytypes.DistributionInspect
resp, err := cli.get(ctx, "/distribution/"+image+"/json", url.Values{}, headers)
if err != nil {
return distributionInspect, err

18
components/engine/client/distribution_inspect_test.go Normal file
View File

@ -0,0 +1,18 @@
package client
import (
"net/http"
"testing"
"github.com/stretchr/testify/assert"
"golang.org/x/net/context"
)
func TestDistributionInspectUnsupported(t *testing.T) {
client := &Client{
version: "1.29",
client: &http.Client{},
}
_, err := client.DistributionInspect(context.Background(), "foobar:1.0", "")
assert.EqualError(t, err, `"distribution inspect" requires API version 1.30, but the Docker daemon API version is 1.29`)
}

85
components/engine/client/hijack.go
View File

@ -1,8 +1,8 @@
package client
import (
"bufio"
"crypto/tls"
"errors"
"fmt"
"net"
"net/http"
@ -14,6 +14,7 @@ import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/pkg/tlsconfig"
"github.com/docker/go-connections/sockets"
"github.com/pkg/errors"
"golang.org/x/net/context"
)
@ -46,37 +47,12 @@ func (cli *Client) postHijacked(ctx context.Context, path string, query url.Valu
}
req = cli.addHeaders(req, headers)
req.Host = cli.addr
req.Header.Set("Connection", "Upgrade")
req.Header.Set("Upgrade", "tcp")
conn, err := dial(cli.proto, cli.addr, resolveTLSConfig(cli.client.Transport))
conn, err := cli.setupHijackConn(req, "tcp")
if err != nil {
if strings.Contains(err.Error(), "connection refused") {
return types.HijackedResponse{}, fmt.Errorf("Cannot connect to the Docker daemon. Is 'docker daemon' running on this host?")
}
return types.HijackedResponse{}, err
}
// When we set up a TCP connection for hijack, there could be long periods
// of inactivity (a long running command with no output) that in certain
// network setups may cause ECONNTIMEOUT, leaving the client in an unknown
// state. Setting TCP KeepAlive on the socket connection will prohibit
// ECONNTIMEOUT unless the socket connection truly is broken
if tcpConn, ok := conn.(*net.TCPConn); ok {
tcpConn.SetKeepAlive(true)
tcpConn.SetKeepAlivePeriod(30 * time.Second)
}
clientconn := httputil.NewClientConn(conn, nil)
defer clientconn.Close()
// Server hijacks the connection, error 'connection closed' expected
_, err = clientconn.Do(req)
rwc, br := clientconn.Hijack()
return types.HijackedResponse{Conn: rwc, Reader: br}, err
return types.HijackedResponse{Conn: conn, Reader: bufio.NewReader(conn)}, err
}
func tlsDial(network, addr string, config *tls.Config) (net.Conn, error) {
@ -175,3 +151,56 @@ func dial(proto, addr string, tlsConfig *tls.Config) (net.Conn, error) {
}
return net.Dial(proto, addr)
}
func (cli *Client) setupHijackConn(req *http.Request, proto string) (net.Conn, error) {
req.Host = cli.addr
req.Header.Set("Connection", "Upgrade")
req.Header.Set("Upgrade", proto)
conn, err := dial(cli.proto, cli.addr, resolveTLSConfig(cli.client.Transport))
if err != nil {
return nil, errors.Wrap(err, "cannot connect to the Docker daemon. Is 'docker daemon' running on this host?")
}
// When we set up a TCP connection for hijack, there could be long periods
// of inactivity (a long running command with no output) that in certain
// network setups may cause ECONNTIMEOUT, leaving the client in an unknown
// state. Setting TCP KeepAlive on the socket connection will prohibit
// ECONNTIMEOUT unless the socket connection truly is broken
if tcpConn, ok := conn.(*net.TCPConn); ok {
tcpConn.SetKeepAlive(true)
tcpConn.SetKeepAlivePeriod(30 * time.Second)
}
clientconn := httputil.NewClientConn(conn, nil)
defer clientconn.Close()
// Server hijacks the connection, error 'connection closed' expected
resp, err := clientconn.Do(req)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusSwitchingProtocols {
resp.Body.Close()
return nil, fmt.Errorf("unable to upgrade to %s, received %d", proto, resp.StatusCode)
}
c, br := clientconn.Hijack()
if br.Buffered() > 0 {
// If there is buffered content, wrap the connection
c = &hijackedConn{c, br}
} else {
br.Reset(nil)
}
return c, nil
}
type hijackedConn struct {
net.Conn
r *bufio.Reader
}
func (c *hijackedConn) Read(b []byte) (int, error) {
return c.r.Read(b)
}

3
components/engine/client/image_build.go
View File

@ -120,6 +120,9 @@ func (cli *Client) imageBuildOptionsToQuery(options types.ImageBuildOptions) (ur
return query, err
}
query.Set("cachefrom", string(cacheFromJSON))
if options.SessionID != "" {
query.Set("session", options.SessionID)
}
return query, nil
}

10
components/engine/client/interface.go
View File

@ -2,6 +2,7 @@ package client
import (
"io"
"net"
"time"
"github.com/docker/docker/api/types"
@ -33,7 +34,9 @@ type CommonAPIClient interface {
ClientVersion() string
DaemonHost() string
ServerVersion(ctx context.Context) (types.Version, error)
UpdateClientVersion(v string)
NegotiateAPIVersion(ctx context.Context)
NegotiateAPIVersionPing(types.Ping)
DialSession(ctx context.Context, proto string, meta map[string][]string) (net.Conn, error)
}
// ContainerAPIClient defines API client methods for the containers
@ -79,6 +82,7 @@ type DistributionAPIClient interface {
// ImageAPIClient defines API client methods for the images
type ImageAPIClient interface {
ImageBuild(ctx context.Context, context io.Reader, options types.ImageBuildOptions) (types.ImageBuildResponse, error)
BuildCachePrune(ctx context.Context) (*types.BuildCachePruneReport, error)
ImageCreate(ctx context.Context, parentReference string, options types.ImageCreateOptions) (io.ReadCloser, error)
ImageHistory(ctx context.Context, image string) ([]image.HistoryResponseItem, error)
ImageImport(ctx context.Context, source types.ImageImportSource, ref string, options types.ImageImportOptions) (io.ReadCloser, error)
@ -99,8 +103,8 @@ type NetworkAPIClient interface {
NetworkConnect(ctx context.Context, networkID, container string, config *network.EndpointSettings) error
NetworkCreate(ctx context.Context, name string, options types.NetworkCreate) (types.NetworkCreateResponse, error)
NetworkDisconnect(ctx context.Context, networkID, container string, force bool) error
NetworkInspect(ctx context.Context, networkID string, verbose bool) (types.NetworkResource, error)
NetworkInspectWithRaw(ctx context.Context, networkID string, verbose bool) (types.NetworkResource, []byte, error)
NetworkInspect(ctx context.Context, networkID string, options types.NetworkInspectOptions) (types.NetworkResource, error)
NetworkInspectWithRaw(ctx context.Context, networkID string, options types.NetworkInspectOptions) (types.NetworkResource, []byte, error)
NetworkList(ctx context.Context, options types.NetworkListOptions) ([]types.NetworkResource, error)
NetworkRemove(ctx context.Context, networkID string) error
NetworksPrune(ctx context.Context, pruneFilter filters.Args) (types.NetworksPruneReport, error)

11
components/engine/client/network_inspect.go
View File

@ -12,22 +12,25 @@ import (
)
// NetworkInspect returns the information for a specific network configured in the docker host.
func (cli *Client) NetworkInspect(ctx context.Context, networkID string, verbose bool) (types.NetworkResource, error) {
networkResource, _, err := cli.NetworkInspectWithRaw(ctx, networkID, verbose)
func (cli *Client) NetworkInspect(ctx context.Context, networkID string, options types.NetworkInspectOptions) (types.NetworkResource, error) {
networkResource, _, err := cli.NetworkInspectWithRaw(ctx, networkID, options)
return networkResource, err
}
// NetworkInspectWithRaw returns the information for a specific network configured in the docker host and its raw representation.
func (cli *Client) NetworkInspectWithRaw(ctx context.Context, networkID string, verbose bool) (types.NetworkResource, []byte, error) {
func (cli *Client) NetworkInspectWithRaw(ctx context.Context, networkID string, options types.NetworkInspectOptions) (types.NetworkResource, []byte, error) {
var (
networkResource types.NetworkResource
resp serverResponse
err error
)
query := url.Values{}
if verbose {
if options.Verbose {
query.Set("verbose", "true")
}
if options.Scope != "" {
query.Set("scope", options.Scope)
}
resp, err = cli.get(ctx, "/networks/"+networkID, query, nil)
if err != nil {
if resp.statusCode == http.StatusNotFound {

21
components/engine/client/network_inspect_test.go
View File

@ -11,6 +11,7 @@ import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/network"
"github.com/stretchr/testify/assert"
"golang.org/x/net/context"
)
@ -19,7 +20,7 @@ func TestNetworkInspectError(t *testing.T) {
client: newMockClient(errorMock(http.StatusInternalServerError, "Server error")),
}
_, err := client.NetworkInspect(context.Background(), "nothing", false)
_, err := client.NetworkInspect(context.Background(), "nothing", types.NetworkInspectOptions{})
if err == nil || err.Error() != "Error response from daemon: Server error" {
t.Fatalf("expected a Server Error, got %v", err)
}
@ -30,7 +31,7 @@ func TestNetworkInspectContainerNotFound(t *testing.T) {
client: newMockClient(errorMock(http.StatusNotFound, "Server error")),
}
_, err := client.NetworkInspect(context.Background(), "unknown", false)
_, err := client.NetworkInspect(context.Background(), "unknown", types.NetworkInspectOptions{})
if err == nil || !IsErrNetworkNotFound(err) {
t.Fatalf("expected a networkNotFound error, got %v", err)
}
@ -51,7 +52,14 @@ func TestNetworkInspect(t *testing.T) {
content []byte
err error
)
if strings.HasPrefix(req.URL.RawQuery, "verbose=true") {
if strings.Contains(req.URL.RawQuery, "scope=global") {
return &http.Response{
StatusCode: http.StatusNotFound,
Body: ioutil.NopCloser(bytes.NewReader(content)),
}, nil
}
if strings.Contains(req.URL.RawQuery, "verbose=true") {
s := map[string]network.ServiceInfo{
"web": {},
}
@ -74,7 +82,7 @@ func TestNetworkInspect(t *testing.T) {
}),
}
r, err := client.NetworkInspect(context.Background(), "network_id", false)
r, err := client.NetworkInspect(context.Background(), "network_id", types.NetworkInspectOptions{})
if err != nil {
t.Fatal(err)
}
@ -82,7 +90,7 @@ func TestNetworkInspect(t *testing.T) {
t.Fatalf("expected `mynetwork`, got %s", r.Name)
}
r, err = client.NetworkInspect(context.Background(), "network_id", true)
r, err = client.NetworkInspect(context.Background(), "network_id", types.NetworkInspectOptions{Verbose: true})
if err != nil {
t.Fatal(err)
}
@ -93,4 +101,7 @@ func TestNetworkInspect(t *testing.T) {
if !ok {
t.Fatalf("expected service `web` missing in the verbose output")
}
_, err = client.NetworkInspect(context.Background(), "network_id", types.NetworkInspectOptions{Scope: "global"})
assert.EqualError(t, err, "Error: No such network: network_id")
}

Some files were not shown because too many files have changed in this diff Show More